botholomew 0.16.4 → 0.18.0

package/src/db/embeddings.ts

@@ -1,317 +0,0 @@
-import { EMBEDDING_DIMENSION } from "../constants.ts";
-import type { DbConnection } from "./connection.ts";
-
-if (!Number.isInteger(EMBEDDING_DIMENSION) || EMBEDDING_DIMENSION <= 0) {
-  throw new Error(`Invalid EMBEDDING_DIMENSION: ${EMBEDDING_DIMENSION}`);
-}
-
-/**
- * Disk-backed search index over `<projectDir>/context/`. One row per
- * `(path, chunk_index)`; `content_hash` is the file-level sha256 so the
- * reindex algorithm can detect adds, updates, and removals in one pass.
- */
-export interface IndexedChunk {
-  path: string;
-  chunk_index: number;
-  content_hash: string;
-  chunk_content: string;
-  embedding: number[];
-  mtime_ms: number;
-  size_bytes: number;
-  indexed_at: Date;
-}
-
-interface IndexRow {
-  path: string;
-  chunk_index: number;
-  content_hash: string;
-  chunk_content: string;
-  embedding: number[] | null;
-  mtime_ms: number;
-  size_bytes: number;
-  indexed_at: string;
-}
-
-function rowToChunk(row: IndexRow): IndexedChunk {
-  return {
-    path: row.path,
-    chunk_index: row.chunk_index,
-    content_hash: row.content_hash,
-    chunk_content: row.chunk_content,
-    embedding: row.embedding ?? [],
-    mtime_ms: Number(row.mtime_ms),
-    size_bytes: Number(row.size_bytes),
-    indexed_at: new Date(row.indexed_at),
-  };
-}
-
-export interface ChunkInput {
-  chunk_index: number;
-  chunk_content: string;
-  embedding: number[];
-}
-
-/**
- * Replace all rows for `path` with the supplied chunks. The file-level
- * `content_hash` / `mtime_ms` / `size_bytes` are stored on every row so a
- * subsequent reindex can short-circuit by comparing just those columns.
- */
-export async function upsertChunksForPath(
-  conn: DbConnection,
-  params: {
-    path: string;
-    contentHash: string;
-    mtimeMs: number;
-    sizeBytes: number;
-    chunks: ChunkInput[];
-  },
-): Promise<void> {
-  await conn.queryRun("DELETE FROM context_index WHERE path = ?1", params.path);
-  for (const c of params.chunks) {
-    await conn.queryRun(
-      `INSERT INTO context_index
-       (path, chunk_index, content_hash, chunk_content, embedding, mtime_ms, size_bytes, indexed_at)
-       VALUES (?1, ?2, ?3, ?4, ?5::FLOAT[${EMBEDDING_DIMENSION}], ?6, ?7, current_timestamp::VARCHAR)`,
-      params.path,
-      c.chunk_index,
-      params.contentHash,
-      c.chunk_content,
-      c.embedding,
-      params.mtimeMs,
-      params.sizeBytes,
-    );
-  }
-}
-
-export async function deleteIndexedPath(
-  conn: DbConnection,
-  path: string,
-): Promise<number> {
-  const result = await conn.queryRun(
-    "DELETE FROM context_index WHERE path = ?1",
-    path,
-  );
-  return result.changes;
-}
-
-/**
- * Remove every indexed entry whose path equals `prefix` or lives beneath
- * `prefix/`. Used when a folder is deleted from `context/` and we need to
- * drop all child entries in one shot.
- */
-export async function deleteIndexedPathsUnder(
-  conn: DbConnection,
-  prefix: string,
-): Promise<number> {
-  const result = await conn.queryRun(
-    "DELETE FROM context_index WHERE path = ?1 OR path LIKE ?2",
-    prefix,
-    `${prefix}/%`,
-  );
-  return result.changes;
-}
-
-export interface IndexedPathSummary {
-  path: string;
-  content_hash: string;
-  mtime_ms: number;
-  size_bytes: number;
-  chunk_count: number;
-}
-
-export async function listIndexedPaths(
-  conn: DbConnection,
-): Promise<IndexedPathSummary[]> {
-  const rows = await conn.queryAll<{
-    path: string;
-    content_hash: string;
-    mtime_ms: number;
-    size_bytes: number;
-    chunk_count: number;
-  }>(
-    `SELECT path,
-            ANY_VALUE(content_hash) AS content_hash,
-            ANY_VALUE(mtime_ms) AS mtime_ms,
-            ANY_VALUE(size_bytes) AS size_bytes,
-            COUNT(*) AS chunk_count
-       FROM context_index
-       GROUP BY path
-       ORDER BY path ASC`,
-  );
-  return rows.map((r) => ({
-    path: r.path,
-    content_hash: r.content_hash,
-    mtime_ms: Number(r.mtime_ms),
-    size_bytes: Number(r.size_bytes),
-    chunk_count: Number(r.chunk_count),
-  }));
-}
-
-export async function getIndexedPath(
-  conn: DbConnection,
-  path: string,
-): Promise<IndexedPathSummary | null> {
-  const row = await conn.queryGet<{
-    path: string;
-    content_hash: string;
-    mtime_ms: number;
-    size_bytes: number;
-    chunk_count: number;
-  }>(
-    `SELECT path,
-            ANY_VALUE(content_hash) AS content_hash,
-            ANY_VALUE(mtime_ms) AS mtime_ms,
-            ANY_VALUE(size_bytes) AS size_bytes,
-            COUNT(*) AS chunk_count
-       FROM context_index
-       WHERE path = ?1
-       GROUP BY path`,
-    path,
-  );
-  if (!row) return null;
-  return {
-    path: row.path,
-    content_hash: row.content_hash,
-    mtime_ms: Number(row.mtime_ms),
-    size_bytes: Number(row.size_bytes),
-    chunk_count: Number(row.chunk_count),
-  };
-}
-
-export interface SearchResult extends IndexedChunk {
-  score: number;
-}
-
-/**
- * Vector similarity over `context_index.embedding`. Returns chunks sorted by
- * cosine similarity (higher = closer). Skips rows whose embedding is NULL.
- */
-export async function searchSemantic(
-  conn: DbConnection,
-  queryEmbedding: number[],
-  limit = 10,
-): Promise<SearchResult[]> {
-  const rows = await conn.queryAll<IndexRow & { distance: number }>(
-    `SELECT *, array_cosine_distance(embedding, ?1::FLOAT[${EMBEDDING_DIMENSION}]) AS distance
-       FROM context_index
-       WHERE embedding IS NOT NULL
-       ORDER BY distance ASC
-       LIMIT ?2`,
-    queryEmbedding,
-    limit,
-  );
-  return rows.map((row) => ({
-    ...rowToChunk(row),
-    score: 1 - row.distance,
-  }));
-}
-
-/**
- * BM25 keyword search over (chunk_content, path). The FTS index is rebuilt
- * lazily by `rebuildSearchIndex`. Returns null-scoring rows filtered out.
- */
-export async function searchKeyword(
-  conn: DbConnection,
-  query: string,
-  limit = 10,
-): Promise<SearchResult[]> {
-  // The FTS index is created with `path` as input_id (see
-  // rebuildSearchIndex), so match_bm25's first argument must be the path
-  // value, not rowid. Passing rowid silently returns no hits — searchHybrid
-  // would then degrade to semantic-only.
-  const rows = await conn.queryAll<IndexRow & { score: number }>(
-    `SELECT context_index.*,
-            fts_main_context_index.match_bm25(context_index.path, ?1) AS score
-       FROM context_index
-      WHERE fts_main_context_index.match_bm25(context_index.path, ?1) IS NOT NULL
-      ORDER BY score DESC
-      LIMIT ?2`,
-    query,
-    limit,
-  );
-  return rows.map((row) => ({ ...rowToChunk(row), score: Number(row.score) }));
-}
-
-/**
- * Reciprocal-rank fusion of semantic + keyword results, deduped by
- * (path, chunk_index).
- */
-export async function searchHybrid(
-  conn: DbConnection,
-  query: string,
-  queryEmbedding: number[],
-  limit = 10,
-): Promise<SearchResult[]> {
-  const k = 60;
-  const [semantic, keyword] = await Promise.all([
-    searchSemantic(conn, queryEmbedding, 100),
-    searchKeyword(conn, query, 100).catch(() => [] as SearchResult[]),
-  ]);
-
-  const scores = new Map<string, { chunk: IndexedChunk; score: number }>();
-  const key = (c: IndexedChunk) => `${c.path}::${c.chunk_index}`;
-
-  for (let i = 0; i < semantic.length; i++) {
-    const c = semantic[i];
-    if (!c) continue;
-    const existing = scores.get(key(c));
-    const rrf = 1 / (k + i + 1);
-    if (existing) existing.score += rrf;
-    else scores.set(key(c), { chunk: c, score: rrf });
-  }
-  for (let i = 0; i < keyword.length; i++) {
-    const c = keyword[i];
-    if (!c) continue;
-    const existing = scores.get(key(c));
-    const rrf = 1 / (k + i + 1);
-    if (existing) existing.score += rrf;
-    else scores.set(key(c), { chunk: c, score: rrf });
-  }
-  const merged = [...scores.values()].sort((a, b) => b.score - a.score);
-  return merged.slice(0, limit).map((m) => ({ ...m.chunk, score: m.score }));
-}
-
-/**
- * Rebuild the FTS index over (chunk_content, path). DuckDB's FTS index is a
- * snapshot — it does not update incrementally on INSERT/UPDATE/DELETE, so any
- * batch writer must call this once its transaction commits.
- *
- * The trailing CHECKPOINT is load-bearing (see history): `overwrite = 1`
- * writes a `DROP SCHEMA fts_main_context_index` record into the WAL; without
- * the checkpoint, replay on the next open can fail with "Cannot drop entry
- * 'fts_main_context_index' because there are entries that depend on it".
- */
-export async function rebuildSearchIndex(conn: DbConnection): Promise<void> {
-  // Skip if the table doesn't exist yet (e.g., fresh tests with an empty
-  // schema). The FTS extension errors out on a missing table.
-  const exists = await conn.queryGet<{ name: string }>(
-    "SELECT table_name AS name FROM information_schema.tables WHERE table_name = 'context_index'",
-  );
-  if (!exists) return;
-  await conn.exec(
-    "PRAGMA create_fts_index('context_index', 'path', 'chunk_content', 'path', overwrite = 1)",
-  );
-  await conn.exec("CHECKPOINT");
-}
-
-export async function indexStats(conn: DbConnection): Promise<{
-  paths: number;
-  chunks: number;
-  embedded: number;
-}> {
-  const row = await conn.queryGet<{
-    paths: number;
-    chunks: number;
-    embedded: number;
-  }>(
-    `SELECT COUNT(DISTINCT path) AS paths,
-            COUNT(*) AS chunks,
-            COUNT(embedding) AS embedded
-       FROM context_index`,
-  );
-  return {
-    paths: Number(row?.paths ?? 0),
-    chunks: Number(row?.chunks ?? 0),
-    embedded: Number(row?.embedded ?? 0),
-  };
-}

package/src/db/query.ts

@@ -1,56 +0,0 @@
-type SqlParam = string | number | null;
-
-/**
- * Validate that a value is a positive integer, suitable for use in
- * LIMIT / OFFSET clauses that must be interpolated into SQL strings.
- */
-export function sanitizeInt(val: number): number {
-  if (!Number.isInteger(val) || val <= 0) {
-    throw new Error(`Expected a positive integer, got: ${val}`);
-  }
-  return val;
-}
-
-/**
- * Build a WHERE clause from column-value pairs.
- * Entries with `undefined` values are skipped.
- */
-export function buildWhereClause(filters: [string, SqlParam | undefined][]): {
-  where: string;
-  params: SqlParam[];
-} {
-  const conditions: string[] = [];
-  const params: SqlParam[] = [];
-
-  for (const [col, val] of filters) {
-    if (val !== undefined) {
-      params.push(val);
-      conditions.push(`${col} = ?${params.length}`);
-    }
-  }
-
-  const where =
-    conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-  return { where, params };
-}
-
-/**
- * Build SET clauses for an UPDATE from column-value pairs.
- * Entries with `undefined` values are skipped.
- */
-export function buildSetClauses(fields: [string, SqlParam | undefined][]): {
-  setClauses: string[];
-  params: SqlParam[];
-} {
-  const setClauses: string[] = [];
-  const params: SqlParam[] = [];
-
-  for (const [col, val] of fields) {
-    if (val !== undefined) {
-      params.push(val);
-      setClauses.push(`${col} = ?${params.length}`);
-    }
-  }
-
-  return { setClauses, params };
-}

package/src/db/schema.ts

@@ -1,93 +0,0 @@
-import { readdirSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-import { logger } from "../utils/logger.ts";
-import type { DbConnection } from "./connection.ts";
-import { rebuildSearchIndex } from "./embeddings.ts";
-
-interface Migration {
-  id: number;
-  name: string;
-  sql: string;
-}
-
-const sqlDir = join(import.meta.dir, "sql");
-
-function loadMigrations(): Migration[] {
-  const files = readdirSync(sqlDir).filter((f) => f.endsWith(".sql"));
-
-  const migrations = files.map((file) => {
-    const match = file.match(/^(\d+)-(.+)\.sql$/);
-    if (!match) throw new Error(`Invalid migration filename: ${file}`);
-    const id = match[1];
-    const name = match[2];
-    if (!id || !name) throw new Error(`Invalid migration filename: ${file}`);
-    return {
-      id: parseInt(id, 10),
-      name,
-      sql: readFileSync(join(sqlDir, file), "utf-8"),
-    };
-  });
-
-  // Sort by numeric id so `12-` runs after `2-`, not between `11-` and `2-`.
-  return migrations.sort((a, b) => a.id - b.id);
-}
-
-export async function migrate(db: DbConnection): Promise<void> {
-  // Create migrations tracking table
-  await db.exec(`
-    CREATE TABLE IF NOT EXISTS _migrations (
-      id INTEGER PRIMARY KEY,
-      name TEXT NOT NULL,
-      applied_at TEXT DEFAULT (current_timestamp::VARCHAR)
-    )
-  `);
-
-  // Get already-applied migrations
-  const rows = await db.queryAll<{ id: number }>("SELECT id FROM _migrations");
-  const applied = new Set(rows.map((row) => row.id));
-
-  // Run pending migrations in order
-  const pending = loadMigrations().filter((m) => !applied.has(m.id));
-  if (pending.length > 0) {
-    logger.info(
-      `applying ${pending.length} migration${pending.length === 1 ? "" : "s"}`,
-    );
-  }
-
-  let appliedAny = false;
-  for (const migration of pending) {
-    logger.info(`  ${migration.id}. ${migration.name}`);
-
-    // Split on semicolons and run each statement individually
-    const statements = migration.sql
-      .split(";")
-      .map((s) => s.trim())
-      .filter((s) => s.length > 0);
-
-    for (const statement of statements) {
-      await db.exec(statement);
-    }
-
-    await db.queryRun(
-      "INSERT INTO _migrations (id, name) VALUES (?1, ?2)",
-      migration.id,
-      migration.name,
-    );
-    appliedAny = true;
-  }
-
-  // Flush the WAL so the next open has no schema entries to replay. DuckDB's
-  // WAL replay of ALTER TABLE re-binds all column defaults on the target
-  // table, and our CREATE TABLE defaults use `current_timestamp::VARCHAR` —
-  // which cannot be resolved during replay (no default database attached yet),
-  // crashing the process on reopen.
-  if (appliedAny) {
-    await db.exec("CHECKPOINT");
-  }
-
-  // Ensure the FTS index exists. Migration 18 drops it (it can't recreate it
-  // in the same SQL run without DuckDB rejecting the dependency commit), and
-  // fresh DBs need it created at least once. `overwrite = 1` makes this
-  // idempotent for DBs that already have a healthy FTS index.
-  await rebuildSearchIndex(db);
-}

package/src/db/sql/1-core_tables.sql

@@ -1,53 +0,0 @@
-CREATE TABLE tasks (
-  id TEXT PRIMARY KEY,
-  name TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  priority TEXT NOT NULL DEFAULT 'medium' CHECK(priority IN ('low', 'medium', 'high')),
-  status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'in_progress', 'failed', 'complete', 'waiting')),
-  waiting_reason TEXT,
-  claimed_by TEXT,
-  claimed_at TEXT,
-  blocked_by TEXT NOT NULL DEFAULT '[]',
-  context_ids TEXT NOT NULL DEFAULT '[]',
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  updated_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR)
-);
-
-CREATE TABLE schedules (
-  id TEXT PRIMARY KEY,
-  name TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  frequency TEXT NOT NULL,
-  last_run_at TEXT,
-  enabled BOOLEAN NOT NULL DEFAULT true,
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  updated_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR)
-);
-
-CREATE TABLE context_items (
-  id TEXT PRIMARY KEY,
-  title TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  content TEXT,
-  content_blob BLOB,
-  mime_type TEXT NOT NULL DEFAULT 'text/plain',
-  is_textual BOOLEAN NOT NULL DEFAULT true,
-  source_path TEXT,
-  context_path TEXT NOT NULL,
-  indexed_at TEXT,
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  updated_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR)
-);
-
-CREATE TABLE embeddings (
-  id TEXT PRIMARY KEY,
-  context_item_id TEXT NOT NULL REFERENCES context_items(id),
-  chunk_index INTEGER NOT NULL,
-  chunk_content TEXT,
-  title TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  source_path TEXT,
-  embedding FLOAT[1536],
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  UNIQUE(context_item_id, chunk_index)
-);

package/src/db/sql/10-dedupe_context_items.sql

@@ -1,26 +0,0 @@
--- Older DBs could accumulate duplicate rows in context_items with the same
--- context_path: migration 4's CREATE UNIQUE INDEX IF NOT EXISTS silently left
--- the index metadata in place without enforcing it when duplicates predated
--- the migration. The resulting "corrupt" unique index triggers a native
--- crash in @duckdb/node-api on UPDATE ... RETURNING. Rebuild cleanly.
-DROP INDEX IF EXISTS idx_context_items_context_path;
-
-DELETE FROM embeddings WHERE context_item_id IN (
-  SELECT id FROM (
-    SELECT id, ROW_NUMBER() OVER (
-      PARTITION BY context_path
-      ORDER BY updated_at DESC, id DESC
-    ) AS rn FROM context_items
-  ) WHERE rn > 1
-);
-
-DELETE FROM context_items WHERE id IN (
-  SELECT id FROM (
-    SELECT id, ROW_NUMBER() OVER (
-      PARTITION BY context_path
-      ORDER BY updated_at DESC, id DESC
-    ) AS rn FROM context_items
-  ) WHERE rn > 1
-);
-
-CREATE UNIQUE INDEX idx_context_items_context_path ON context_items(context_path);

package/src/db/sql/11-rebuild_hnsw.sql

@@ -1,8 +0,0 @@
--- Historical: this migration used to drop and recreate the HNSW index
--- to clean up an internally-inconsistent state after native-side crashes
--- during embedding writes. HNSW is now gone (see migration 14) and the
--- VSS extension is no longer loaded at connection time, so the original
--- DDL would fail on fresh DBs. Kept as a no-op to preserve migration
--- numbering for existing databases that have already recorded id 11 in
--- _migrations.
-SELECT 1;

package/src/db/sql/12-workers.sql

@@ -1,66 +0,0 @@
--- Worker agents: replaces the PID-file + OS-watchdog single-daemon model
--- with multiple in-DB registered workers that heartbeat and can be reaped.
-
-CREATE TABLE workers (
-  id TEXT PRIMARY KEY,
-  pid INTEGER NOT NULL,
-  hostname TEXT NOT NULL,
-  mode TEXT NOT NULL CHECK(mode IN ('persist', 'once')),
-  task_id TEXT,
-  status TEXT NOT NULL CHECK(status IN ('running', 'stopped', 'dead')),
-  started_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  last_heartbeat_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  stopped_at TEXT
-);
-
-CREATE INDEX idx_workers_status_heartbeat ON workers(status, last_heartbeat_at);
-
--- Schedule claim columns: only one worker evaluates a schedule per window.
-ALTER TABLE schedules ADD COLUMN claimed_by TEXT;
-ALTER TABLE schedules ADD COLUMN claimed_at TEXT;
-
--- Rewrite threads.type values: daemon_tick → worker_tick. The existing
--- CHECK constraint forbids the new value, so we rebuild both threads and
--- interactions (whose FK to threads would block a DROP). Dropping the FK
--- follows the 7-drop_embeddings_fk.sql precedent.
-CREATE TABLE threads_backup AS SELECT * FROM threads;
-CREATE TABLE interactions_backup AS SELECT * FROM interactions;
-
-DROP TABLE interactions;
-DROP TABLE threads;
-
-CREATE TABLE threads (
-  id TEXT PRIMARY KEY,
-  type TEXT NOT NULL CHECK(type IN ('worker_tick', 'chat_session')),
-  task_id TEXT,
-  title TEXT NOT NULL DEFAULT '',
-  started_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  ended_at TEXT,
-  metadata TEXT
-);
-
-CREATE TABLE interactions (
-  id TEXT PRIMARY KEY,
-  thread_id TEXT NOT NULL,
-  sequence INTEGER NOT NULL,
-  role TEXT NOT NULL CHECK(role IN ('user', 'assistant', 'system', 'tool')),
-  kind TEXT NOT NULL CHECK(kind IN ('message', 'thinking', 'tool_use', 'tool_result', 'context_update', 'status_change')),
-  content TEXT NOT NULL,
-  tool_name TEXT,
-  tool_input TEXT,
-  duration_ms INTEGER,
-  token_count INTEGER,
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  UNIQUE(thread_id, sequence)
-);
-
-INSERT INTO threads
-SELECT id,
-       CASE WHEN type = 'daemon_tick' THEN 'worker_tick' ELSE type END,
-       task_id, title, started_at, ended_at, metadata
-FROM threads_backup;
-
-INSERT INTO interactions SELECT * FROM interactions_backup;
-
-DROP TABLE threads_backup;
-DROP TABLE interactions_backup;

package/src/db/sql/13-drive-paths.sql

@@ -1,47 +0,0 @@
--- Milestone 10: collapse `source_path` + `context_path` + `source_type` into a
--- single `(drive, path)` identity pair. Pre-1.0, no backwards-compat promise —
--- we wipe context_items + embeddings and have the user re-add their content.
---
--- DuckDB's ALTER TABLE support is thin (no SET NOT NULL, flaky DROP COLUMN with
--- existing indexes), so this is a table rebuild. Order matters: drop indexes
--- first, then the old tables, then recreate with the new shape.
-
-DELETE FROM embeddings;
-DELETE FROM context_items;
-
-DROP INDEX IF EXISTS idx_embeddings_cosine;
-DROP INDEX IF EXISTS idx_context_items_context_path;
-
-DROP TABLE embeddings;
-DROP TABLE context_items;
-
-CREATE TABLE context_items (
-  id TEXT PRIMARY KEY,
-  title TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  content TEXT,
-  content_blob BLOB,
-  mime_type TEXT NOT NULL DEFAULT 'text/plain',
-  is_textual BOOLEAN NOT NULL DEFAULT true,
-  drive TEXT NOT NULL,
-  path TEXT NOT NULL,
-  indexed_at TEXT,
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  updated_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR)
-);
-
-CREATE UNIQUE INDEX idx_context_items_drive_path ON context_items(drive, path);
-
-CREATE TABLE embeddings (
-  id TEXT PRIMARY KEY,
-  context_item_id TEXT NOT NULL,
-  chunk_index INTEGER NOT NULL,
-  chunk_content TEXT,
-  title TEXT NOT NULL,
-  description TEXT NOT NULL DEFAULT '',
-  embedding FLOAT[1536],
-  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
-  UNIQUE(context_item_id, chunk_index)
-);
-
-CHECKPOINT;

package/src/db/sql/14-drop_hnsw_index.sql

@@ -1,8 +0,0 @@
--- HNSW has caused two separate corruption modes in this project: the
--- "Duplicate keys not allowed in high-level wrappers" failure addressed by
--- migration 11, and a second mode where the index silently returns zero rows
--- for cosine top-K queries (its stored SQL loses the `WITH (metric = 'cosine')`
--- clause). At our scale a linear scan of array_cosine_distance is plenty fast
--- and array_cosine_distance is a core DuckDB function — no VSS extension
--- required. Drop the index and move on.
-DROP INDEX IF EXISTS idx_embeddings_cosine;

package/src/db/sql/15-fts_index.sql

@@ -1,8 +0,0 @@
--- Keyword search uses DuckDB's FTS extension for BM25 ranking over
--- chunk_content and title. The index is a snapshot and must be rebuilt
--- after any write to the embeddings table. rebuildSearchIndex() in
--- src/db/embeddings.ts is the single entry point and is called from the
--- ingest transaction. overwrite = 1 makes this PRAGMA idempotent, which
--- also gives us a first-run rebuild for users upgrading from a DB that
--- never had FTS.
-PRAGMA create_fts_index('embeddings', 'id', 'chunk_content', 'title', overwrite = 1);

package/src/db/sql/16-source_url.sql

@@ -1,7 +0,0 @@
--- Issue #145: preserve the original URL that produced each context item so
--- `context refresh` can re-fetch loss-lessly for service-specific drives
--- (google-docs, github, ...). Nullable — local-origin drives (disk, agent,
--- tool writes) leave it NULL and use their own refresh path. Legacy rows
--- ingested before this column existed also leave it NULL and surface a
--- "re-add from URL" error on refresh.
-ALTER TABLE context_items ADD COLUMN source_url TEXT;