botholomew 0.5.0 → 0.6.0

package/src/db/context.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildSetClauses, buildWhereClause } from "./query.ts";
+import { buildSetClauses, buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface ContextItem {
@@ -29,7 +29,7 @@ interface ContextItemRow {
   content: string | null;
   content_blob: unknown;
   mime_type: string;
-  is_textual: number;
+  is_textual: boolean;
   source_path: string | null;
   context_path: string;
   indexed_at: string | null;
@@ -44,7 +44,7 @@ function rowToContextItem(row: ContextItemRow): ContextItem {
     description: row.description,
     content: row.content,
     mime_type: row.mime_type,
-    is_textual: row.is_textual === 1,
+    is_textual: !!row.is_textual,
     source_path: row.source_path,
     context_path: row.context_path,
     indexed_at: row.indexed_at ? new Date(row.indexed_at) : null,
@@ -68,33 +68,64 @@ export async function createContextItem(
   },
 ): Promise<ContextItem> {
   const id = uuidv7();
-  const row = db
-    .query(
-      `INSERT INTO context_items (id, title, description, content, mime_type, is_textual, source_path, context_path)
+  const row = await db.queryGet<ContextItemRow>(
+    `INSERT INTO context_items (id, title, description, content, mime_type, is_textual, source_path, context_path)
      VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
      RETURNING *`,
-    )
-    .get(
-      id,
-      params.title,
-      params.description ?? "",
-      params.content ?? null,
-      params.mimeType ?? "text/plain",
-      params.isTextual !== false ? 1 : 0,
-      params.sourcePath ?? null,
-      params.contextPath,
-    ) as ContextItemRow | null;
+    id,
+    params.title,
+    params.description ?? "",
+    params.content ?? null,
+    params.mimeType ?? "text/plain",
+    params.isTextual !== false,
+    params.sourcePath ?? null,
+    params.contextPath,
+  );
   if (!row) throw new Error("INSERT did not return a row");
   return rowToContextItem(row);
 }
 
+/**
+ * Atomic upsert by context_path: updates if the path exists, inserts otherwise.
+ *
+ * DuckDB implements UPDATE as delete+insert on tables with unique indexes,
+ * which violates foreign keys from the embeddings table. We must delete
+ * embeddings before updating; callers (context add, file_write) re-create
+ * them in their ingestion phase.
+ */
+export async function upsertContextItem(
+  db: DbConnection,
+  params: {
+    title: string;
+    content?: string;
+    mimeType?: string;
+    sourcePath?: string;
+    contextPath: string;
+    description?: string;
+    isTextual?: boolean;
+  },
+): Promise<ContextItem> {
+  const existing = await getContextItemByPath(db, params.contextPath);
+  if (existing) {
+    const updated = await updateContextItem(db, existing.id, {
+      title: params.title,
+      content: params.content,
+      mime_type: params.mimeType,
+    });
+    if (!updated) throw new Error(`Failed to update: ${params.contextPath}`);
+    return updated;
+  }
+  return createContextItem(db, params);
+}
+
 export async function getContextItem(
   db: DbConnection,
   id: string,
 ): Promise<ContextItem | null> {
-  const row = db
-    .query("SELECT * FROM context_items WHERE id = ?1")
-    .get(id) as ContextItemRow | null;
+  const row = await db.queryGet<ContextItemRow>(
+    "SELECT * FROM context_items WHERE id = ?1",
+    id,
+  );
   return row ? rowToContextItem(row) : null;
 }
 
@@ -102,9 +133,10 @@ export async function getContextItemByPath(
   db: DbConnection,
   contextPath: string,
 ): Promise<ContextItem | null> {
-  const row = db
-    .query("SELECT * FROM context_items WHERE context_path = ?1")
-    .get(contextPath) as ContextItemRow | null;
+  const row = await db.queryGet<ContextItemRow>(
+    "SELECT * FROM context_items WHERE context_path = ?1",
+    contextPath,
+  );
   return row ? rowToContextItem(row) : null;
 }
 
@@ -121,14 +153,13 @@ export async function listContextItems(
     ["context_path", filters?.contextPath],
     ["mime_type", filters?.mimeType],
   ]);
-  const limit = filters?.limit ? `LIMIT ${filters.limit}` : "";
-  const offset = filters?.offset ? `OFFSET ${filters.offset}` : "";
-
-  const rows = db
-    .query(
-      `SELECT * FROM context_items ${where} ORDER BY context_path ASC ${limit} ${offset}`,
-    )
-    .all(...params) as ContextItemRow[];
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
+
+  const rows = await db.queryAll<ContextItemRow>(
+    `SELECT * FROM context_items ${where} ORDER BY context_path ASC ${limit} ${offset}`,
+    ...params,
+  );
   return rows.map(rowToContextItem);
 }
 
@@ -139,31 +170,27 @@ export async function listContextItemsByPrefix(
 ): Promise<ContextItem[]> {
   const normalizedPrefix = prefix.endsWith("/") ? prefix : `${prefix}/`;
 
-  const limit = opts?.limit ? `LIMIT ${opts.limit}` : "";
-  const offset = opts?.offset ? `OFFSET ${opts.offset}` : "";
+  const limit = opts?.limit ? `LIMIT ${sanitizeInt(opts.limit)}` : "";
+  const offset = opts?.offset ? `OFFSET ${sanitizeInt(opts.offset)}` : "";
 
   let rows: ContextItemRow[];
   if (opts?.recursive) {
-    rows = db
-      .query(
-        `SELECT * FROM context_items
+    rows = await db.queryAll<ContextItemRow>(
+      `SELECT * FROM context_items
        WHERE context_path LIKE ?1
        ORDER BY context_path ASC ${limit} ${offset}`,
-      )
-      .all(`${normalizedPrefix}%`) as ContextItemRow[];
+      `${normalizedPrefix}%`,
+    );
   } else {
     // Only immediate children: match prefix but no further slashes
-    rows = db
-      .query(
-        `SELECT * FROM context_items
+    rows = await db.queryAll<ContextItemRow>(
+      `SELECT * FROM context_items
        WHERE context_path LIKE ?1
          AND context_path NOT LIKE ?2
        ORDER BY context_path ASC ${limit} ${offset}`,
-      )
-      .all(
-        `${normalizedPrefix}%`,
-        `${normalizedPrefix}%/%`,
-      ) as ContextItemRow[];
+      `${normalizedPrefix}%`,
+      `${normalizedPrefix}%/%`,
+    );
   }
 
   return rows.map(rowToContextItem);
@@ -173,11 +200,10 @@ export async function contextPathExists(
   db: DbConnection,
   contextPath: string,
 ): Promise<boolean> {
-  const row = db
-    .query(
-      "SELECT 1 AS found FROM context_items WHERE context_path = ?1 LIMIT 1",
-    )
-    .get(contextPath);
+  const row = await db.queryGet(
+    "SELECT 1 AS found FROM context_items WHERE context_path = ?1 LIMIT 1",
+    contextPath,
+  );
   return row != null;
 }
 
@@ -192,19 +218,19 @@ export async function getDistinctDirectories(
     : "/";
 
   // Extract the first path segment after the prefix
-  const rows = db
-    .query(
-      `SELECT DISTINCT
+  const rows = await db.queryAll<{ dir: string }>(
+    `SELECT DISTINCT
         ?1 || CASE
-          WHEN instr(substr(context_path, length(?1) + 1), '/') > 0
-          THEN substr(substr(context_path, length(?1) + 1), 1, instr(substr(context_path, length(?1) + 1), '/') - 1)
+          WHEN strpos(substr(context_path, length(?1) + 1), '/') > 0
+          THEN substr(substr(context_path, length(?1) + 1), 1, strpos(substr(context_path, length(?1) + 1), '/') - 1)
           ELSE substr(context_path, length(?1) + 1)
         END AS dir
       FROM context_items
       WHERE context_path LIKE ?2
       ORDER BY dir ASC`,
-    )
-    .all(normalizedPrefix, `${normalizedPrefix}%/%`) as { dir: string }[];
+    normalizedPrefix,
+    `${normalizedPrefix}%/%`,
+  );
 
   return rows.map((row) => row.dir);
 }
@@ -225,17 +251,16 @@ export async function updateContextItem(
     ["mime_type", updates.mime_type],
   ]);
 
-  setClauses.push("updated_at = datetime('now')");
+  setClauses.push("updated_at = current_timestamp::VARCHAR");
   params.push(id);
 
-  const row = db
-    .query(
-      `UPDATE context_items
+  const row = await db.queryGet<ContextItemRow>(
+    `UPDATE context_items
      SET ${setClauses.join(", ")}
      WHERE id = ?${params.length}
      RETURNING *`,
-    )
-    .get(...params) as ContextItemRow | null;
+    ...params,
+  );
   return row ? rowToContextItem(row) : null;
 }
 
@@ -244,14 +269,14 @@ export async function updateContextItemContent(
   contextPath: string,
   content: string,
 ): Promise<ContextItem | null> {
-  const row = db
-    .query(
-      `UPDATE context_items
-     SET content = ?1, updated_at = datetime('now')
+  const row = await db.queryGet<ContextItemRow>(
+    `UPDATE context_items
+     SET content = ?1, updated_at = current_timestamp::VARCHAR
      WHERE context_path = ?2
      RETURNING *`,
-    )
-    .get(content, contextPath) as ContextItemRow | null;
+    content,
+    contextPath,
+  );
   return row ? rowToContextItem(row) : null;
 }
 
@@ -312,14 +337,14 @@ export async function moveContextItem(
   oldPath: string,
   newPath: string,
 ): Promise<void> {
-  const row = db
-    .query(
-      `UPDATE context_items
-     SET context_path = ?1, updated_at = datetime('now')
+  const row = await db.queryGet(
+    `UPDATE context_items
+     SET context_path = ?1, updated_at = current_timestamp::VARCHAR
      WHERE context_path = ?2
      RETURNING id`,
-    )
-    .get(newPath, oldPath);
+    newPath,
+    oldPath,
+  );
   if (!row) {
     throw new Error(`Not found: ${oldPath}`);
   }
@@ -332,10 +357,11 @@ export async function deleteContextItem(
   id: string,
 ): Promise<boolean> {
   // Delete embeddings first (foreign key)
-  db.query("DELETE FROM embeddings WHERE context_item_id = ?1").run(id);
-  const row = db
-    .query("DELETE FROM context_items WHERE id = ?1 RETURNING id")
-    .get(id);
+  await db.queryRun("DELETE FROM embeddings WHERE context_item_id = ?1", id);
+  const row = await db.queryGet(
+    "DELETE FROM context_items WHERE id = ?1 RETURNING id",
+    id,
+  );
   return row != null;
 }
 
@@ -356,21 +382,21 @@ export async function deleteContextItemsByPrefix(
   const normalizedPrefix = prefix.endsWith("/") ? prefix : `${prefix}/`;
 
   // Delete embeddings for all matching items
-  db.query(
+  await db.queryRun(
     `DELETE FROM embeddings
      WHERE context_item_id IN (
        SELECT id FROM context_items
        WHERE context_path LIKE ?1
      )`,
-  ).run(`${normalizedPrefix}%`);
+    `${normalizedPrefix}%`,
+  );
 
-  const rows = db
-    .query(
-      `DELETE FROM context_items
+  const rows = await db.queryAll(
+    `DELETE FROM context_items
      WHERE context_path LIKE ?1
      RETURNING id`,
-    )
-    .all(`${normalizedPrefix}%`);
+    `${normalizedPrefix}%`,
+  );
   return rows.length;
 }
 
@@ -382,17 +408,17 @@ export async function searchContextByKeyword(
   limit = 20,
 ): Promise<ContextItem[]> {
   const pattern = `%${query}%`;
-  const rows = db
-    .query(
-      `SELECT * FROM context_items
+  const rows = await db.queryAll<ContextItemRow>(
+    `SELECT * FROM context_items
      WHERE content IS NOT NULL
        AND (
-         content LIKE ?1 COLLATE NOCASE
-         OR title LIKE ?1 COLLATE NOCASE
+         content ILIKE ?1
+         OR title ILIKE ?1
        )
      ORDER BY updated_at DESC
      LIMIT ?2`,
-    )
-    .all(pattern, limit) as ContextItemRow[];
+    pattern,
+    limit,
+  );
   return rows.map(rowToContextItem);
 }

package/src/db/embeddings.ts

@@ -2,23 +2,8 @@ import { EMBEDDING_DIMENSION } from "../constants.ts";
 import type { DbConnection } from "./connection.ts";
 import { uuidv7 } from "./uuid.ts";
 
-// Track which connections have been initialized for vector search
-const initializedConnections = new WeakSet<DbConnection>();
-
-/**
- * Initialize sqlite-vector on the embeddings table for this connection.
- * Must be called once per connection before vector operations.
- * The dimension parameter allows overriding for tests.
- */
-export function initVectorSearch(
-  conn: DbConnection,
-  dimension = EMBEDDING_DIMENSION,
-): void {
-  if (initializedConnections.has(conn)) return;
-  conn.exec(
-    `SELECT vector_init('embeddings', 'embedding', 'dimension=${dimension},type=FLOAT32,distance=COSINE')`,
-  );
-  initializedConnections.add(conn);
+if (!Number.isInteger(EMBEDDING_DIMENSION) || EMBEDDING_DIMENSION <= 0) {
+  throw new Error(`Invalid EMBEDDING_DIMENSION: ${EMBEDDING_DIMENSION}`);
 }
 
 export interface Embedding {
@@ -45,7 +30,7 @@ interface EmbeddingRow {
   title: string;
   description: string;
   source_path: string | null;
-  embedding: Uint8Array | null;
+  embedding: number[] | null;
   created_at: string;
 }
 
@@ -58,14 +43,12 @@ function rowToEmbedding(row: EmbeddingRow): Embedding {
     title: row.title,
     description: row.description,
     source_path: row.source_path,
-    embedding: row.embedding
-      ? Array.from(new Float32Array(row.embedding.buffer))
-      : [],
+    embedding: row.embedding ?? [],
     created_at: new Date(row.created_at),
   };
 }
 
-export function createEmbedding(
+export async function createEmbedding(
   conn: DbConnection,
   params: {
     contextItemId: string;
@@ -76,23 +59,20 @@ export function createEmbedding(
     sourcePath?: string | null;
     embedding: number[];
   },
-): Embedding {
+): Promise<Embedding> {
   const id = uuidv7();
-  conn
-    .query(
-      `INSERT INTO embeddings (id, context_item_id, chunk_index, chunk_content, title, description, source_path, embedding)
-       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, vector_as_f32(?8))`,
-    )
-    .run(
-      id,
-      params.contextItemId,
-      params.chunkIndex,
-      params.chunkContent,
-      params.title,
-      params.description ?? "",
-      params.sourcePath ?? null,
-      JSON.stringify(params.embedding),
-    );
+  await conn.queryRun(
+    `INSERT INTO embeddings (id, context_item_id, chunk_index, chunk_content, title, description, source_path, embedding)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8::FLOAT[${EMBEDDING_DIMENSION}])`,
+    id,
+    params.contextItemId,
+    params.chunkIndex,
+    params.chunkContent,
+    params.title,
+    params.description ?? "",
+    params.sourcePath ?? null,
+    params.embedding,
+  );
 
   return {
     id,
@@ -107,52 +87,51 @@ export function createEmbedding(
   };
 }
 
-export function getEmbeddingsForItem(
+export async function getEmbeddingsForItem(
   conn: DbConnection,
   contextItemId: string,
-): Embedding[] {
-  const rows = conn
-    .query(
-      "SELECT * FROM embeddings WHERE context_item_id = ?1 ORDER BY chunk_index ASC",
-    )
-    .all(contextItemId) as EmbeddingRow[];
+): Promise<Embedding[]> {
+  const rows = await conn.queryAll<EmbeddingRow>(
+    "SELECT * FROM embeddings WHERE context_item_id = ?1 ORDER BY chunk_index ASC",
+    contextItemId,
+  );
   return rows.map(rowToEmbedding);
 }
 
-export function deleteEmbeddingsForItem(
+export async function deleteEmbeddingsForItem(
   conn: DbConnection,
   contextItemId: string,
-): number {
-  const result = conn
-    .query("DELETE FROM embeddings WHERE context_item_id = ?1")
-    .run(contextItemId);
+): Promise<number> {
+  const result = await conn.queryRun(
+    "DELETE FROM embeddings WHERE context_item_id = ?1",
+    contextItemId,
+  );
   return result.changes;
 }
 
-interface VectorScanRow extends EmbeddingRow {
+interface VectorSearchRow extends EmbeddingRow {
   distance: number;
 }
 
 /**
- * Vector similarity search using sqlite-vector's SIMD-accelerated
- * cosine distance via vector_full_scan(). Returns results sorted by
+ * Vector similarity search using DuckDB's array_cosine_distance().
+ * With an HNSW index on the embedding column, DuckDB automatically
+ * uses the index for top-k queries. Returns results sorted by
  * similarity (closest first), with score = 1 - distance.
  */
-export function searchEmbeddings(
+export async function searchEmbeddings(
   conn: DbConnection,
   queryEmbedding: number[],
   limit = 10,
-): EmbeddingSearchResult[] {
-  const queryJson = JSON.stringify(queryEmbedding);
-
-  const rows = conn
-    .query(
-      `SELECT e.*, v.distance
-       FROM embeddings e
-       JOIN vector_full_scan('embeddings', 'embedding', vector_as_f32(?1), ?2) v
-         ON e.rowid = v.rowid`,
-    )
-    .all(queryJson, limit) as VectorScanRow[];
+): Promise<EmbeddingSearchResult[]> {
+  const rows = await conn.queryAll<VectorSearchRow>(
+    `SELECT *, array_cosine_distance(embedding, ?1::FLOAT[${EMBEDDING_DIMENSION}]) AS distance
+     FROM embeddings
+     ORDER BY distance ASC
+     LIMIT ?2`,
+    queryEmbedding,
+    limit,
+  );
 
   return rows.map((row) => ({
     ...rowToEmbedding(row),
@@ -160,28 +139,27 @@ export function searchEmbeddings(
   }));
 }
 
-export function hybridSearch(
+export async function hybridSearch(
   conn: DbConnection,
   query: string,
   queryEmbedding: number[],
   limit = 10,
-): EmbeddingSearchResult[] {
+): Promise<EmbeddingSearchResult[]> {
   const k = 60; // RRF constant
 
   // Keyword search: match on chunk_content and title
-  const keywordRows = conn
-    .query(
-      `SELECT * FROM embeddings
-       WHERE chunk_content LIKE '%' || ?1 || '%'
-          OR title LIKE '%' || ?1 || '%'
-       LIMIT 100`,
-    )
-    .all(query) as EmbeddingRow[];
+  const keywordRows = await conn.queryAll<EmbeddingRow>(
+    `SELECT * FROM embeddings
+     WHERE chunk_content ILIKE '%' || ?1 || '%'
+        OR title ILIKE '%' || ?1 || '%'
+     LIMIT 100`,
+    query,
+  );
 
   const keywordRanked = keywordRows.map(rowToEmbedding);
 
-  // Vector search via sqlite-vector
-  const vectorResults = searchEmbeddings(conn, queryEmbedding, 100);
+  // Vector search via DuckDB VSS
+  const vectorResults = await searchEmbeddings(conn, queryEmbedding, 100);
 
   // Reciprocal rank fusion
   const scores = new Map<string, { embedding: Embedding; score: number }>();

package/src/db/query.ts

@@ -1,5 +1,16 @@
 type SqlParam = string | number | null;
 
+/**
+ * Validate that a value is a positive integer, suitable for use in
+ * LIMIT / OFFSET clauses that must be interpolated into SQL strings.
+ */
+export function sanitizeInt(val: number): number {
+  if (!Number.isInteger(val) || val <= 0) {
+    throw new Error(`Expected a positive integer, got: ${val}`);
+  }
+  return val;
+}
+
 /**
  * Build a WHERE clause from column-value pairs.
  * Entries with `undefined` values are skipped.