botholomew 0.5.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "An AI agent for knowledge work",
   "type": "module",
   "bin": {
@@ -23,8 +23,8 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.88.0",
+    "@duckdb/node-api": "^1.5.2-r.1",
     "@evantahler/mcpx": "0.18.3",
-"@sqliteai/sqlite-vector": "^0.9.95",
     "ansis": "^4.2.0",
     "commander": "^14.0.0",
     "gray-matter": "^4.0.3",

package/src/chat/session.ts

@@ -44,8 +44,8 @@ export async function startChatSession(
     );
   }
 
-  const conn = getConnection(getDbPath(projectDir));
-  migrate(conn);
+  const conn = await getConnection(getDbPath(projectDir));
+  await migrate(conn);
 
   let threadId: string;
   const messages: MessageParam[] = [];

package/src/commands/context.ts

@@ -5,6 +5,8 @@ import type { Command } from "commander";
 import { isText } from "istextorbinary";
 import { createSpinner } from "nanospinner";
 import { loadConfig } from "../config/loader.ts";
+import type { BotholomewConfig } from "../config/schemas.ts";
+import { generateDescription } from "../context/describer.ts";
 import { embedSingle } from "../context/embedder.ts";
 import {
   type PreparedIngestion,
@@ -14,18 +16,14 @@ import {
 import type { DbConnection } from "../db/connection.ts";
 import {
   type ContextItem,
-  createContextItem,
   deleteContextItemByPath,
   getContextItemByPath,
   listContextItems,
   listContextItemsByPrefix,
   updateContextItem,
+  upsertContextItem,
 } from "../db/context.ts";
-import {
-  getEmbeddingsForItem,
-  hybridSearch,
-  initVectorSearch,
-} from "../db/embeddings.ts";
+import { getEmbeddingsForItem, hybridSearch } from "../db/embeddings.ts";
 import { logger } from "../utils/logger.ts";
 import { withDb } from "./with-db.ts";
 
@@ -61,7 +59,7 @@ export function registerContextCommand(program: Command) {
           return;
         }
 
-        const header = `${ansis.bold("Path".padEnd(40))} ${"Title".padEnd(25)} ${"Type".padEnd(20)} ${"Updated".padEnd(18)} Indexed`;
+        const header = `${ansis.bold("Path".padEnd(35))} ${"Title".padEnd(20)} ${"Description".padEnd(30)} ${"Type".padEnd(15)} ${"Updated".padEnd(18)} Indexed`;
         console.log(header);
         console.log("-".repeat(header.length));
 
@@ -70,8 +68,11 @@ export function registerContextCommand(program: Command) {
             ? ansis.green("yes")
             : ansis.dim("no");
           const updated = ansis.dim(fmtDate(item.updated_at).padEnd(18));
+          const desc = item.description
+            ? ansis.dim(item.description.slice(0, 29).padEnd(30))
+            : ansis.dim("".padEnd(30));
           console.log(
-            `${item.context_path.padEnd(40)} ${item.title.slice(0, 24).padEnd(25)} ${item.mime_type.slice(0, 19).padEnd(20)} ${updated} ${indexed}`,
+            `${item.context_path.slice(0, 34).padEnd(35)} ${item.title.slice(0, 19).padEnd(20)} ${desc} ${item.mime_type.slice(0, 14).padEnd(15)} ${updated} ${indexed}`,
           );
         }
 
@@ -91,6 +92,7 @@ export function registerContextCommand(program: Command) {
         }
 
         console.log(ansis.bold(item.title));
+        if (item.description) console.log(`  Description: ${item.description}`);
         console.log(`  Path:        ${item.context_path}`);
         console.log(`  MIME type:   ${item.mime_type}`);
         if (item.source_path) console.log(`  Source:      ${item.source_path}`);
@@ -150,18 +152,34 @@ export function registerContextCommand(program: Command) {
           text: `Found ${filesToAdd.length} file(s) to add.`,
         });
 
-        // Phase 2: Load config and upsert DB records (sequential, fast)
+        // Phase 2: Load config and upsert DB records (batched, parallel LLM descriptions)
         const config = await loadConfig(dir);
+        const CONCURRENCY = 10;
+        let addCompleted = 0;
         const upsertSpinner = createSpinner(
-          "Adding files to database...",
+          `Adding and describing 0/${filesToAdd.length} files...`,
         ).start();
         const itemIds: { id: string; contextPath: string }[] = [];
-        for (const { filePath, contextPath } of filesToAdd) {
-          const result = await addFile(conn, filePath, contextPath);
-          if (result) itemIds.push({ id: result, contextPath });
+
+        for (let i = 0; i < filesToAdd.length; i += CONCURRENCY) {
+          const batch = filesToAdd.slice(i, i + CONCURRENCY);
+          const results = await Promise.all(
+            batch.map(async ({ filePath, contextPath }) => {
+              const result = await addFile(conn, filePath, contextPath, config);
+              addCompleted++;
+              upsertSpinner.update({
+                text: `Adding and describing ${addCompleted}/${filesToAdd.length} files...`,
+              });
+              return result ? { id: result, contextPath } : null;
+            }),
+          );
+          for (const r of results) {
+            if (r) itemIds.push(r);
+          }
         }
+
         upsertSpinner.success({
-          text: `Added ${itemIds.length} file(s) to database.`,
+          text: `Added and described ${itemIds.length} file(s).`,
         });
 
         // Phase 3: Chunk + embed in parallel (network I/O)
@@ -173,7 +191,6 @@ export function registerContextCommand(program: Command) {
           process.exit(0);
         }
 
-        const CONCURRENCY = 10;
         let completed = 0;
         const embedSpinner = createSpinner(
           `Embedding 0/${itemIds.length} files...`,
@@ -205,7 +222,7 @@ export function registerContextCommand(program: Command) {
         let filesAdded = 0;
         let filesUpdated = 0;
         for (const p of prepared) {
-          const result = storeIngestion(conn, p);
+          const result = await storeIngestion(conn, p);
           chunks += result.chunks;
           if (result.isUpdate) filesUpdated++;
           else filesAdded++;
@@ -226,9 +243,8 @@ export function registerContextCommand(program: Command) {
     .action((query, opts) =>
       withDb(program, async (conn, dir) => {
         const config = await loadConfig(dir);
-        initVectorSearch(conn);
         const queryVec = await embedSingle(query, config);
-        const results = hybridSearch(conn, query, queryVec, opts.topK);
+        const results = await hybridSearch(conn, query, queryVec, opts.topK);
 
         if (results.length === 0) {
           logger.dim("No results found.");
@@ -280,7 +296,7 @@ export function registerContextCommand(program: Command) {
           return;
         }
 
-        const embeddings = getEmbeddingsForItem(conn, item.id);
+        const embeddings = await getEmbeddingsForItem(conn, item.id);
 
         console.log(ansis.bold(item.title));
         console.log(`  Path:      ${item.context_path}`);
@@ -411,7 +427,7 @@ export function registerContextCommand(program: Command) {
 
         let chunks = 0;
         for (const p of prepared) {
-          const result = storeIngestion(conn, p);
+          const result = await storeIngestion(conn, p);
           chunks += result.chunks;
         }
 
@@ -443,36 +459,31 @@ async function addFile(
   conn: DbConnection,
   filePath: string,
   contextPath: string,
+  config: Required<BotholomewConfig>,
 ): Promise<string | null> {
   try {
     const bunFile = Bun.file(filePath);
     const mimeType = bunFile.type.split(";")[0] || "application/octet-stream";
     const filename = basename(filePath);
     const textual = isText(filename) !== false;
-
     const content = textual ? await bunFile.text() : null;
 
-    const existing = await getContextItemByPath(conn, contextPath);
-    let item: ContextItem;
-
-    if (existing) {
-      const updated = await updateContextItem(conn, existing.id, {
-        title: filename,
-        content: content ?? undefined,
-        mime_type: mimeType,
-      });
-      if (!updated) throw new Error(`Failed to update: ${contextPath}`);
-      item = updated;
-    } else {
-      item = await createContextItem(conn, {
-        title: filename,
-        content: content ?? undefined,
-        mimeType,
-        sourcePath: filePath,
-        contextPath,
-        isTextual: textual,
-      });
-    }
+    const description = await generateDescription(config, {
+      filename,
+      mimeType,
+      content,
+      filePath,
+    });
+
+    const item = await upsertContextItem(conn, {
+      title: filename,
+      description,
+      content: content ?? undefined,
+      mimeType,
+      sourcePath: filePath,
+      contextPath,
+      isTextual: textual,
+    });
 
     return textual && content ? item.id : null;
   } catch (err) {

package/src/commands/daemon.ts

@@ -12,7 +12,7 @@ export function registerDaemonCommand(program: Command) {
     .action(async () => {
       const dir = program.opts().dir;
       const { startDaemon } = await import("../daemon/index.ts");
-      await startDaemon(dir);
+      await startDaemon(dir, { foreground: true });
     });
 
   daemon

package/src/commands/schedule.ts

@@ -163,7 +163,7 @@ function enabledColor(enabled: boolean): string {
 }
 
 function printSchedule(s: Schedule) {
-  const id = ansis.dim(s.id.slice(0, 8));
+  const id = ansis.dim(s.id);
   const lastRun = s.last_run_at
     ? s.last_run_at.toISOString()
     : ansis.dim("never");

package/src/commands/task.ts

@@ -155,7 +155,7 @@ function priorityColor(priority: Task["priority"]): string {
 }
 
 function printTask(t: Task) {
-  const id = ansis.dim(t.id.slice(0, 8));
+  const id = ansis.dim(t.id);
   console.log(
     `  ${id}  ${statusColor(t.status)}  ${priorityColor(t.priority)}  ${t.name}`,
   );
@@ -168,6 +168,7 @@ function printTaskDetail(t: Task) {
   console.log(`  Priority:    ${priorityColor(t.priority)}`);
   if (t.description) console.log(`  Description: ${t.description}`);
   if (t.waiting_reason) console.log(`  Waiting:     ${t.waiting_reason}`);
+  if (t.output) console.log(`  Output:      ${t.output}`);
   if (t.claimed_by) console.log(`  Claimed by:  ${t.claimed_by}`);
   if (t.blocked_by.length > 0)
     console.log(`  Blocked by:  ${t.blocked_by.join(", ")}`);

package/src/commands/thread.ts

@@ -1,6 +1,5 @@
 import ansis from "ansis";
 import type { Command } from "commander";
-import type { DbConnection } from "../db/connection.ts";
 import type { Interaction, Thread } from "../db/threads.ts";
 import {
   deleteThread,
@@ -48,12 +47,7 @@ export function registerThreadCommand(program: Command) {
     )
     .action((id, opts) =>
       withDb(program, async (conn) => {
-        const resolvedId = await resolveThreadId(conn, id);
-        if (!resolvedId) {
-          logger.error(`Thread not found: ${id}`);
-          process.exit(1);
-        }
-        const result = await getThread(conn, resolvedId);
+        const result = await getThread(conn, id);
         if (!result) {
           logger.error(`Thread not found: ${id}`);
           process.exit(1);
@@ -72,17 +66,12 @@ export function registerThreadCommand(program: Command) {
     .description("Delete a thread and its interactions")
     .action((id) =>
       withDb(program, async (conn) => {
-        const resolvedId = await resolveThreadId(conn, id);
-        if (!resolvedId) {
-          logger.error(`Thread not found: ${id}`);
-          process.exit(1);
-        }
-        const deleted = await deleteThread(conn, resolvedId);
+        const deleted = await deleteThread(conn, id);
         if (!deleted) {
           logger.error(`Thread not found: ${id}`);
           process.exit(1);
         }
-        logger.success(`Deleted thread: ${resolvedId}`);
+        logger.success(`Deleted thread: ${id}`);
       }),
     );
 
@@ -94,12 +83,7 @@ export function registerThreadCommand(program: Command) {
       withDb(program, async (conn) => {
         let resolvedId: string;
         if (id) {
-          const found = await resolveThreadId(conn, id);
-          if (!found) {
-            logger.error(`Thread not found: ${id}`);
-            process.exit(1);
-          }
-          resolvedId = found;
+          resolvedId = id;
         } else {
           const active = await getActiveThread(conn);
           if (!active) {
@@ -130,7 +114,7 @@ export function registerThreadCommand(program: Command) {
 
         const pollMs = opts.interval ?? 500;
         logger.info(
-          `Following thread ${ansis.dim(resolvedId.slice(0, 8))}... (Ctrl+C to stop)`,
+          `Following thread ${ansis.dim(resolvedId)}... (Ctrl+C to stop)`,
         );
 
         const interval = setInterval(async () => {
@@ -168,24 +152,6 @@ export function registerThreadCommand(program: Command) {
     );
 }
 
-async function resolveThreadId(
-  conn: DbConnection,
-  idPrefix: string,
-): Promise<string | null> {
-  if (idPrefix.length >= 36) return idPrefix;
-  const all = await listThreads(conn);
-  const matches = all.filter((t) => t.id.startsWith(idPrefix));
-  if (matches.length === 1) {
-    const match = matches[0] as Thread;
-    return match.id;
-  }
-  if (matches.length === 0) return null;
-  logger.error(
-    `Ambiguous thread prefix "${idPrefix}" matches ${matches.length} threads`,
-  );
-  process.exit(1);
-}
-
 function typeColor(type: Thread["type"]): string {
   switch (type) {
     case "daemon_tick":
@@ -213,7 +179,7 @@ function roleColor(role: Interaction["role"]): string {
 }
 
 function printThread(t: Thread) {
-  const id = ansis.dim(t.id.slice(0, 8));
+  const id = ansis.dim(t.id);
   const title = t.title || ansis.dim("(untitled)");
   console.log(`  ${id}  ${typeColor(t.type)}  ${statusLabel(t)}  ${title}`);
 }

package/src/commands/with-db.ts

@@ -13,8 +13,8 @@ export async function withDb<T>(
   fn: (conn: DbConnection, dir: string) => Promise<T>,
 ): Promise<T> {
   const dir = program.opts().dir;
-  const conn = getConnection(getDbPath(dir));
-  migrate(conn);
+  const conn = await getConnection(getDbPath(dir));
+  await migrate(conn);
   try {
     return await fn(conn, dir);
   } finally {

package/src/constants.ts

@@ -12,7 +12,7 @@ export const DEFAULTS = {
   UPDATE_CHECK_INTERVAL_MS: 24 * 60 * 60 * 1000, // 24 hours
   UPDATE_CHECK_TIMEOUT_MS: 5_000,
 } as const;
-export const DB_FILENAME = "data.sqlite";
+export const DB_FILENAME = "data.duckdb";
 export const PID_FILENAME = "daemon.pid";
 export const LOG_FILENAME = "daemon.log";
 export const CONFIG_FILENAME = "config.json";

package/src/context/chunker.ts

@@ -1,16 +1,14 @@
 import Anthropic from "@anthropic-ai/sdk";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { logger } from "../utils/logger.ts";
 
 export interface Chunk {
   index: number;
   content: string;
 }
 
-const DEFAULT_WINDOW_CHARS = 2000;
-const DEFAULT_OVERLAP_CHARS = 200;
 const SHORT_CONTENT_THRESHOLD = 200;
 const LLM_TIMEOUT_MS = 10_000;
+const DEFAULT_OVERLAP_LINES = 2;
 
 const CHUNKER_TOOL_NAME = "return_chunks";
 const CHUNKER_TOOL = {
@@ -44,42 +42,23 @@ const CHUNKER_TOOL = {
 };
 
 /**
- * Deterministic sliding-window chunker.
- * Splits content into overlapping windows of approximately `windowChars` characters,
- * breaking at newlines when possible.
+ * Add overlapping lines from the end of each chunk to the start of the next.
+ * Improves retrieval when concepts span chunk boundaries.
  */
-export function chunkWithSlidingWindow(
-  content: string,
-  windowChars = DEFAULT_WINDOW_CHARS,
-  overlapChars = DEFAULT_OVERLAP_CHARS,
+export function addOverlapToChunks(
+  chunks: Chunk[],
+  overlapLines = DEFAULT_OVERLAP_LINES,
 ): Chunk[] {
-  if (content.length <= windowChars) {
-    return [{ index: 0, content }];
-  }
-
-  const chunks: Chunk[] = [];
-  let start = 0;
-  let index = 0;
-
-  while (start < content.length) {
-    let end = Math.min(start + windowChars, content.length);
-
-    // Try to break at a newline near the end of the window
-    if (end < content.length) {
-      const lastNewline = content.lastIndexOf("\n", end);
-      if (lastNewline > start + windowChars / 2) {
-        end = lastNewline + 1;
-      }
-    }
-
-    chunks.push({ index, content: content.slice(start, end) });
-    index++;
-
-    if (end >= content.length) break;
-    start = end - overlapChars;
-  }
-
-  return chunks;
+  if (chunks.length <= 1 || overlapLines <= 0) return chunks;
+
+  return chunks.map((c, i) => {
+    if (i === 0) return { ...c };
+    const prevChunk = chunks[i - 1];
+    if (!prevChunk) return { ...c };
+    const prevLines = prevChunk.content.split("\n");
+    const overlap = prevLines.slice(-overlapLines).join("\n");
+    return { ...c, content: `${overlap}\n${c.content}` };
+  });
 }
 
 /**
@@ -139,7 +118,7 @@ ${content}`,
 }
 
 /**
- * Chunk content using LLM when possible, falling back to sliding window.
+ * Chunk content using the LLM chunker.
  * Short content (<200 chars) is returned as a single chunk.
  */
 export async function chunk(
@@ -151,14 +130,12 @@ export async function chunk(
     return [{ index: 0, content }];
   }
 
-  // Only try LLM chunking if we have an API key
-  if (config.anthropic_api_key) {
-    try {
-      return await chunkWithLLM(content, mimeType, config);
-    } catch (err) {
-      logger.debug(`LLM chunking failed, using sliding window: ${err}`);
-    }
+  if (!config.anthropic_api_key) {
+    throw new Error(
+      "Anthropic API key is required for chunking. Set anthropic_api_key in config.",
+    );
   }
 
-  return chunkWithSlidingWindow(content);
+  const chunks = await chunkWithLLM(content, mimeType, config);
+  return addOverlapToChunks(chunks);
 }

package/src/context/describer.ts

@@ -0,0 +1,146 @@
+import Anthropic from "@anthropic-ai/sdk";
+import type { BotholomewConfig } from "../config/schemas.ts";
+import { logger } from "../utils/logger.ts";
+
+const DESCRIBE_TOOL_NAME = "return_description";
+const DESCRIBE_TOOL = {
+  name: DESCRIBE_TOOL_NAME,
+  description: "Return a one-sentence description of this content.",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      description: {
+        type: "string",
+        description:
+          "A concise one-sentence summary of what this content is about.",
+      },
+    },
+    required: ["description"],
+  },
+};
+
+const TIMEOUT_MS = 10_000;
+const MAX_CONTENT_CHARS = 8000;
+const MAX_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
+
+const IMAGE_TYPES = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+]);
+
+type ImageMediaType = "image/jpeg" | "image/png" | "image/gif" | "image/webp";
+
+/**
+ * Build the message content array for the LLM description request.
+ * Attaches the file as an image or document block when possible.
+ */
+async function buildMessageContent(
+  opts: DescriberOpts,
+): Promise<Anthropic.Messages.ContentBlockParam[]> {
+  const textPrompt = `Describe this file in one sentence. Be specific about what it contains, not generic.\n\nFilename: ${opts.filename}\nMIME type: ${opts.mimeType}`;
+
+  // Text file — include content inline
+  if (opts.content) {
+    const truncated =
+      opts.content.length > MAX_CONTENT_CHARS
+        ? `${opts.content.slice(0, MAX_CONTENT_CHARS)}\n... (truncated)`
+        : opts.content;
+    return [{ type: "text", text: `${textPrompt}\n\nContent:\n${truncated}` }];
+  }
+
+  // Binary file — try to attach if we have a file path
+  if (opts.filePath) {
+    const file = Bun.file(opts.filePath);
+    const size = file.size;
+
+    if (size > 0 && size <= MAX_FILE_BYTES) {
+      const data = Buffer.from(await file.arrayBuffer()).toString("base64");
+
+      if (IMAGE_TYPES.has(opts.mimeType)) {
+        return [
+          {
+            type: "image",
+            source: {
+              type: "base64",
+              media_type: opts.mimeType as ImageMediaType,
+              data,
+            },
+          },
+          { type: "text", text: textPrompt },
+        ];
+      }
+
+      if (opts.mimeType === "application/pdf") {
+        return [
+          {
+            type: "document",
+            source: { type: "base64", media_type: "application/pdf", data },
+          },
+          { type: "text", text: textPrompt },
+        ];
+      }
+    }
+  }
+
+  // Fallback — describe from filename and MIME type only
+  return [
+    {
+      type: "text",
+      text: `${textPrompt}\n\n(Binary file — no content preview available)`,
+    },
+  ];
+}
+
+interface DescriberOpts {
+  filename: string;
+  mimeType: string;
+  content: string | null;
+  filePath?: string;
+}
+
+/**
+ * Generate a short description of a file using the LLM.
+ * For textual files, summarises the content.
+ * For binary files, attaches images/PDFs directly or describes from metadata.
+ */
+export async function generateDescription(
+  config: Required<BotholomewConfig>,
+  opts: DescriberOpts,
+): Promise<string> {
+  if (!config.anthropic_api_key) {
+    return "";
+  }
+
+  const client = new Anthropic({ apiKey: config.anthropic_api_key });
+
+  try {
+    const content = await buildMessageContent(opts);
+
+    const response = await Promise.race([
+      client.messages.create({
+        model: config.chunker_model,
+        max_tokens: 256,
+        tools: [DESCRIBE_TOOL],
+        tool_choice: { type: "tool", name: DESCRIBE_TOOL_NAME },
+        messages: [{ role: "user", content }],
+      }),
+      new Promise<never>((_, reject) =>
+        setTimeout(
+          () => reject(new Error("Description generation timeout")),
+          TIMEOUT_MS,
+        ),
+      ),
+    ]);
+
+    const toolBlock = response.content.find((b) => b.type === "tool_use");
+    if (!toolBlock || toolBlock.type !== "tool_use") return "";
+
+    const input = toolBlock.input as { description: string };
+    return input.description || "";
+  } catch (err) {
+    logger.debug(`Description generation failed: ${err}`);
+    return "";
+  }
+}