botholomew 0.7.8 → 0.7.10

package/README.md

@@ -128,6 +128,8 @@ Everything the agent can touch is here. No surprises.
 | `botholomew nuke context\|tasks\|schedules\|threads\|all` | Bulk-erase sections of the database |
 | `botholomew upgrade` | Self-update |
 
+All `list` subcommands support `-l, --limit <n>` and `-o, --offset <n>` for pagination.
+
 ---
 
 ## How it works

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.7.8",
+  "version": "0.7.10",
   "description": "Local, autonomous AI agent for knowledge work — works your task queue while you sleep.",
   "type": "module",
   "bin": {

package/src/chat/agent.ts

@@ -4,6 +4,7 @@ import type {
   ToolResultBlockParam,
   ToolUseBlock,
 } from "@anthropic-ai/sdk/resources/messages";
+import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { embedSingle } from "../context/embedder.ts";
 import { fitToContextWindow, getMaxInputTokens } from "../daemon/context.ts";
@@ -13,7 +14,7 @@ import {
   extractKeywords,
   loadPersistentContext,
 } from "../daemon/prompt.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { hybridSearch } from "../db/embeddings.ts";
 import { logInteraction } from "../db/threads.ts";
 import { registerAllTools } from "../tools/registry.ts";
@@ -60,7 +61,7 @@ export async function buildChatSystemPrompt(
   projectDir: string,
   options?: {
     keywordSource?: string;
-    conn?: DbConnection;
+    dbPath?: string;
     config?: Required<BotholomewConfig>;
   },
 ): Promise<string> {
@@ -74,12 +75,14 @@ export async function buildChatSystemPrompt(
   parts.push(...(await loadPersistentContext(projectDir, taskKeywords)));
 
   // Relevant context from embeddings search
-  const conn = options?.conn;
+  const dbPath = options?.dbPath;
   const config = options?.config;
-  if (conn && config?.openai_api_key && keywordSource) {
+  if (dbPath && config?.openai_api_key && keywordSource) {
     try {
       const queryVec = await embedSingle(keywordSource, config);
-      const results = await hybridSearch(conn, keywordSource, queryVec, 5);
+      const results = await withDb(dbPath, (conn) =>
+        hybridSearch(conn, keywordSource, queryVec, 5),
+      );
 
       if (results.length > 0) {
         parts.push("## Relevant Context");
@@ -160,13 +163,20 @@ export async function runChatTurn(input: {
   messages: MessageParam[];
   projectDir: string;
   config: Required<BotholomewConfig>;
-  conn: DbConnection;
+  dbPath: string;
   threadId: string;
-  toolCtx: ToolContext;
+  mcpxClient: McpxClient | null;
   callbacks: ChatTurnCallbacks;
 }): Promise<void> {
-  const { messages, projectDir, config, conn, threadId, toolCtx, callbacks } =
-    input;
+  const {
+    messages,
+    projectDir,
+    config,
+    dbPath,
+    threadId,
+    mcpxClient,
+    callbacks,
+  } = input;
 
   const client = new Anthropic({
     apiKey: config.anthropic_api_key || undefined,
@@ -190,7 +200,7 @@ export async function runChatTurn(input: {
     const keywordSource = findLastUserText(messages);
     const systemPrompt = await buildChatSystemPrompt(projectDir, {
       keywordSource,
-      conn,
+      dbPath,
       config,
     });
 
@@ -230,13 +240,15 @@ export async function runChatTurn(input: {
 
     // Log assistant text
     if (assistantText) {
-      await logInteraction(conn, threadId, {
-        role: "assistant",
-        kind: "message",
-        content: assistantText,
-        durationMs,
-        tokenCount,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "assistant",
+          kind: "message",
+          content: assistantText,
+          durationMs,
+          tokenCount,
+        }),
+      );
     }
 
     // Check for tool calls
@@ -260,20 +272,29 @@ export async function runChatTurn(input: {
         callbacks.onToolStart(toolUse.id, toolUse.name, toolInput);
       }
 
-      await logInteraction(conn, threadId, {
-        role: "assistant",
-        kind: "tool_use",
-        content: `Calling ${toolUse.name}`,
-        toolName: toolUse.name,
-        toolInput,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "assistant",
+          kind: "tool_use",
+          content: `Calling ${toolUse.name}`,
+          toolName: toolUse.name,
+          toolInput,
+        }),
+      );
     }
 
-    // Execute all tools in parallel
+    // Execute all tools in parallel. Each tool call opens its own short-lived
+    // connection; parallel calls share the process-local DuckDB instance and
+    // release the file lock as soon as the last one finishes.
     const execResults = await Promise.all(
       toolUseBlocks.map(async (toolUse) => {
         const start = Date.now();
-        const result = await executeChatToolCall(toolUse, toolCtx);
+        const result = await executeChatToolCall(toolUse, {
+          dbPath,
+          projectDir,
+          config,
+          mcpxClient,
+        });
         const durationMs = Date.now() - start;
         const stored = maybeStoreResult(toolUse.name, result.output);
         const meta: ToolEndMeta | undefined = stored.stored
@@ -293,13 +314,15 @@ export async function runChatTurn(input: {
     // Log results and collect tool_result messages
     const toolResults: ToolResultBlockParam[] = [];
     for (const { toolUse, result, durationMs, stored } of execResults) {
-      await logInteraction(conn, threadId, {
-        role: "tool",
-        kind: "tool_result",
-        content: result.output,
-        toolName: toolUse.name,
-        durationMs,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "tool",
+          kind: "tool_result",
+          content: result.output,
+          toolName: toolUse.name,
+          durationMs,
+        }),
+      );
 
       toolResults.push({
         type: "tool_result",
@@ -314,9 +337,16 @@ export async function runChatTurn(input: {
   }
 }
 
+interface ChatToolCallCtx {
+  dbPath: string;
+  projectDir: string;
+  config: Required<BotholomewConfig>;
+  mcpxClient: McpxClient | null;
+}
+
 async function executeChatToolCall(
   toolUse: ToolUseBlock,
-  ctx: ToolContext,
+  baseCtx: ChatToolCallCtx,
 ): Promise<{ output: string; isError: boolean }> {
   const tool = getTool(toolUse.name);
   if (!tool) return { output: `Unknown tool: ${toolUse.name}`, isError: true };
@@ -335,7 +365,10 @@ async function executeChatToolCall(
   }
 
   try {
-    const result = await tool.execute(parsed.data, ctx);
+    const result = await withDb(baseCtx.dbPath, (conn) => {
+      const ctx: ToolContext = { ...baseCtx, conn };
+      return tool.execute(parsed.data, ctx);
+    });
     const isError =
       typeof result === "object" && result !== null && "is_error" in result
         ? (result as { is_error: boolean }).is_error

package/src/chat/session.ts

@@ -2,8 +2,7 @@ import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
 import { loadConfig } from "../config/loader.ts";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { getDbPath } from "../constants.ts";
-import type { DbConnection } from "../db/connection.ts";
-import { getConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { migrate } from "../db/schema.ts";
 import {
   createThread,
@@ -15,18 +14,18 @@ import {
 import { createMcpxClient } from "../mcpx/client.ts";
 import { loadSkills } from "../skills/loader.ts";
 import type { SkillDefinition } from "../skills/parser.ts";
-import type { ToolContext } from "../tools/tool.ts";
 import { generateThreadTitle } from "../utils/title.ts";
 import { type ChatTurnCallbacks, runChatTurn } from "./agent.ts";
 
 export interface ChatSession {
-  conn: DbConnection;
+  dbPath: string;
   threadId: string;
   projectDir: string;
   config: Required<BotholomewConfig>;
   messages: MessageParam[];
-  toolCtx: ToolContext;
   skills: Map<string, SkillDefinition>;
+  // biome-ignore lint/suspicious/noExplicitAny: mcpx client
+  mcpxClient: any;
   cleanup: () => Promise<void>;
 }
 
@@ -42,21 +41,22 @@ export async function startChatSession(
     );
   }
 
-  const conn = await getConnection(getDbPath(projectDir));
-  await migrate(conn);
+  const dbPath = getDbPath(projectDir);
+  await withDb(dbPath, (conn) => migrate(conn));
 
   let threadId: string;
   const messages: MessageParam[] = [];
 
   if (existingThreadId) {
     // Resume existing thread
-    const result = await getThread(conn, existingThreadId);
+    const result = await withDb(dbPath, (conn) =>
+      getThread(conn, existingThreadId),
+    );
     if (!result) {
-      conn.close();
       throw new Error(`Thread not found: ${existingThreadId}`);
     }
     threadId = existingThreadId;
-    await reopenThread(conn, threadId);
+    await withDb(dbPath, (conn) => reopenThread(conn, threadId));
 
     // Rebuild message history from interactions
     let firstUserMessage: string | undefined;
@@ -72,34 +72,29 @@ export async function startChatSession(
 
     // Backfill title for threads that still have the default
     if (result.thread.title === "New chat" && firstUserMessage) {
-      void generateThreadTitle(config, conn, threadId, firstUserMessage);
+      void generateThreadTitle(config, dbPath, threadId, firstUserMessage);
     }
   } else {
-    threadId = await createThread(conn, "chat_session", undefined, "New chat");
+    threadId = await withDb(dbPath, (conn) =>
+      createThread(conn, "chat_session", undefined, "New chat"),
+    );
   }
 
   const mcpxClient = await createMcpxClient(projectDir);
   const skills = await loadSkills(projectDir);
 
-  const toolCtx: ToolContext = {
-    conn,
-    projectDir,
-    config,
-    mcpxClient,
-  };
-
   const cleanup = async () => {
     await mcpxClient?.close();
   };
 
   return {
-    conn,
+    dbPath,
     threadId,
     projectDir,
     config,
     messages,
-    toolCtx,
     skills,
+    mcpxClient,
     cleanup,
   };
 }
@@ -110,11 +105,13 @@ export async function sendMessage(
   callbacks: ChatTurnCallbacks,
 ): Promise<void> {
   // Log and append user message
-  await logInteraction(session.conn, session.threadId, {
-    role: "user",
-    kind: "message",
-    content: userMessage,
-  });
+  await withDb(session.dbPath, (conn) =>
+    logInteraction(conn, session.threadId, {
+      role: "user",
+      kind: "message",
+      content: userMessage,
+    }),
+  );
 
   session.messages.push({ role: "user", content: userMessage });
 
@@ -122,7 +119,7 @@ export async function sendMessage(
   if (session.messages.length === 1) {
     void generateThreadTitle(
       session.config,
-      session.conn,
+      session.dbPath,
       session.threadId,
       userMessage,
     );
@@ -132,15 +129,14 @@ export async function sendMessage(
     messages: session.messages,
     projectDir: session.projectDir,
     config: session.config,
-    conn: session.conn,
+    dbPath: session.dbPath,
     threadId: session.threadId,
-    toolCtx: session.toolCtx,
+    mcpxClient: session.mcpxClient,
     callbacks,
   });
 }
 
 export async function endChatSession(session: ChatSession): Promise<void> {
-  await endThread(session.conn, session.threadId);
+  await withDb(session.dbPath, (conn) => endThread(conn, session.threadId));
   await session.cleanup();
-  session.conn.close();
 }

package/src/commands/context.ts

@@ -25,6 +25,7 @@ import {
   createContextItemStrict,
   deleteContextItemByPath,
   getContextItemByPath,
+  getContextItemBySourcePath,
   listContextItems,
   listContextItemsByPrefix,
   PathConflictError,
@@ -193,9 +194,126 @@ export function registerContextCommand(program: Command) {
           text: `Found ${totalCount} item(s) to add (${filesToAdd.length} file(s), ${urlsToAdd.length} URL(s)).`,
         });
 
-        // Phase 1.5: LLM placement for files without an explicit path
         const config = await loadConfig(dir);
         const CONCURRENCY = 10;
+
+        // Phase 0: Source-path dedup — items whose source_path is already in
+        // context are routed per --on-conflict before we pay for LLM placement.
+        type AlreadyInContext = {
+          sourcePath: string;
+          sourceType: "file" | "url";
+          existing: ContextItem;
+        };
+        const alreadyInContext: AlreadyInContext[] = [];
+        const remainingFiles: FileToAdd[] = [];
+        const remainingUrls: { url: string; contextPath: string }[] = [];
+
+        for (const f of filesToAdd) {
+          const existing = await getContextItemBySourcePath(
+            conn,
+            f.filePath,
+            "file",
+          );
+          if (existing) {
+            alreadyInContext.push({
+              sourcePath: f.filePath,
+              sourceType: "file",
+              existing,
+            });
+          } else {
+            remainingFiles.push(f);
+          }
+        }
+        for (const u of urlsToAdd) {
+          const existing = await getContextItemBySourcePath(conn, u.url, "url");
+          if (existing) {
+            alreadyInContext.push({
+              sourcePath: u.url,
+              sourceType: "url",
+              existing,
+            });
+          } else {
+            remainingUrls.push(u);
+          }
+        }
+
+        let refreshedCount = 0;
+        let refreshedChunks = 0;
+        const dedupSkipped: string[] = [];
+
+        if (alreadyInContext.length > 0) {
+          if (policy === "error") {
+            logger.error(
+              `${alreadyInContext.length} item(s) already in context (matched by source path):`,
+            );
+            for (const a of alreadyInContext) {
+              console.log(
+                `  ${ansis.red("✗")} ${a.sourcePath} → ${a.existing.context_path} (id: ${a.existing.id})`,
+              );
+            }
+            logger.dim(
+              "Re-run with --on-conflict=skip to ignore these items or --on-conflict=overwrite to refresh them from disk.",
+            );
+            process.exit(1);
+          }
+
+          if (policy === "skip") {
+            for (const a of alreadyInContext) {
+              logger.dim(
+                `⊘ already in context: ${a.sourcePath} → ${a.existing.context_path}`,
+              );
+              dedupSkipped.push(a.existing.context_path);
+            }
+          } else {
+            // overwrite: refresh existing items (diff + selective re-embed),
+            // preserving their original context_path.
+            const itemsToRefresh = alreadyInContext.map((a) => a.existing);
+            const hasUrls = itemsToRefresh.some((i) => i.source_type === "url");
+            const mcpxClient = hasUrls ? await createMcpxClient(dir) : null;
+
+            const refreshSpinner = createSpinner(
+              `Refreshing 0/${itemsToRefresh.length} existing item(s)...`,
+            ).start();
+            const refreshResult = await refreshContextItems(
+              conn,
+              itemsToRefresh,
+              config,
+              mcpxClient,
+              {
+                onItemProgress: (done, total) => {
+                  refreshSpinner.update({
+                    text: `Refreshing ${done}/${total} existing item(s)...`,
+                  });
+                },
+              },
+            );
+            refreshSpinner.success({
+              text: `Refreshed ${refreshResult.checked} existing item(s): ${refreshResult.updated} updated, ${refreshResult.unchanged} unchanged, ${refreshResult.missing} missing.`,
+            });
+
+            // Count everything we processed OK (updated + unchanged) as
+            // "refreshed" for the summary. Missing/error items are reported
+            // inline below and don't count toward success.
+            refreshedCount = refreshResult.updated + refreshResult.unchanged;
+            refreshedChunks = refreshResult.chunks;
+            for (const item of refreshResult.items) {
+              if (item.status === "missing") {
+                logger.warn(`  Missing: ${item.source_path}`);
+              } else if (item.status === "error") {
+                logger.warn(
+                  `  Error refreshing ${item.source_path}: ${item.error}`,
+                );
+              }
+            }
+          }
+        }
+
+        // Drop already-handled items from the work lists so downstream phases
+        // (LLM placement, description, insert, embed) see only truly-new items.
+        filesToAdd.splice(0, filesToAdd.length, ...remainingFiles);
+        urlsToAdd.splice(0, urlsToAdd.length, ...remainingUrls);
+
+        // Phase 1.5: LLM placement for files without an explicit path
         const needsPlacement = filesToAdd.filter((f) => f.contextPath === null);
         // description cache keyed by filePath — populated when LLM placement runs,
         // reused in addFile to avoid a second describe call.
@@ -378,10 +496,13 @@ export function registerContextCommand(program: Command) {
           }
         }
 
-        // Report conflicts before embeddings so the user sees them prominently
+        // Report conflicts before embeddings so the user sees them prominently.
+        // Phase 0 already handled source-path matches, so anything here is a
+        // target-path collision — an LLM-suggested (or explicit) path that
+        // another unrelated item already occupies.
         if (conflicts.length > 0) {
           logger.error(
-            `${conflicts.length} path collision(s) — nothing written for these items:`,
+            `${conflicts.length} target-path collision(s) — nothing written for these items:`,
           );
           for (const c of conflicts) {
             console.log(
@@ -389,24 +510,34 @@ export function registerContextCommand(program: Command) {
             );
           }
           logger.dim(
-            "Re-run with --on-conflict=overwrite to replace, --on-conflict=skip to ignore, or --name / --prefix to place elsewhere.",
+            "The suggested path is already in use by a different source. Re-run with --prefix to place these items elsewhere, or delete the existing item first.",
           );
         }
 
+        // Merge Phase 0 skips into the skip list used by the final summary.
+        skipped.push(...dedupSkipped);
+
         // Phase 3: Chunk + embed in parallel (network I/O)
         if (itemIds.length === 0 || !config.openai_api_key) {
           if (!config.openai_api_key) {
             logger.dim("Skipping embeddings (no OpenAI API key configured).");
           }
-          const msg = `Added ${itemIds.length}/${totalCount} item(s), 0 chunks indexed.`;
+          const msg = buildSummary({
+            added: itemIds.length,
+            refreshed: refreshedCount,
+            skipped: skipped.length,
+            chunks: refreshedChunks,
+            totalCount,
+            handled: itemIds.length + refreshedCount + skipped.length,
+          });
           if (conflicts.length > 0) {
             logger.error(msg);
             process.exit(1);
           }
-          if (itemIds.length === totalCount - skipped.length) {
+          if (itemIds.length + skipped.length + refreshedCount >= totalCount) {
             logger.success(msg);
             process.exit(0);
-          } else if (itemIds.length === 0) {
+          } else if (itemIds.length === 0 && refreshedCount === 0) {
             logger.error(msg);
             process.exit(1);
           } else {
@@ -452,15 +583,20 @@ export function registerContextCommand(program: Command) {
           else filesAdded++;
         }
 
-        const parts: string[] = [];
-        if (filesAdded > 0) parts.push(`${filesAdded} added`);
-        if (filesUpdated > 0) parts.push(`${filesUpdated} updated`);
-        const summary = `${parts.join(", ")} — ${chunks} chunk(s) indexed (${itemIds.length}/${totalCount} item(s)).`;
+        const summary = buildSummary({
+          added: filesAdded,
+          updated: filesUpdated,
+          refreshed: refreshedCount,
+          skipped: skipped.length,
+          chunks: chunks + refreshedChunks,
+          totalCount,
+          handled: itemIds.length + refreshedCount + skipped.length,
+        });
         if (conflicts.length > 0) {
           logger.error(summary);
           process.exit(1);
         }
-        if (itemIds.length === totalCount - skipped.length) {
+        if (itemIds.length + skipped.length + refreshedCount >= totalCount) {
           logger.success(summary);
           process.exit(0);
         } else {
@@ -675,6 +811,26 @@ async function resolveItems(
 
 type ConflictPolicy = "error" | "overwrite" | "skip";
 
+/** Format the final "X added, Y refreshed, Z skipped — N chunks" line. */
+function buildSummary(args: {
+  added: number;
+  updated?: number;
+  refreshed: number;
+  skipped: number;
+  chunks: number;
+  totalCount: number;
+  handled?: number;
+}): string {
+  const parts: string[] = [];
+  if (args.added > 0) parts.push(`${args.added} added`);
+  if (args.updated && args.updated > 0) parts.push(`${args.updated} updated`);
+  if (args.refreshed > 0) parts.push(`${args.refreshed} refreshed`);
+  if (args.skipped > 0) parts.push(`${args.skipped} skipped`);
+  const body = parts.length > 0 ? parts.join(", ") : "0 added";
+  const handled = args.handled ?? args.added + args.refreshed + args.skipped;
+  return `${body} — ${args.chunks} chunk(s) indexed (${handled}/${args.totalCount} item(s)).`;
+}
+
 type AddFileResult =
   | { kind: "added"; id: string; contextPath: string }
   | { kind: "skipped"; contextPath: string }

package/src/commands/daemon.ts

@@ -109,7 +109,9 @@ export function registerDaemonCommand(program: Command) {
   daemon
     .command("list")
     .description("List all registered Botholomew projects on this machine")
-    .action(async () => {
+    .option("-l, --limit <n>", "max number of projects", Number.parseInt)
+    .option("-o, --offset <n>", "skip first N projects", Number.parseInt)
+    .action(async (opts: { limit?: number; offset?: number }) => {
       const { listAllWatchdogProjects } = await import("../daemon/watchdog.ts");
       try {
         const projects = await listAllWatchdogProjects();
@@ -117,10 +119,21 @@ export function registerDaemonCommand(program: Command) {
           logger.dim("No registered projects found.");
           return;
         }
-        for (const p of projects) {
+        const total = projects.length;
+        const start = opts.offset ?? 0;
+        const end = opts.limit ? start + opts.limit : undefined;
+        const page = projects.slice(start, end);
+        if (page.length === 0) {
+          logger.dim(`No projects on this page (total: ${total}).`);
+          return;
+        }
+        for (const p of page) {
           logger.info(p.projectDir);
           logger.dim(`  Config: ${p.configPath}`);
         }
+        if (page.length !== total) {
+          logger.dim(`\nshowing ${page.length} of ${total} project(s)`);
+        }
       } catch (err) {
         logger.error(
           `Failed to list projects: ${err instanceof Error ? err.message : err}`,

package/src/commands/schedule.ts

@@ -19,9 +19,18 @@ export function registerScheduleCommand(program: Command) {
     .description("List all schedules")
     .option("--enabled", "show only enabled schedules")
     .option("--disabled", "show only disabled schedules")
+    .option("-l, --limit <n>", "max number of schedules", Number.parseInt)
+    .option("-o, --offset <n>", "skip first N schedules", Number.parseInt)
     .action((opts) =>
       withDb(program, async (conn) => {
-        const filters: { enabled?: boolean } = {};
+        const filters: {
+          enabled?: boolean;
+          limit?: number;
+          offset?: number;
+        } = {
+          limit: opts.limit,
+          offset: opts.offset,
+        };
         if (opts.enabled) filters.enabled = true;
         if (opts.disabled) filters.enabled = false;