botholomew 0.7.8 → 0.7.9

package/src/daemon/llm.ts

@@ -7,7 +7,7 @@ import type {
 } from "@anthropic-ai/sdk/resources/messages";
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { getTask, type Task } from "../db/tasks.ts";
 import { logInteraction } from "../db/threads.ts";
 import { registerAllTools } from "../tools/registry.ts";
@@ -44,32 +44,32 @@ export async function runAgentLoop(input: {
   systemPrompt: string;
   task: Task;
   config: Required<BotholomewConfig>;
-  conn: DbConnection;
+  dbPath: string;
   threadId: string;
   projectDir: string;
   mcpxClient?: McpxClient | null;
   callbacks?: DaemonStreamCallbacks;
 }): Promise<AgentLoopResult> {
-  const { systemPrompt, task, config, conn, threadId, projectDir, callbacks } =
-    input;
+  const {
+    systemPrompt,
+    task,
+    config,
+    dbPath,
+    threadId,
+    projectDir,
+    callbacks,
+  } = input;
 
   const client = new Anthropic({
     apiKey: config.anthropic_api_key || undefined,
   });
 
-  const toolCtx: ToolContext = {
-    conn,
-    projectDir,
-    config,
-    mcpxClient: input.mcpxClient ?? null,
-  };
-
   // Build predecessor context from completed blocking tasks
   let predecessorContext = "";
   if (task.blocked_by.length > 0) {
     const predecessorOutputs: string[] = [];
     for (const blockerId of task.blocked_by) {
-      const blocker = await getTask(conn, blockerId);
+      const blocker = await withDb(dbPath, (conn) => getTask(conn, blockerId));
       if (blocker?.output) {
         predecessorOutputs.push(
           `### ${blocker.name} (${blocker.id})\n${blocker.output}`,
@@ -86,11 +86,13 @@ export async function runAgentLoop(input: {
   const messages: MessageParam[] = [{ role: "user", content: userMessage }];
 
   // Log the initial user message
-  await logInteraction(conn, threadId, {
-    role: "user",
-    kind: "message",
-    content: userMessage,
-  });
+  await withDb(dbPath, (conn) =>
+    logInteraction(conn, threadId, {
+      role: "user",
+      kind: "message",
+      content: userMessage,
+    }),
+  );
 
   clearLargeResults();
   const daemonTools = toAnthropicTools();
@@ -144,13 +146,15 @@ export async function runAgentLoop(input: {
     // Log assistant text blocks
     for (const block of response.content) {
       if (block.type === "text" && block.text) {
-        await logInteraction(conn, threadId, {
-          role: "assistant",
-          kind: "message",
-          content: block.text,
-          durationMs,
-          tokenCount,
-        });
+        await withDb(dbPath, (conn) =>
+          logInteraction(conn, threadId, {
+            role: "assistant",
+            kind: "message",
+            content: block.text,
+            durationMs,
+            tokenCount,
+          }),
+        );
       }
     }
 
@@ -173,20 +177,30 @@ export async function runAgentLoop(input: {
     for (const toolUse of toolUseBlocks) {
       const toolInput = JSON.stringify(toolUse.input);
       callbacks?.onToolStart(toolUse.name, toolInput);
-      await logInteraction(conn, threadId, {
-        role: "assistant",
-        kind: "tool_use",
-        content: `Calling ${toolUse.name}`,
-        toolName: toolUse.name,
-        toolInput,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "assistant",
+          kind: "tool_use",
+          content: `Calling ${toolUse.name}`,
+          toolName: toolUse.name,
+          toolInput,
+        }),
+      );
     }
 
-    // Execute all tools in parallel
+    // Execute all tools in parallel. Each tool call opens its own short-lived
+    // connection (or none, if the tool uses dbPath internally) via
+    // executeToolCall — so parallel tool calls share the process-local
+    // DuckDB instance and release the file lock as soon as they finish.
     const execResults = await Promise.all(
       toolUseBlocks.map(async (toolUse) => {
         const start = Date.now();
-        const result = await executeToolCall(toolUse, toolCtx);
+        const result = await executeToolCall(toolUse, {
+          dbPath,
+          projectDir,
+          config,
+          mcpxClient: input.mcpxClient ?? null,
+        });
         const elapsed = Date.now() - start;
         callbacks?.onToolEnd(
           toolUse.name,
@@ -201,13 +215,15 @@ export async function runAgentLoop(input: {
     // Log results and collect tool_result messages
     const toolResults: ToolResultBlockParam[] = [];
     for (const { toolUse, result, durationMs } of execResults) {
-      await logInteraction(conn, threadId, {
-        role: "tool",
-        kind: "tool_result",
-        content: result.output,
-        toolName: toolUse.name,
-        durationMs,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "tool",
+          kind: "tool_result",
+          content: result.output,
+          toolName: toolUse.name,
+          durationMs,
+        }),
+      );
 
       if (result.terminal && result.agentResult) {
         return result.agentResult;
@@ -234,9 +250,16 @@ interface ToolCallResult {
   agentResult?: AgentLoopResult;
 }
 
+interface ToolCallCtx {
+  dbPath: string;
+  projectDir: string;
+  config: Required<BotholomewConfig>;
+  mcpxClient: McpxClient | null;
+}
+
 async function executeToolCall(
   toolUse: ToolUseBlock,
-  ctx: ToolContext,
+  baseCtx: ToolCallCtx,
 ): Promise<ToolCallResult> {
   const tool = getTool(toolUse.name);
   if (!tool) {
@@ -261,7 +284,10 @@ async function executeToolCall(
 
   let result: unknown;
   try {
-    result = await tool.execute(parsed.data, ctx);
+    result = await withDb(baseCtx.dbPath, (conn) => {
+      const ctx: ToolContext = { ...baseCtx, conn };
+      return tool.execute(parsed.data, ctx);
+    });
   } catch (err) {
     return {
       output: `Tool ${toolUse.name} threw an error: ${err}. You may retry with different parameters or try an alternative approach.`,

package/src/daemon/prompt.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { getBotholomewDir } from "../constants.ts";
 import { embedSingle } from "../context/embedder.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { hybridSearch } from "../db/embeddings.ts";
 import type { Task } from "../db/tasks.ts";
 import { parseContextFile } from "../utils/frontmatter.ts";
@@ -89,7 +89,7 @@ export function buildMetaHeader(projectDir: string): string[] {
 export async function buildSystemPrompt(
   projectDir: string,
   task?: Task,
-  conn?: DbConnection,
+  dbPath?: string,
   _config?: Required<BotholomewConfig>,
   options?: { hasMcpTools?: boolean },
 ): Promise<string> {
@@ -107,11 +107,13 @@ export async function buildSystemPrompt(
   parts.push(...(await loadPersistentContext(projectDir, taskKeywords)));
 
   // Relevant context from embeddings search
-  if (task && conn && _config?.openai_api_key) {
+  if (task && dbPath && _config?.openai_api_key) {
     try {
       const query = `${task.name} ${task.description}`;
       const queryVec = await embedSingle(query, _config);
-      const results = await hybridSearch(conn, query, queryVec, 5);
+      const results = await withDb(dbPath, (conn) =>
+        hybridSearch(conn, query, queryVec, 5),
+      );
 
       if (results.length > 0) {
         parts.push("## Relevant Context");

package/src/daemon/schedules.ts

@@ -1,6 +1,6 @@
 import Anthropic from "@anthropic-ai/sdk";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import {
   listSchedules,
   markScheduleRun,
@@ -105,14 +105,17 @@ Is this schedule due to run? If yes, what tasks should be created?`;
 }
 
 export async function processSchedules(
-  conn: DbConnection,
+  dbPath: string,
   config: Required<BotholomewConfig>,
 ): Promise<void> {
-  const schedules = await listSchedules(conn, { enabled: true });
+  const schedules = await withDb(dbPath, (conn) =>
+    listSchedules(conn, { enabled: true }),
+  );
   if (schedules.length === 0) return;
 
   for (const schedule of schedules) {
     try {
+      // LLM evaluation does no DB work — no connection held here.
       const evaluation = await evaluateSchedule(config, schedule);
 
       if (!evaluation.isDue) {
@@ -128,16 +131,18 @@ export async function processSchedules(
           .map((i: number) => createdIds[i])
           .filter(Boolean) as string[];
 
-        const task = await createTask(conn, {
-          name: taskDef.name,
-          description: taskDef.description,
-          priority: taskDef.priority,
-          blocked_by: blockedBy,
-        });
+        const task = await withDb(dbPath, (conn) =>
+          createTask(conn, {
+            name: taskDef.name,
+            description: taskDef.description,
+            priority: taskDef.priority,
+            blocked_by: blockedBy,
+          }),
+        );
         createdIds.push(task.id);
       }
 
-      await markScheduleRun(conn, schedule.id);
+      await withDb(dbPath, (conn) => markScheduleRun(conn, schedule.id));
       logger.info(
         `Schedule "${schedule.name}" fired, created ${createdIds.length} task(s)`,
       );

package/src/daemon/tick.ts

@@ -1,6 +1,6 @@
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { listSchedules } from "../db/schedules.ts";
 import {
   claimNextTask,
@@ -17,7 +17,7 @@ import { processSchedules } from "./schedules.ts";
 
 export async function tick(
   projectDir: string,
-  conn: DbConnection,
+  dbPath: string,
   config: Required<BotholomewConfig>,
   mcpxClient?: McpxClient | null,
   callbacks?: DaemonStreamCallbacks,
@@ -27,9 +27,8 @@ export async function tick(
   logger.phase("tick-start", `#${tickNum}`);
 
   // Reset stale tasks stuck in in_progress
-  const resetIds = await resetStaleTasks(
-    conn,
-    config.max_tick_duration_seconds * 3,
+  const resetIds = await withDb(dbPath, (conn) =>
+    resetStaleTasks(conn, config.max_tick_duration_seconds * 3),
   );
   if (resetIds.length > 0) {
     logger.warn(
@@ -37,12 +36,16 @@ export async function tick(
     );
   }
 
-  // Process schedules (may create new tasks)
-  const enabledSchedules = await listSchedules(conn, { enabled: true });
+  // Process schedules (may create new tasks). Check enabled count so we only
+  // log the phase when there's work to evaluate — the call itself is a no-op
+  // otherwise.
+  const enabledSchedules = await withDb(dbPath, (conn) =>
+    listSchedules(conn, { enabled: true }),
+  );
   if (enabledSchedules.length > 0) {
     logger.phase("evaluating-schedules", `${enabledSchedules.length} enabled`);
     try {
-      await processSchedules(conn, config);
+      await processSchedules(dbPath, config);
     } catch (err) {
       logger.error(`Schedule processing failed: ${err}`);
     }
@@ -50,7 +53,7 @@ export async function tick(
 
   // Claim a task
   logger.phase("claiming-task");
-  const task = await claimNextTask(conn);
+  const task = await withDb(dbPath, (conn) => claimNextTask(conn));
   if (!task) {
     logger.info("No task claimed (queue empty or all blocked)");
     const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);
@@ -62,67 +65,80 @@ export async function tick(
   callbacks?.onTaskStart(task);
 
   // Create a thread for this tick
-  const threadId = await createThread(
-    conn,
-    "daemon_tick",
-    task.id,
-    `Working: ${task.name}`,
+  const threadId = await withDb(dbPath, (conn) =>
+    createThread(conn, "daemon_tick", task.id, `Working: ${task.name}`),
   );
 
   // Build system prompt (includes task-relevant context from embeddings)
-  const systemPrompt = await buildSystemPrompt(projectDir, task, conn, config, {
-    hasMcpTools: mcpxClient != null,
-  });
+  const systemPrompt = await buildSystemPrompt(
+    projectDir,
+    task,
+    dbPath,
+    config,
+    {
+      hasMcpTools: mcpxClient != null,
+    },
+  );
 
   try {
     const result = await runAgentLoop({
       systemPrompt,
       task,
       config,
-      conn,
+      dbPath,
       threadId,
       projectDir,
       mcpxClient,
       callbacks,
     });
 
+    // Update task status and store output. Only completed tasks have an
+    // `output`; waiting/failed tasks put their reason in `waiting_reason`.
     const isComplete = result.status === "complete";
-    await updateTaskStatus(
-      conn,
-      task.id,
-      result.status,
-      isComplete ? null : result.reason,
-      isComplete ? result.reason : null,
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(
+        conn,
+        task.id,
+        result.status,
+        isComplete ? null : result.reason,
+        isComplete ? result.reason : null,
+      ),
     );
 
     // Log the status change
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
+      }),
+    );
 
     logger.info(`Task ${task.id} -> ${result.status}`);
 
-    // Generate a descriptive title for the thread
+    // Generate a descriptive title for the thread (fire-and-forget)
     void generateThreadTitle(
       config,
-      conn,
+      dbPath,
       threadId,
       `Task: ${task.name}\nDescription: ${task.description}\nOutcome: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`,
     );
   } catch (err) {
-    await updateTaskStatus(conn, task.id, "failed", String(err), null);
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(conn, task.id, "failed", String(err), null),
+    );
 
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} failed: ${err}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} failed: ${err}`,
+      }),
+    );
 
     logger.error(`Task ${task.id} failed: ${err}`);
   } finally {
-    await endThread(conn, threadId);
+    await withDb(dbPath, (conn) => endThread(conn, threadId));
   }
 
   const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);

package/src/db/connection.ts

@@ -11,12 +11,20 @@ export class DbConnection {
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
   private conn: any;
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  private instance: any;
+  private readonly ownedInstance: any;
+  private readonly dbPath: string;
+  private closed = false;
 
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  constructor(conn: any, instance: any) {
+  constructor(
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    conn: any,
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    ownedInstance: any,
+    dbPath: string,
+  ) {
     this.conn = conn;
-    this.instance = instance;
+    this.ownedInstance = ownedInstance;
+    this.dbPath = dbPath;
   }
 
   /** Execute raw SQL with no return value. */
@@ -62,10 +70,22 @@ export class DbConnection {
     return { changes: result.rowsChanged };
   }
 
-  /** Close the connection and dispose of the instance. */
+  /**
+   * Disconnect and release this connection's share of the DuckDB instance.
+   * For file-backed DBs, the instance is closed (and the OS file lock
+   * released) once every overlapping connection in this process has closed.
+   * For `:memory:` DBs, the instance is owned by this connection and closed
+   * immediately.
+   */
   close(): void {
+    if (this.closed) return;
+    this.closed = true;
     this.conn.disconnectSync();
-    this.instance.closeSync();
+    if (this.ownedInstance) {
+      this.ownedInstance.closeSync();
+    } else {
+      releaseInstance(this.dbPath);
+    }
   }
 }
 
@@ -98,31 +118,140 @@ function flattenParams(params: SqlParam[]): SqlParam[] {
   return params.map((p) => (Array.isArray(p) ? JSON.stringify(p) : p));
 }
 
+/**
+ * Refcounted, process-local cache of open DuckDB instances keyed by dbPath.
+ *
+ * DuckDB's file lock is held at the instance level, so we must close the
+ * instance — not just the connection — to let another process acquire the
+ * writer lock. At the same time, opening two instances for the same file
+ * from one process is unsafe. This cache resolves both: overlapping
+ * `getConnection` calls in the same process share a single instance; once
+ * every connection has closed, the instance is closed and evicted, which
+ * releases the OS file lock.
+ *
+ * `:memory:` paths bypass the cache so each test/caller gets its own
+ * isolated in-memory database.
+ */
+interface CachedInstance {
+  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+  instance: any;
+  refCount: number;
+}
+const instanceCache = new Map<string, CachedInstance>();
+const pendingInstance = new Map<string, Promise<CachedInstance>>();
+
+function isMemoryPath(path: string): boolean {
+  return path === ":memory:" || path.startsWith(":memory:");
+}
+
+async function acquireSharedInstance(dbPath: string): Promise<CachedInstance> {
+  const existing = instanceCache.get(dbPath);
+  if (existing) {
+    existing.refCount += 1;
+    return existing;
+  }
+  const inFlight = pendingInstance.get(dbPath);
+  if (inFlight) {
+    const cached = await inFlight;
+    cached.refCount += 1;
+    return cached;
+  }
+  const creation = (async () => {
+    const instance = await DuckDBInstance.create(dbPath);
+    const cached: CachedInstance = { instance, refCount: 1 };
+    instanceCache.set(dbPath, cached);
+    return cached;
+  })();
+  pendingInstance.set(dbPath, creation);
+  try {
+    return await creation;
+  } finally {
+    pendingInstance.delete(dbPath);
+  }
+}
+
+function releaseInstance(dbPath: string): void {
+  const cached = instanceCache.get(dbPath);
+  if (!cached) return;
+  cached.refCount -= 1;
+  if (cached.refCount <= 0) {
+    instanceCache.delete(dbPath);
+    cached.instance.closeSync();
+  }
+}
+
 export async function getConnection(dbPath?: string): Promise<DbConnection> {
-  const instance = await DuckDBInstance.create(dbPath ?? ":memory:");
-  const conn = await instance.connect();
+  const path = dbPath ?? ":memory:";
 
-  // Load VSS extension for vector similarity search
-  await conn.run("INSTALL vss; LOAD vss;");
-  // Enable HNSW index persistence for file-backed databases
-  await conn.run("SET hnsw_enable_experimental_persistence = true;");
+  if (isMemoryPath(path)) {
+    const instance = await DuckDBInstance.create(path);
+    const conn = await instance.connect();
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, instance, path);
+  }
 
-  return new DbConnection(conn, instance);
+  const cached = await acquireSharedInstance(path);
+  try {
+    const conn = await cached.instance.connect();
+    // INSTALL is a no-op after the first successful install (the extension
+    // is persisted to the user's DuckDB extension directory). LOAD is
+    // cheap per connection.
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, null, path);
+  } catch (err) {
+    releaseInstance(path);
+    throw err;
+  }
 }
 
+/**
+ * Open a DuckDB connection for a single logical unit of work and guarantee
+ * it is closed afterward. Retries on lock conflicts so two processes that
+ * race on the file lock cooperate instead of failing hard.
+ *
+ * Prefer one `withDb` per logical operation. The file lock is only released
+ * when every connection (across this process's overlapping callers) has
+ * been closed, so holding the connection across non-DB work (LLM calls,
+ * network I/O, filesystem walks) keeps other processes blocked.
+ */
+export async function withDb<T>(
+  dbPath: string,
+  fn: (conn: DbConnection) => Promise<T>,
+): Promise<T> {
+  const conn = await withRetry(() => getConnection(dbPath));
+  try {
+    return await fn(conn);
+  } finally {
+    conn.close();
+  }
+}
+
+/**
+ * Retry `fn` with exponential backoff when it fails with a DuckDB file-lock
+ * conflict ("Conflicting lock is held…"). Other errors propagate immediately.
+ */
 export async function withRetry<T>(
   fn: () => Promise<T>,
-  maxRetries = 5,
+  maxRetries = 8,
 ): Promise<T> {
   let lastError: unknown;
   for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
       return await fn();
     } catch (err) {
+      if (!isLockConflict(err)) throw err;
       lastError = err;
       if (attempt === maxRetries - 1) throw err;
+      // 100, 200, 400, 800, 1600, 3200, 6400, 12800 — up to ~25s total
       await Bun.sleep(100 * 2 ** attempt);
     }
   }
   throw lastError;
 }
+
+function isLockConflict(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return msg.includes("Conflicting lock") || msg.includes("could not be set");
+}

package/src/db/schedules.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildSetClauses, buildWhereClause } from "./query.ts";
+import { buildSetClauses, buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface Schedule {
@@ -72,7 +72,7 @@ export async function getSchedule(
 
 export async function listSchedules(
   db: DbConnection,
-  filters?: { enabled?: boolean },
+  filters?: { enabled?: boolean; limit?: number; offset?: number },
 ): Promise<Schedule[]> {
   const { where, params } = buildWhereClause([
     [
@@ -80,9 +80,13 @@ export async function listSchedules(
       filters?.enabled !== undefined ? (filters.enabled ? 1 : 0) : undefined,
     ],
   ]);
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<ScheduleRow>(
-    `SELECT * FROM schedules ${where} ORDER BY created_at ASC`,
+    `SELECT * FROM schedules ${where}
+     ORDER BY created_at ASC, id ASC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToSchedule);