botholomew 0.7.8 → 0.7.10

package/src/daemon/tick.ts

@@ -1,6 +1,6 @@
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { listSchedules } from "../db/schedules.ts";
 import {
   claimNextTask,
@@ -17,7 +17,7 @@ import { processSchedules } from "./schedules.ts";
 
 export async function tick(
   projectDir: string,
-  conn: DbConnection,
+  dbPath: string,
   config: Required<BotholomewConfig>,
   mcpxClient?: McpxClient | null,
   callbacks?: DaemonStreamCallbacks,
@@ -27,9 +27,8 @@ export async function tick(
   logger.phase("tick-start", `#${tickNum}`);
 
   // Reset stale tasks stuck in in_progress
-  const resetIds = await resetStaleTasks(
-    conn,
-    config.max_tick_duration_seconds * 3,
+  const resetIds = await withDb(dbPath, (conn) =>
+    resetStaleTasks(conn, config.max_tick_duration_seconds * 3),
   );
   if (resetIds.length > 0) {
     logger.warn(
@@ -37,12 +36,16 @@ export async function tick(
     );
   }
 
-  // Process schedules (may create new tasks)
-  const enabledSchedules = await listSchedules(conn, { enabled: true });
+  // Process schedules (may create new tasks). Check enabled count so we only
+  // log the phase when there's work to evaluate — the call itself is a no-op
+  // otherwise.
+  const enabledSchedules = await withDb(dbPath, (conn) =>
+    listSchedules(conn, { enabled: true }),
+  );
   if (enabledSchedules.length > 0) {
     logger.phase("evaluating-schedules", `${enabledSchedules.length} enabled`);
     try {
-      await processSchedules(conn, config);
+      await processSchedules(dbPath, config);
     } catch (err) {
       logger.error(`Schedule processing failed: ${err}`);
     }
@@ -50,7 +53,7 @@ export async function tick(
 
   // Claim a task
   logger.phase("claiming-task");
-  const task = await claimNextTask(conn);
+  const task = await withDb(dbPath, (conn) => claimNextTask(conn));
   if (!task) {
     logger.info("No task claimed (queue empty or all blocked)");
     const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);
@@ -62,67 +65,80 @@ export async function tick(
   callbacks?.onTaskStart(task);
 
   // Create a thread for this tick
-  const threadId = await createThread(
-    conn,
-    "daemon_tick",
-    task.id,
-    `Working: ${task.name}`,
+  const threadId = await withDb(dbPath, (conn) =>
+    createThread(conn, "daemon_tick", task.id, `Working: ${task.name}`),
   );
 
   // Build system prompt (includes task-relevant context from embeddings)
-  const systemPrompt = await buildSystemPrompt(projectDir, task, conn, config, {
-    hasMcpTools: mcpxClient != null,
-  });
+  const systemPrompt = await buildSystemPrompt(
+    projectDir,
+    task,
+    dbPath,
+    config,
+    {
+      hasMcpTools: mcpxClient != null,
+    },
+  );
 
   try {
     const result = await runAgentLoop({
       systemPrompt,
       task,
       config,
-      conn,
+      dbPath,
       threadId,
       projectDir,
       mcpxClient,
       callbacks,
     });
 
+    // Update task status and store output. Only completed tasks have an
+    // `output`; waiting/failed tasks put their reason in `waiting_reason`.
     const isComplete = result.status === "complete";
-    await updateTaskStatus(
-      conn,
-      task.id,
-      result.status,
-      isComplete ? null : result.reason,
-      isComplete ? result.reason : null,
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(
+        conn,
+        task.id,
+        result.status,
+        isComplete ? null : result.reason,
+        isComplete ? result.reason : null,
+      ),
     );
 
     // Log the status change
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
+      }),
+    );
 
     logger.info(`Task ${task.id} -> ${result.status}`);
 
-    // Generate a descriptive title for the thread
+    // Generate a descriptive title for the thread (fire-and-forget)
     void generateThreadTitle(
       config,
-      conn,
+      dbPath,
       threadId,
       `Task: ${task.name}\nDescription: ${task.description}\nOutcome: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`,
     );
   } catch (err) {
-    await updateTaskStatus(conn, task.id, "failed", String(err), null);
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(conn, task.id, "failed", String(err), null),
+    );
 
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} failed: ${err}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} failed: ${err}`,
+      }),
+    );
 
     logger.error(`Task ${task.id} failed: ${err}`);
   } finally {
-    await endThread(conn, threadId);
+    await withDb(dbPath, (conn) => endThread(conn, threadId));
   }
 
   const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);

package/src/db/connection.ts

@@ -11,12 +11,20 @@ export class DbConnection {
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
   private conn: any;
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  private instance: any;
+  private readonly ownedInstance: any;
+  private readonly dbPath: string;
+  private closed = false;
 
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  constructor(conn: any, instance: any) {
+  constructor(
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    conn: any,
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    ownedInstance: any,
+    dbPath: string,
+  ) {
     this.conn = conn;
-    this.instance = instance;
+    this.ownedInstance = ownedInstance;
+    this.dbPath = dbPath;
   }
 
   /** Execute raw SQL with no return value. */
@@ -62,10 +70,22 @@ export class DbConnection {
     return { changes: result.rowsChanged };
   }
 
-  /** Close the connection and dispose of the instance. */
+  /**
+   * Disconnect and release this connection's share of the DuckDB instance.
+   * For file-backed DBs, the instance is closed (and the OS file lock
+   * released) once every overlapping connection in this process has closed.
+   * For `:memory:` DBs, the instance is owned by this connection and closed
+   * immediately.
+   */
   close(): void {
+    if (this.closed) return;
+    this.closed = true;
     this.conn.disconnectSync();
-    this.instance.closeSync();
+    if (this.ownedInstance) {
+      this.ownedInstance.closeSync();
+    } else {
+      releaseInstance(this.dbPath);
+    }
   }
 }
 
@@ -98,31 +118,140 @@ function flattenParams(params: SqlParam[]): SqlParam[] {
   return params.map((p) => (Array.isArray(p) ? JSON.stringify(p) : p));
 }
 
+/**
+ * Refcounted, process-local cache of open DuckDB instances keyed by dbPath.
+ *
+ * DuckDB's file lock is held at the instance level, so we must close the
+ * instance — not just the connection — to let another process acquire the
+ * writer lock. At the same time, opening two instances for the same file
+ * from one process is unsafe. This cache resolves both: overlapping
+ * `getConnection` calls in the same process share a single instance; once
+ * every connection has closed, the instance is closed and evicted, which
+ * releases the OS file lock.
+ *
+ * `:memory:` paths bypass the cache so each test/caller gets its own
+ * isolated in-memory database.
+ */
+interface CachedInstance {
+  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+  instance: any;
+  refCount: number;
+}
+const instanceCache = new Map<string, CachedInstance>();
+const pendingInstance = new Map<string, Promise<CachedInstance>>();
+
+function isMemoryPath(path: string): boolean {
+  return path === ":memory:" || path.startsWith(":memory:");
+}
+
+async function acquireSharedInstance(dbPath: string): Promise<CachedInstance> {
+  const existing = instanceCache.get(dbPath);
+  if (existing) {
+    existing.refCount += 1;
+    return existing;
+  }
+  const inFlight = pendingInstance.get(dbPath);
+  if (inFlight) {
+    const cached = await inFlight;
+    cached.refCount += 1;
+    return cached;
+  }
+  const creation = (async () => {
+    const instance = await DuckDBInstance.create(dbPath);
+    const cached: CachedInstance = { instance, refCount: 1 };
+    instanceCache.set(dbPath, cached);
+    return cached;
+  })();
+  pendingInstance.set(dbPath, creation);
+  try {
+    return await creation;
+  } finally {
+    pendingInstance.delete(dbPath);
+  }
+}
+
+function releaseInstance(dbPath: string): void {
+  const cached = instanceCache.get(dbPath);
+  if (!cached) return;
+  cached.refCount -= 1;
+  if (cached.refCount <= 0) {
+    instanceCache.delete(dbPath);
+    cached.instance.closeSync();
+  }
+}
+
 export async function getConnection(dbPath?: string): Promise<DbConnection> {
-  const instance = await DuckDBInstance.create(dbPath ?? ":memory:");
-  const conn = await instance.connect();
+  const path = dbPath ?? ":memory:";
 
-  // Load VSS extension for vector similarity search
-  await conn.run("INSTALL vss; LOAD vss;");
-  // Enable HNSW index persistence for file-backed databases
-  await conn.run("SET hnsw_enable_experimental_persistence = true;");
+  if (isMemoryPath(path)) {
+    const instance = await DuckDBInstance.create(path);
+    const conn = await instance.connect();
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, instance, path);
+  }
 
-  return new DbConnection(conn, instance);
+  const cached = await acquireSharedInstance(path);
+  try {
+    const conn = await cached.instance.connect();
+    // INSTALL is a no-op after the first successful install (the extension
+    // is persisted to the user's DuckDB extension directory). LOAD is
+    // cheap per connection.
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, null, path);
+  } catch (err) {
+    releaseInstance(path);
+    throw err;
+  }
 }
 
+/**
+ * Open a DuckDB connection for a single logical unit of work and guarantee
+ * it is closed afterward. Retries on lock conflicts so two processes that
+ * race on the file lock cooperate instead of failing hard.
+ *
+ * Prefer one `withDb` per logical operation. The file lock is only released
+ * when every connection (across this process's overlapping callers) has
+ * been closed, so holding the connection across non-DB work (LLM calls,
+ * network I/O, filesystem walks) keeps other processes blocked.
+ */
+export async function withDb<T>(
+  dbPath: string,
+  fn: (conn: DbConnection) => Promise<T>,
+): Promise<T> {
+  const conn = await withRetry(() => getConnection(dbPath));
+  try {
+    return await fn(conn);
+  } finally {
+    conn.close();
+  }
+}
+
+/**
+ * Retry `fn` with exponential backoff when it fails with a DuckDB file-lock
+ * conflict ("Conflicting lock is held…"). Other errors propagate immediately.
+ */
 export async function withRetry<T>(
   fn: () => Promise<T>,
-  maxRetries = 5,
+  maxRetries = 8,
 ): Promise<T> {
   let lastError: unknown;
   for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
       return await fn();
     } catch (err) {
+      if (!isLockConflict(err)) throw err;
       lastError = err;
       if (attempt === maxRetries - 1) throw err;
+      // 100, 200, 400, 800, 1600, 3200, 6400, 12800 — up to ~25s total
       await Bun.sleep(100 * 2 ** attempt);
     }
   }
   throw lastError;
 }
+
+function isLockConflict(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return msg.includes("Conflicting lock") || msg.includes("could not be set");
+}

package/src/db/context.ts

@@ -179,6 +179,19 @@ export async function getContextItemByPath(
   return row ? rowToContextItem(row) : null;
 }
 
+export async function getContextItemBySourcePath(
+  db: DbConnection,
+  sourcePath: string,
+  sourceType: "file" | "url",
+): Promise<ContextItem | null> {
+  const row = await db.queryGet<ContextItemRow>(
+    "SELECT * FROM context_items WHERE source_path = ?1 AND source_type = ?2 LIMIT 1",
+    sourcePath,
+    sourceType,
+  );
+  return row ? rowToContextItem(row) : null;
+}
+
 /**
  * Look up a context item by UUID (if the value looks like one) or by context_path.
  */

package/src/db/schedules.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildSetClauses, buildWhereClause } from "./query.ts";
+import { buildSetClauses, buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface Schedule {
@@ -72,7 +72,7 @@ export async function getSchedule(
 
 export async function listSchedules(
   db: DbConnection,
-  filters?: { enabled?: boolean },
+  filters?: { enabled?: boolean; limit?: number; offset?: number },
 ): Promise<Schedule[]> {
   const { where, params } = buildWhereClause([
     [
@@ -80,9 +80,13 @@ export async function listSchedules(
       filters?.enabled !== undefined ? (filters.enabled ? 1 : 0) : undefined,
     ],
   ]);
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<ScheduleRow>(
-    `SELECT * FROM schedules ${where} ORDER BY created_at ASC`,
+    `SELECT * FROM schedules ${where}
+     ORDER BY created_at ASC, id ASC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToSchedule);

package/src/db/tasks.ts

@@ -109,6 +109,7 @@ export async function listTasks(
     status?: Task["status"];
     priority?: Task["priority"];
     limit?: number;
+    offset?: number;
   },
 ): Promise<Task[]> {
   const { where, params } = buildWhereClause([
@@ -116,13 +117,12 @@ export async function listTasks(
     ["priority", filters?.priority],
   ]);
   const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<TaskRow>(
     `SELECT * FROM tasks ${where}
-     ORDER BY
-       CASE priority WHEN 'high' THEN 0 WHEN 'medium' THEN 1 WHEN 'low' THEN 2 END,
-       created_at ASC
-     ${limit}`,
+     ORDER BY created_at DESC, id DESC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToTask);

package/src/db/threads.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildWhereClause } from "./query.ts";
+import { buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface Thread {
@@ -256,18 +256,20 @@ export async function listThreads(
     type?: Thread["type"];
     taskId?: string;
     limit?: number;
+    offset?: number;
   },
 ): Promise<Thread[]> {
   const { where, params } = buildWhereClause([
     ["type", filters?.type],
     ["task_id", filters?.taskId],
   ]);
-  const limit = filters?.limit ? `LIMIT ${filters.limit}` : "";
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<ThreadRow>(
     `SELECT * FROM threads ${where}
-     ORDER BY started_at DESC
-     ${limit}`,
+     ORDER BY started_at DESC, id DESC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToThread);

package/src/tools/tool.ts

@@ -5,7 +5,15 @@ import type { BotholomewConfig } from "../config/schemas.ts";
 import type { DbConnection } from "../db/connection.ts";
 
 export interface ToolContext {
+  /**
+   * Short-lived DB connection scoped to this tool call. Safe for single-query
+   * tools. Do NOT hold it across slow work (network, embedding, long loops) —
+   * the instance-level file lock stays held until every connection closes.
+   * For long-running tools, use `dbPath` with `withDb` per logical operation.
+   */
   conn: DbConnection;
+  /** Path to the DuckDB file. Use with `withDb` for long-running tools. */
+  dbPath: string;
   projectDir: string;
   config: Required<BotholomewConfig>;
   mcpxClient: McpxClient | null;

package/src/tui/App.tsx

@@ -7,6 +7,7 @@ import {
   startChatSession,
 } from "../chat/session.ts";
 import { MAX_INLINE_CHARS, PAGE_SIZE_CHARS } from "../daemon/large-results.ts";
+import { withDb } from "../db/connection.ts";
 import type { Interaction } from "../db/threads.ts";
 import { getThread } from "../db/threads.ts";
 import {
@@ -163,7 +164,9 @@ export function App({
         sessionRef.current = session;
 
         if (session.messages.length > 0) {
-          const threadData = await getThread(session.conn, session.threadId);
+          const threadData = await withDb(session.dbPath, (conn) =>
+            getThread(conn, session.threadId),
+          );
           if (threadData) {
             setMessages(
               restoreMessagesFromInteractions(threadData.interactions),
@@ -409,7 +412,9 @@ export function App({
     const refreshTitle = async () => {
       const session = sessionRef.current;
       if (!session) return;
-      const result = await getThread(session.conn, session.threadId);
+      const result = await withDb(session.dbPath, (conn) =>
+        getThread(conn, session.threadId),
+      );
       if (mounted && result?.thread.title) {
         setChatTitle(result.thread.title);
       }
@@ -547,18 +552,18 @@ export function App({
     [exit, processQueue, syncQueue],
   );
 
-  const sessionConn = sessionRef.current?.conn;
+  const sessionDbPath = sessionRef.current?.dbPath;
   const inputBarHeader = useMemo(
     () =>
-      sessionConn ? (
+      sessionDbPath ? (
         <StatusBar
           projectDir={projectDir}
-          conn={sessionConn}
+          dbPath={sessionDbPath}
           chatTitle={chatTitle}
           onDaemonStatusChange={setDaemonRunning}
         />
       ) : null,
-    [projectDir, sessionConn, chatTitle],
+    [projectDir, sessionDbPath, chatTitle],
   );
 
   const sessionSkills = ready ? sessionRef.current?.skills : undefined;
@@ -602,7 +607,7 @@ export function App({
     );
   }
 
-  const conn = sessionRef.current.conn;
+  const dbPath = sessionRef.current.dbPath;
   const threadId = sessionRef.current.threadId;
 
   return (
@@ -642,14 +647,14 @@ export function App({
         flexDirection="column"
         flexGrow={1}
       >
-        <ContextPanel conn={conn} isActive={activeTab === 3} />
+        <ContextPanel dbPath={dbPath} isActive={activeTab === 3} />
       </Box>
       <Box
         display={activeTab === 4 ? "flex" : "none"}
         flexDirection="column"
         flexGrow={1}
       >
-        <TaskPanel conn={conn} isActive={activeTab === 4} />
+        <TaskPanel dbPath={dbPath} isActive={activeTab === 4} />
       </Box>
       <Box
         display={activeTab === 5 ? "flex" : "none"}
@@ -657,7 +662,7 @@ export function App({
         flexGrow={1}
       >
         <ThreadPanel
-          conn={conn}
+          dbPath={dbPath}
           activeThreadId={threadId}
           isActive={activeTab === 5}
         />
@@ -667,7 +672,7 @@ export function App({
         flexDirection="column"
         flexGrow={1}
       >
-        <SchedulePanel conn={conn} isActive={activeTab === 6} />
+        <SchedulePanel dbPath={dbPath} isActive={activeTab === 6} />
       </Box>
       <Box
         display={activeTab === 7 ? "flex" : "none"}

package/src/tui/components/ContextPanel.tsx

@@ -1,6 +1,6 @@
 import { Box, Text, useInput, useStdout } from "ink";
 import { memo, useCallback, useEffect, useMemo, useState } from "react";
-import type { DbConnection } from "../../db/connection.ts";
+import { withDb } from "../../db/connection.ts";
 import {
   type ContextItem,
   deleteContextItem,
@@ -11,7 +11,7 @@ import {
 } from "../../db/context.ts";
 
 interface ContextPanelProps {
-  conn: DbConnection;
+  dbPath: string;
   isActive: boolean;
 }
 
@@ -32,7 +32,7 @@ type Entry = DirEntry | FileEntry;
 const CHROME_LINES = 8;
 
 export const ContextPanel = memo(function ContextPanel({
-  conn,
+  dbPath,
   isActive,
 }: ContextPanelProps) {
   const { stdout } = useStdout();
@@ -64,10 +64,10 @@ export const ContextPanel = memo(function ContextPanel({
 
   const loadEntries = useCallback(
     async (path: string) => {
-      const dirs = await getDistinctDirectories(conn, path);
-      const files = await listContextItemsByPrefix(conn, path, {
-        recursive: false,
-      });
+      const [dirs, files] = await withDb(dbPath, async (conn) => [
+        await getDistinctDirectories(conn, path),
+        await listContextItemsByPrefix(conn, path, { recursive: false }),
+      ]);
 
       const dirEntries: DirEntry[] = dirs.map((d) => ({
         type: "directory",
@@ -84,7 +84,7 @@ export const ContextPanel = memo(function ContextPanel({
       setScrollOffset(0);
       setPreview(null);
     },
-    [conn],
+    [dbPath],
   );
 
   useEffect(() => {
@@ -99,13 +99,15 @@ export const ContextPanel = memo(function ContextPanel({
         setSearchResults(null);
         return;
       }
-      const results = await searchContextByKeyword(conn, query.trim(), 50);
+      const results = await withDb(dbPath, (conn) =>
+        searchContextByKeyword(conn, query.trim(), 50),
+      );
       setSearchResults(results);
       setCursor(0);
       setScrollOffset(0);
       setPreview(null);
     },
-    [conn],
+    [dbPath],
   );
 
   // Compute the items list and visible window for the current view
@@ -171,11 +173,13 @@ export const ContextPanel = memo(function ContextPanel({
         if (input === "y" || input === "d") {
           const entry = entries[cursor];
           if (entry) {
-            if (entry.type === "directory") {
-              deleteContextItemsByPrefix(conn, entry.path);
-            } else {
-              deleteContextItem(conn, entry.item.id);
-            }
+            void withDb(dbPath, async (conn) => {
+              if (entry.type === "directory") {
+                await deleteContextItemsByPrefix(conn, entry.path);
+              } else {
+                await deleteContextItem(conn, entry.item.id);
+              }
+            });
             setConfirmDelete(false);
             loadEntries(currentPath);
           }