botholomew 0.7.7 → 0.7.9

package/src/daemon/tick.ts

@@ -1,6 +1,6 @@
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { listSchedules } from "../db/schedules.ts";
 import {
   claimNextTask,
@@ -17,7 +17,7 @@ import { processSchedules } from "./schedules.ts";
 
 export async function tick(
   projectDir: string,
-  conn: DbConnection,
+  dbPath: string,
   config: Required<BotholomewConfig>,
   mcpxClient?: McpxClient | null,
   callbacks?: DaemonStreamCallbacks,
@@ -27,9 +27,8 @@ export async function tick(
   logger.phase("tick-start", `#${tickNum}`);
 
   // Reset stale tasks stuck in in_progress
-  const resetIds = await resetStaleTasks(
-    conn,
-    config.max_tick_duration_seconds * 3,
+  const resetIds = await withDb(dbPath, (conn) =>
+    resetStaleTasks(conn, config.max_tick_duration_seconds * 3),
   );
   if (resetIds.length > 0) {
     logger.warn(
@@ -37,12 +36,16 @@ export async function tick(
     );
   }
 
-  // Process schedules (may create new tasks)
-  const enabledSchedules = await listSchedules(conn, { enabled: true });
+  // Process schedules (may create new tasks). Check enabled count so we only
+  // log the phase when there's work to evaluate — the call itself is a no-op
+  // otherwise.
+  const enabledSchedules = await withDb(dbPath, (conn) =>
+    listSchedules(conn, { enabled: true }),
+  );
   if (enabledSchedules.length > 0) {
     logger.phase("evaluating-schedules", `${enabledSchedules.length} enabled`);
     try {
-      await processSchedules(conn, config);
+      await processSchedules(dbPath, config);
     } catch (err) {
       logger.error(`Schedule processing failed: ${err}`);
     }
@@ -50,7 +53,7 @@ export async function tick(
 
   // Claim a task
   logger.phase("claiming-task");
-  const task = await claimNextTask(conn);
+  const task = await withDb(dbPath, (conn) => claimNextTask(conn));
   if (!task) {
     logger.info("No task claimed (queue empty or all blocked)");
     const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);
@@ -62,67 +65,80 @@ export async function tick(
   callbacks?.onTaskStart(task);
 
   // Create a thread for this tick
-  const threadId = await createThread(
-    conn,
-    "daemon_tick",
-    task.id,
-    `Working: ${task.name}`,
+  const threadId = await withDb(dbPath, (conn) =>
+    createThread(conn, "daemon_tick", task.id, `Working: ${task.name}`),
   );
 
   // Build system prompt (includes task-relevant context from embeddings)
-  const systemPrompt = await buildSystemPrompt(projectDir, task, conn, config, {
-    hasMcpTools: mcpxClient != null,
-  });
+  const systemPrompt = await buildSystemPrompt(
+    projectDir,
+    task,
+    dbPath,
+    config,
+    {
+      hasMcpTools: mcpxClient != null,
+    },
+  );
 
   try {
     const result = await runAgentLoop({
       systemPrompt,
       task,
       config,
-      conn,
+      dbPath,
       threadId,
       projectDir,
       mcpxClient,
       callbacks,
     });
 
-    // Update task status and store output
-    await updateTaskStatus(
-      conn,
-      task.id,
-      result.status,
-      result.reason,
-      result.reason,
+    // Update task status and store output. Only completed tasks have an
+    // `output`; waiting/failed tasks put their reason in `waiting_reason`.
+    const isComplete = result.status === "complete";
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(
+        conn,
+        task.id,
+        result.status,
+        isComplete ? null : result.reason,
+        isComplete ? result.reason : null,
+      ),
     );
 
     // Log the status change
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
+      }),
+    );
 
     logger.info(`Task ${task.id} -> ${result.status}`);
 
-    // Generate a descriptive title for the thread
+    // Generate a descriptive title for the thread (fire-and-forget)
     void generateThreadTitle(
       config,
-      conn,
+      dbPath,
       threadId,
       `Task: ${task.name}\nDescription: ${task.description}\nOutcome: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`,
     );
   } catch (err) {
-    await updateTaskStatus(conn, task.id, "failed", String(err), String(err));
+    await withDb(dbPath, (conn) =>
+      updateTaskStatus(conn, task.id, "failed", String(err), null),
+    );
 
-    await logInteraction(conn, threadId, {
-      role: "system",
-      kind: "status_change",
-      content: `Task ${task.id} failed: ${err}`,
-    });
+    await withDb(dbPath, (conn) =>
+      logInteraction(conn, threadId, {
+        role: "system",
+        kind: "status_change",
+        content: `Task ${task.id} failed: ${err}`,
+      }),
+    );
 
     logger.error(`Task ${task.id} failed: ${err}`);
   } finally {
-    await endThread(conn, threadId);
+    await withDb(dbPath, (conn) => endThread(conn, threadId));
   }
 
   const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);

package/src/db/connection.ts

@@ -11,12 +11,20 @@ export class DbConnection {
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
   private conn: any;
   // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  private instance: any;
+  private readonly ownedInstance: any;
+  private readonly dbPath: string;
+  private closed = false;
 
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  constructor(conn: any, instance: any) {
+  constructor(
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    conn: any,
+    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+    ownedInstance: any,
+    dbPath: string,
+  ) {
     this.conn = conn;
-    this.instance = instance;
+    this.ownedInstance = ownedInstance;
+    this.dbPath = dbPath;
   }
 
   /** Execute raw SQL with no return value. */
@@ -62,10 +70,22 @@ export class DbConnection {
     return { changes: result.rowsChanged };
   }
 
-  /** Close the connection and dispose of the instance. */
+  /**
+   * Disconnect and release this connection's share of the DuckDB instance.
+   * For file-backed DBs, the instance is closed (and the OS file lock
+   * released) once every overlapping connection in this process has closed.
+   * For `:memory:` DBs, the instance is owned by this connection and closed
+   * immediately.
+   */
   close(): void {
+    if (this.closed) return;
+    this.closed = true;
     this.conn.disconnectSync();
-    this.instance.closeSync();
+    if (this.ownedInstance) {
+      this.ownedInstance.closeSync();
+    } else {
+      releaseInstance(this.dbPath);
+    }
   }
 }
 
@@ -98,31 +118,140 @@ function flattenParams(params: SqlParam[]): SqlParam[] {
   return params.map((p) => (Array.isArray(p) ? JSON.stringify(p) : p));
 }
 
+/**
+ * Refcounted, process-local cache of open DuckDB instances keyed by dbPath.
+ *
+ * DuckDB's file lock is held at the instance level, so we must close the
+ * instance — not just the connection — to let another process acquire the
+ * writer lock. At the same time, opening two instances for the same file
+ * from one process is unsafe. This cache resolves both: overlapping
+ * `getConnection` calls in the same process share a single instance; once
+ * every connection has closed, the instance is closed and evicted, which
+ * releases the OS file lock.
+ *
+ * `:memory:` paths bypass the cache so each test/caller gets its own
+ * isolated in-memory database.
+ */
+interface CachedInstance {
+  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
+  instance: any;
+  refCount: number;
+}
+const instanceCache = new Map<string, CachedInstance>();
+const pendingInstance = new Map<string, Promise<CachedInstance>>();
+
+function isMemoryPath(path: string): boolean {
+  return path === ":memory:" || path.startsWith(":memory:");
+}
+
+async function acquireSharedInstance(dbPath: string): Promise<CachedInstance> {
+  const existing = instanceCache.get(dbPath);
+  if (existing) {
+    existing.refCount += 1;
+    return existing;
+  }
+  const inFlight = pendingInstance.get(dbPath);
+  if (inFlight) {
+    const cached = await inFlight;
+    cached.refCount += 1;
+    return cached;
+  }
+  const creation = (async () => {
+    const instance = await DuckDBInstance.create(dbPath);
+    const cached: CachedInstance = { instance, refCount: 1 };
+    instanceCache.set(dbPath, cached);
+    return cached;
+  })();
+  pendingInstance.set(dbPath, creation);
+  try {
+    return await creation;
+  } finally {
+    pendingInstance.delete(dbPath);
+  }
+}
+
+function releaseInstance(dbPath: string): void {
+  const cached = instanceCache.get(dbPath);
+  if (!cached) return;
+  cached.refCount -= 1;
+  if (cached.refCount <= 0) {
+    instanceCache.delete(dbPath);
+    cached.instance.closeSync();
+  }
+}
+
 export async function getConnection(dbPath?: string): Promise<DbConnection> {
-  const instance = await DuckDBInstance.create(dbPath ?? ":memory:");
-  const conn = await instance.connect();
+  const path = dbPath ?? ":memory:";
 
-  // Load VSS extension for vector similarity search
-  await conn.run("INSTALL vss; LOAD vss;");
-  // Enable HNSW index persistence for file-backed databases
-  await conn.run("SET hnsw_enable_experimental_persistence = true;");
+  if (isMemoryPath(path)) {
+    const instance = await DuckDBInstance.create(path);
+    const conn = await instance.connect();
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, instance, path);
+  }
 
-  return new DbConnection(conn, instance);
+  const cached = await acquireSharedInstance(path);
+  try {
+    const conn = await cached.instance.connect();
+    // INSTALL is a no-op after the first successful install (the extension
+    // is persisted to the user's DuckDB extension directory). LOAD is
+    // cheap per connection.
+    await conn.run("INSTALL vss; LOAD vss;");
+    await conn.run("SET hnsw_enable_experimental_persistence = true;");
+    return new DbConnection(conn, null, path);
+  } catch (err) {
+    releaseInstance(path);
+    throw err;
+  }
 }
 
+/**
+ * Open a DuckDB connection for a single logical unit of work and guarantee
+ * it is closed afterward. Retries on lock conflicts so two processes that
+ * race on the file lock cooperate instead of failing hard.
+ *
+ * Prefer one `withDb` per logical operation. The file lock is only released
+ * when every connection (across this process's overlapping callers) has
+ * been closed, so holding the connection across non-DB work (LLM calls,
+ * network I/O, filesystem walks) keeps other processes blocked.
+ */
+export async function withDb<T>(
+  dbPath: string,
+  fn: (conn: DbConnection) => Promise<T>,
+): Promise<T> {
+  const conn = await withRetry(() => getConnection(dbPath));
+  try {
+    return await fn(conn);
+  } finally {
+    conn.close();
+  }
+}
+
+/**
+ * Retry `fn` with exponential backoff when it fails with a DuckDB file-lock
+ * conflict ("Conflicting lock is held…"). Other errors propagate immediately.
+ */
 export async function withRetry<T>(
   fn: () => Promise<T>,
-  maxRetries = 5,
+  maxRetries = 8,
 ): Promise<T> {
   let lastError: unknown;
   for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
       return await fn();
     } catch (err) {
+      if (!isLockConflict(err)) throw err;
       lastError = err;
       if (attempt === maxRetries - 1) throw err;
+      // 100, 200, 400, 800, 1600, 3200, 6400, 12800 — up to ~25s total
       await Bun.sleep(100 * 2 ** attempt);
     }
   }
   throw lastError;
 }
+
+function isLockConflict(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return msg.includes("Conflicting lock") || msg.includes("could not be set");
+}

package/src/db/context.ts

@@ -56,6 +56,17 @@ function rowToContextItem(row: ContextItemRow): ContextItem {
   };
 }
 
+export class PathConflictError extends Error {
+  existingId: string;
+  contextPath: string;
+  constructor(existingId: string, contextPath: string) {
+    super(`context_path already exists: ${contextPath}`);
+    this.name = "PathConflictError";
+    this.existingId = existingId;
+    this.contextPath = contextPath;
+  }
+}
+
 // --- Basic CRUD ---
 
 export async function createContextItem(
@@ -124,6 +135,28 @@ export async function upsertContextItem(
   return createContextItem(db, params);
 }
 
+/**
+ * Strict creator: throws PathConflictError if context_path already exists.
+ * Use when callers want to surface collisions instead of silently overwriting.
+ */
+export async function createContextItemStrict(
+  db: DbConnection,
+  params: {
+    title: string;
+    content?: string;
+    mimeType?: string;
+    sourceType?: "file" | "url";
+    sourcePath?: string;
+    contextPath: string;
+    description?: string;
+    isTextual?: boolean;
+  },
+): Promise<ContextItem> {
+  const existing = await getContextItemByPath(db, params.contextPath);
+  if (existing) throw new PathConflictError(existing.id, params.contextPath);
+  return createContextItem(db, params);
+}
+
 export async function getContextItem(
   db: DbConnection,
   id: string,

package/src/db/schedules.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildSetClauses, buildWhereClause } from "./query.ts";
+import { buildSetClauses, buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface Schedule {
@@ -72,7 +72,7 @@ export async function getSchedule(
 
 export async function listSchedules(
   db: DbConnection,
-  filters?: { enabled?: boolean },
+  filters?: { enabled?: boolean; limit?: number; offset?: number },
 ): Promise<Schedule[]> {
   const { where, params } = buildWhereClause([
     [
@@ -80,9 +80,13 @@ export async function listSchedules(
       filters?.enabled !== undefined ? (filters.enabled ? 1 : 0) : undefined,
     ],
   ]);
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<ScheduleRow>(
-    `SELECT * FROM schedules ${where} ORDER BY created_at ASC`,
+    `SELECT * FROM schedules ${where}
+     ORDER BY created_at ASC, id ASC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToSchedule);

package/src/db/tasks.ts

@@ -109,6 +109,7 @@ export async function listTasks(
     status?: Task["status"];
     priority?: Task["priority"];
     limit?: number;
+    offset?: number;
   },
 ): Promise<Task[]> {
   const { where, params } = buildWhereClause([
@@ -116,13 +117,12 @@ export async function listTasks(
     ["priority", filters?.priority],
   ]);
   const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<TaskRow>(
     `SELECT * FROM tasks ${where}
-     ORDER BY
-       CASE priority WHEN 'high' THEN 0 WHEN 'medium' THEN 1 WHEN 'low' THEN 2 END,
-       created_at ASC
-     ${limit}`,
+     ORDER BY created_at DESC, id DESC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToTask);
@@ -132,8 +132,8 @@ export async function updateTaskStatus(
   db: DbConnection,
   id: string,
   status: Task["status"],
-  reason?: string,
-  output?: string,
+  reason?: string | null,
+  output?: string | null,
 ): Promise<void> {
   await db.queryRun(
     `UPDATE tasks

package/src/db/threads.ts

@@ -1,5 +1,5 @@
 import type { DbConnection } from "./connection.ts";
-import { buildWhereClause } from "./query.ts";
+import { buildWhereClause, sanitizeInt } from "./query.ts";
 import { uuidv7 } from "./uuid.ts";
 
 export interface Thread {
@@ -256,18 +256,20 @@ export async function listThreads(
     type?: Thread["type"];
     taskId?: string;
     limit?: number;
+    offset?: number;
   },
 ): Promise<Thread[]> {
   const { where, params } = buildWhereClause([
     ["type", filters?.type],
     ["task_id", filters?.taskId],
   ]);
-  const limit = filters?.limit ? `LIMIT ${filters.limit}` : "";
+  const limit = filters?.limit ? `LIMIT ${sanitizeInt(filters.limit)}` : "";
+  const offset = filters?.offset ? `OFFSET ${sanitizeInt(filters.offset)}` : "";
 
   const rows = await db.queryAll<ThreadRow>(
     `SELECT * FROM threads ${where}
-     ORDER BY started_at DESC
-     ${limit}`,
+     ORDER BY started_at DESC, id DESC
+     ${limit} ${offset}`,
     ...params,
   );
   return rows.map(rowToThread);

package/src/tools/file/write.ts

@@ -1,7 +1,11 @@
 import { isText } from "istextorbinary";
 import { z } from "zod";
 import { ingestByPath } from "../../context/ingest.ts";
-import { upsertContextItem } from "../../db/context.ts";
+import {
+  createContextItemStrict,
+  PathConflictError,
+  upsertContextItem,
+} from "../../db/context.ts";
 import type { ToolDefinition } from "../tool.ts";
 
 function mimeFromPath(path: string): string {
@@ -30,18 +34,27 @@ const inputSchema = z.object({
     .optional()
     .describe("Title for the file (defaults to filename)"),
   description: z.string().optional().describe("Description of the file"),
+  on_conflict: z
+    .enum(["error", "overwrite"])
+    .optional()
+    .describe(
+      "What to do if a file already exists at this path. Defaults to 'error'. Pass 'overwrite' to replace.",
+    ),
 });
 
 const outputSchema = z.object({
-  id: z.string(),
+  id: z.string().nullable(),
   path: z.string(),
   is_error: z.boolean(),
+  error_type: z.string().optional(),
+  message: z.string().optional(),
+  next_action_hint: z.string().optional(),
 });
 
 export const contextWriteTool = {
   name: "context_write",
   description:
-    "Write content to a context item. Creates the item if it doesn't exist, or overwrites if it does.",
+    "Write content to a context item. By default, fails if the path already exists — pass on_conflict='overwrite' to replace.",
   group: "context",
   inputSchema,
   outputSchema,
@@ -50,17 +63,43 @@ export const contextWriteTool = {
     const isTextual = isTextualPath(input.path);
     const title =
       input.title ?? input.path.split("/").filter(Boolean).pop() ?? input.path;
+    const onConflict = input.on_conflict ?? "error";
 
-    const item = await upsertContextItem(ctx.conn, {
-      title,
-      description: input.description,
-      content: input.content_base64 ?? input.content,
-      contextPath: input.path,
-      mimeType,
-      isTextual,
-    });
+    try {
+      const item =
+        onConflict === "overwrite"
+          ? await upsertContextItem(ctx.conn, {
+              title,
+              description: input.description,
+              content: input.content_base64 ?? input.content,
+              contextPath: input.path,
+              mimeType,
+              isTextual,
+            })
+          : await createContextItemStrict(ctx.conn, {
+              title,
+              description: input.description,
+              content: input.content_base64 ?? input.content,
+              contextPath: input.path,
+              mimeType,
+              isTextual,
+            });
 
-    await ingestByPath(ctx.conn, input.path, ctx.config);
-    return { id: item.id, path: item.context_path, is_error: false };
+      await ingestByPath(ctx.conn, input.path, ctx.config);
+      return { id: item.id, path: item.context_path, is_error: false };
+    } catch (err) {
+      if (err instanceof PathConflictError) {
+        return {
+          id: null,
+          path: input.path,
+          is_error: true,
+          error_type: "path_conflict",
+          message: `A file already exists at ${input.path} (id: ${err.existingId}).`,
+          next_action_hint:
+            "Call context_read to inspect the existing file, or retry with on_conflict='overwrite' to replace it.",
+        };
+      }
+      throw err;
+    }
   },
 } satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;

package/src/tools/tool.ts

@@ -5,7 +5,15 @@ import type { BotholomewConfig } from "../config/schemas.ts";
 import type { DbConnection } from "../db/connection.ts";
 
 export interface ToolContext {
+  /**
+   * Short-lived DB connection scoped to this tool call. Safe for single-query
+   * tools. Do NOT hold it across slow work (network, embedding, long loops) —
+   * the instance-level file lock stays held until every connection closes.
+   * For long-running tools, use `dbPath` with `withDb` per logical operation.
+   */
   conn: DbConnection;
+  /** Path to the DuckDB file. Use with `withDb` for long-running tools. */
+  dbPath: string;
   projectDir: string;
   config: Required<BotholomewConfig>;
   mcpxClient: McpxClient | null;