botholomew 0.7.7 → 0.7.9

package/src/commands/task.ts

@@ -17,16 +17,18 @@ export function registerTaskCommand(program: Command) {
 
   task
     .command("list")
-    .description("List all tasks")
+    .description("List all tasks (newest first)")
     .option("-s, --status <status>", "filter by status")
     .option("-p, --priority <priority>", "filter by priority")
-    .option("-l, --limit <n>", "max number of tasks", parseInt)
+    .option("-l, --limit <n>", "max number of tasks", Number.parseInt)
+    .option("-o, --offset <n>", "skip first N tasks", Number.parseInt)
     .action((opts) =>
       withDb(program, async (conn) => {
         const tasks = await listTasks(conn, {
           status: opts.status,
           priority: opts.priority,
           limit: opts.limit,
+          offset: opts.offset,
         });
 
         if (tasks.length === 0) {
@@ -34,9 +36,15 @@ export function registerTaskCommand(program: Command) {
           return;
         }
 
+        const header = `${ansis.bold("ID".padEnd(36))}  ${ansis.bold("Status".padEnd(11))}  ${ansis.bold("Priority".padEnd(6))}  ${ansis.bold("Created".padEnd(19))}  ${ansis.bold("Updated".padEnd(19))}  ${ansis.bold("Name")}`;
+        console.log(header);
+        console.log("-".repeat(120));
+
         for (const t of tasks) {
           printTask(t);
         }
+
+        console.log(`\n${ansis.dim(`${tasks.length} task(s)`)}`);
       }),
     );
 
@@ -154,10 +162,26 @@ function priorityColor(priority: Task["priority"]): string {
   }
 }
 
+function formatTime(date: Date): string {
+  return date
+    .toISOString()
+    .replace("T", " ")
+    .replace(/\.\d{3}Z$/, "");
+}
+
+function padColored(colored: string, raw: string, width: number): string {
+  const padding = Math.max(0, width - raw.length);
+  return colored + " ".repeat(padding);
+}
+
 function printTask(t: Task) {
-  const id = ansis.dim(t.id);
+  const id = ansis.dim(t.id.padEnd(36));
+  const status = padColored(statusColor(t.status), t.status, 11);
+  const priority = padColored(priorityColor(t.priority), t.priority, 6);
+  const created = ansis.dim(formatTime(t.created_at).padEnd(19));
+  const updated = ansis.dim(formatTime(t.updated_at).padEnd(19));
   console.log(
-    `  ${id}  ${statusColor(t.status)}  ${priorityColor(t.priority)}  ${t.name}`,
+    `${id}  ${status}  ${priority}  ${created}  ${updated}  ${t.name}`,
   );
 }
 

package/src/commands/thread.ts

@@ -19,12 +19,14 @@ export function registerThreadCommand(program: Command) {
     .command("list")
     .description("List threads")
     .option("-t, --type <type>", "filter by type (daemon_tick, chat_session)")
-    .option("-l, --limit <n>", "max number of threads", parseInt)
+    .option("-l, --limit <n>", "max number of threads", Number.parseInt)
+    .option("-o, --offset <n>", "skip first N threads", Number.parseInt)
     .action((opts) =>
       withDb(program, async (conn) => {
         const threads = await listThreads(conn, {
           type: opts.type,
           limit: opts.limit,
+          offset: opts.offset,
         });
 
         if (threads.length === 0) {

package/src/commands/tools.ts

@@ -2,6 +2,7 @@ import ansis from "ansis";
 import type { Command } from "commander";
 import { z } from "zod";
 import { loadConfig } from "../config/loader.ts";
+import { getDbPath } from "../constants.ts";
 import { registerAllTools } from "../tools/registry.ts";
 import {
   type AnyToolDefinition,
@@ -112,6 +113,7 @@ function registerToolAsCLI(parent: Command, tool: AnyToolDefinition) {
 
         const ctx: ToolContext = {
           conn,
+          dbPath: getDbPath(dir),
           projectDir: dir,
           config: await loadConfig(dir),
           mcpxClient: null,

package/src/commands/with-db.ts

@@ -1,23 +1,22 @@
 import type { Command } from "commander";
 import { getDbPath } from "../constants.ts";
 import type { DbConnection } from "../db/connection.ts";
-import { getConnection } from "../db/connection.ts";
+import { withDb as coreWithDb } from "../db/connection.ts";
 import { migrate } from "../db/schema.ts";
 
 /**
  * Open a migrated DB connection from the CLI --dir flag, run the callback,
- * and guarantee the connection is closed afterward.
+ * and guarantee the connection is closed afterward. Retries on lock
+ * conflicts so CLI invocations cooperate with a running daemon/chat.
  */
 export async function withDb<T>(
   program: Command,
   fn: (conn: DbConnection, dir: string) => Promise<T>,
 ): Promise<T> {
   const dir = program.opts().dir;
-  const conn = await getConnection(getDbPath(dir));
-  await migrate(conn);
-  try {
-    return await fn(conn, dir);
-  } finally {
-    conn.close();
-  }
+  const dbPath = getDbPath(dir);
+  return coreWithDb(dbPath, async (conn) => {
+    await migrate(conn);
+    return fn(conn, dir);
+  });
 }

package/src/context/describer.ts

@@ -3,6 +3,8 @@ import type { BotholomewConfig } from "../config/schemas.ts";
 import { logger } from "../utils/logger.ts";
 
 const DESCRIBE_TOOL_NAME = "return_description";
+const DESCRIBE_AND_PLACE_TOOL_NAME = "return_description_and_path";
+
 const DESCRIBE_TOOL = {
   name: DESCRIBE_TOOL_NAME,
   description: "Return a one-sentence description of this content.",
@@ -19,6 +21,28 @@ const DESCRIBE_TOOL = {
   },
 };
 
+const DESCRIBE_AND_PLACE_TOOL = {
+  name: DESCRIBE_AND_PLACE_TOOL_NAME,
+  description:
+    "Return a one-sentence description AND a suggested absolute folder path for this file.",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      description: {
+        type: "string",
+        description:
+          "A concise one-sentence summary of what this content is about.",
+      },
+      suggested_path: {
+        type: "string",
+        description:
+          "Absolute virtual-filesystem path (starts with /) where this file should live, including the filename. Prefer existing folders. Include a project/source disambiguator (e.g. /projects/<source-dir>/README.md) when the basename is likely to collide.",
+      },
+    },
+    required: ["description", "suggested_path"],
+  },
+};
+
 const TIMEOUT_MS = 10_000;
 const MAX_CONTENT_CHARS = 8000;
 const MAX_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
@@ -38,8 +62,27 @@ type ImageMediaType = "image/jpeg" | "image/png" | "image/gif" | "image/webp";
  */
 async function buildMessageContent(
   opts: DescriberOpts,
+  includePlacement: boolean,
 ): Promise<Anthropic.Messages.ContentBlockParam[]> {
-  const textPrompt = `Describe this file in one sentence. Be specific about what it contains, not generic.\n\nFilename: ${opts.filename}\nMIME type: ${opts.mimeType}`;
+  const placementBlock = includePlacement
+    ? [
+        "",
+        "Also suggest an absolute folder path where this file should live in the virtual filesystem. Rules:",
+        "- Start with /",
+        "- Keep the basename close to the source filename",
+        "- STRONGLY prefer folders that already exist below — reuse them unless the new file is clearly unrelated to everything there. Do NOT invent a new folder that is a near-synonym of an existing one.",
+        "- Use at most 3 nested folders unless an existing folder already goes deeper",
+        "- If the basename is common (README.md, index.md, notes.md), include a project/source disambiguator from the source path",
+        opts.existingTree
+          ? `\nExisting filesystem (folders end with /, files are listed under the folders they live in so you can see what kinds of documents are already there):\n${opts.existingTree}`
+          : "\nExisting filesystem: (empty — you are placing the first file)",
+        opts.sourcePath ? `\nSource filesystem path: ${opts.sourcePath}` : "",
+      ]
+        .filter((s) => s.length > 0)
+        .join("\n")
+    : "";
+
+  const textPrompt = `Describe this file in one sentence. Be specific about what it contains, not generic.\n\nFilename: ${opts.filename}\nMIME type: ${opts.mimeType}${placementBlock ? `\n${placementBlock}` : ""}`;
 
   // Text file — include content inline
   if (opts.content) {
@@ -98,6 +141,20 @@ interface DescriberOpts {
   mimeType: string;
   content: string | null;
   filePath?: string;
+  sourcePath?: string;
+  existingTree?: string;
+}
+
+/** Normalize and validate an LLM-suggested path. Returns null if invalid. */
+export function sanitizeSuggestedPath(raw: string): string | null {
+  const trimmed = raw.trim();
+  if (!trimmed) return null;
+  if (!trimmed.startsWith("/")) return null;
+  if (trimmed.includes("..")) return null;
+  // Collapse repeated slashes, strip trailing slash (unless root).
+  const collapsed = trimmed.replace(/\/+/g, "/");
+  if (collapsed === "/") return null; // needs a filename
+  return collapsed.endsWith("/") ? collapsed.slice(0, -1) : collapsed;
 }
 
 /**
@@ -116,7 +173,7 @@ export async function generateDescription(
   const client = new Anthropic({ apiKey: config.anthropic_api_key });
 
   try {
-    const content = await buildMessageContent(opts);
+    const content = await buildMessageContent(opts, false);
 
     const response = await Promise.race([
       client.messages.create({
@@ -144,3 +201,55 @@ export async function generateDescription(
     return "";
   }
 }
+
+/**
+ * Generate description + suggested_path in a single LLM call.
+ * Returns { description, suggested_path } on success, or null on failure.
+ */
+export async function generateDescriptionAndPath(
+  config: Required<BotholomewConfig>,
+  opts: DescriberOpts,
+): Promise<{ description: string; suggested_path: string } | null> {
+  if (!config.anthropic_api_key) return null;
+
+  const client = new Anthropic({ apiKey: config.anthropic_api_key });
+
+  try {
+    const content = await buildMessageContent(opts, true);
+
+    const response = await Promise.race([
+      client.messages.create({
+        model: config.chunker_model,
+        max_tokens: 512,
+        tools: [DESCRIBE_AND_PLACE_TOOL],
+        tool_choice: { type: "tool", name: DESCRIBE_AND_PLACE_TOOL_NAME },
+        messages: [{ role: "user", content }],
+      }),
+      new Promise<never>((_, reject) =>
+        setTimeout(
+          () => reject(new Error("Description+path generation timeout")),
+          TIMEOUT_MS,
+        ),
+      ),
+    ]);
+
+    const toolBlock = response.content.find((b) => b.type === "tool_use");
+    if (!toolBlock || toolBlock.type !== "tool_use") return null;
+
+    const input = toolBlock.input as {
+      description?: string;
+      suggested_path?: string;
+    };
+    const suggested = input.suggested_path
+      ? sanitizeSuggestedPath(input.suggested_path)
+      : null;
+    if (!suggested) return null;
+    return {
+      description: input.description || "",
+      suggested_path: suggested,
+    };
+  } catch (err) {
+    logger.debug(`Description+path generation failed: ${err}`);
+    return null;
+  }
+}

package/src/context/fetcher.ts

@@ -161,6 +161,7 @@ async function runFetcherLoop(
 
   const toolCtx: ToolContext = {
     conn: null as unknown as DbConnection,
+    dbPath: "",
     projectDir: "",
     config,
     mcpxClient,

package/src/context/ingest.ts

@@ -79,7 +79,9 @@ export interface IngestionResult {
 
 /**
  * Store a prepared ingestion into the database.
- * This is the fast DB-write phase and must be called sequentially.
+ * All statements in BEGIN/COMMIT/ROLLBACK must share one connection, so the
+ * caller must pass a connection that lives long enough for the transaction
+ * (the tool executor wraps each tool call in `withDb`, which satisfies this).
  */
 export async function storeIngestion(
   conn: DbConnection,

package/src/daemon/index.ts

@@ -1,7 +1,7 @@
 import ansis from "ansis";
 import { loadConfig } from "../config/loader.ts";
 import { getDbPath } from "../constants.ts";
-import { getConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { migrate } from "../db/schema.ts";
 import { createMcpxClient } from "../mcpx/client.ts";
 import { logger } from "../utils/logger.ts";
@@ -49,8 +49,10 @@ export async function startDaemon(
 ): Promise<void> {
   const config = await loadConfig(projectDir);
   const dbPath = getDbPath(projectDir);
-  const conn = await getConnection(dbPath);
-  await migrate(conn);
+
+  // One short-lived connection to apply migrations. After this returns,
+  // the file lock is released so CLI invocations can run freely.
+  await withDb(dbPath, (conn) => migrate(conn));
 
   // Initialize MCPX client for external tool access
   const mcpxClient = await createMcpxClient(projectDir);
@@ -64,7 +66,6 @@ export async function startDaemon(
     logger.info("Daemon shutting down...");
     await mcpxClient?.close();
     await removePidFile(projectDir);
-    conn.close();
     process.exit(0);
   };
 
@@ -87,7 +88,7 @@ export async function startDaemon(
     try {
       didWork = await tick(
         projectDir,
-        conn,
+        dbPath,
         config,
         mcpxClient,
         callbacks,

package/src/daemon/llm.ts

@@ -7,7 +7,7 @@ import type {
 } from "@anthropic-ai/sdk/resources/messages";
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { getTask, type Task } from "../db/tasks.ts";
 import { logInteraction } from "../db/threads.ts";
 import { registerAllTools } from "../tools/registry.ts";
@@ -44,32 +44,32 @@ export async function runAgentLoop(input: {
   systemPrompt: string;
   task: Task;
   config: Required<BotholomewConfig>;
-  conn: DbConnection;
+  dbPath: string;
   threadId: string;
   projectDir: string;
   mcpxClient?: McpxClient | null;
   callbacks?: DaemonStreamCallbacks;
 }): Promise<AgentLoopResult> {
-  const { systemPrompt, task, config, conn, threadId, projectDir, callbacks } =
-    input;
+  const {
+    systemPrompt,
+    task,
+    config,
+    dbPath,
+    threadId,
+    projectDir,
+    callbacks,
+  } = input;
 
   const client = new Anthropic({
     apiKey: config.anthropic_api_key || undefined,
   });
 
-  const toolCtx: ToolContext = {
-    conn,
-    projectDir,
-    config,
-    mcpxClient: input.mcpxClient ?? null,
-  };
-
   // Build predecessor context from completed blocking tasks
   let predecessorContext = "";
   if (task.blocked_by.length > 0) {
     const predecessorOutputs: string[] = [];
     for (const blockerId of task.blocked_by) {
-      const blocker = await getTask(conn, blockerId);
+      const blocker = await withDb(dbPath, (conn) => getTask(conn, blockerId));
       if (blocker?.output) {
         predecessorOutputs.push(
           `### ${blocker.name} (${blocker.id})\n${blocker.output}`,
@@ -86,11 +86,13 @@ export async function runAgentLoop(input: {
   const messages: MessageParam[] = [{ role: "user", content: userMessage }];
 
   // Log the initial user message
-  await logInteraction(conn, threadId, {
-    role: "user",
-    kind: "message",
-    content: userMessage,
-  });
+  await withDb(dbPath, (conn) =>
+    logInteraction(conn, threadId, {
+      role: "user",
+      kind: "message",
+      content: userMessage,
+    }),
+  );
 
   clearLargeResults();
   const daemonTools = toAnthropicTools();
@@ -144,13 +146,15 @@ export async function runAgentLoop(input: {
     // Log assistant text blocks
     for (const block of response.content) {
       if (block.type === "text" && block.text) {
-        await logInteraction(conn, threadId, {
-          role: "assistant",
-          kind: "message",
-          content: block.text,
-          durationMs,
-          tokenCount,
-        });
+        await withDb(dbPath, (conn) =>
+          logInteraction(conn, threadId, {
+            role: "assistant",
+            kind: "message",
+            content: block.text,
+            durationMs,
+            tokenCount,
+          }),
+        );
       }
     }
 
@@ -173,20 +177,30 @@ export async function runAgentLoop(input: {
     for (const toolUse of toolUseBlocks) {
       const toolInput = JSON.stringify(toolUse.input);
       callbacks?.onToolStart(toolUse.name, toolInput);
-      await logInteraction(conn, threadId, {
-        role: "assistant",
-        kind: "tool_use",
-        content: `Calling ${toolUse.name}`,
-        toolName: toolUse.name,
-        toolInput,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "assistant",
+          kind: "tool_use",
+          content: `Calling ${toolUse.name}`,
+          toolName: toolUse.name,
+          toolInput,
+        }),
+      );
     }
 
-    // Execute all tools in parallel
+    // Execute all tools in parallel. Each tool call opens its own short-lived
+    // connection (or none, if the tool uses dbPath internally) via
+    // executeToolCall — so parallel tool calls share the process-local
+    // DuckDB instance and release the file lock as soon as they finish.
     const execResults = await Promise.all(
       toolUseBlocks.map(async (toolUse) => {
         const start = Date.now();
-        const result = await executeToolCall(toolUse, toolCtx);
+        const result = await executeToolCall(toolUse, {
+          dbPath,
+          projectDir,
+          config,
+          mcpxClient: input.mcpxClient ?? null,
+        });
         const elapsed = Date.now() - start;
         callbacks?.onToolEnd(
           toolUse.name,
@@ -201,13 +215,15 @@ export async function runAgentLoop(input: {
     // Log results and collect tool_result messages
     const toolResults: ToolResultBlockParam[] = [];
     for (const { toolUse, result, durationMs } of execResults) {
-      await logInteraction(conn, threadId, {
-        role: "tool",
-        kind: "tool_result",
-        content: result.output,
-        toolName: toolUse.name,
-        durationMs,
-      });
+      await withDb(dbPath, (conn) =>
+        logInteraction(conn, threadId, {
+          role: "tool",
+          kind: "tool_result",
+          content: result.output,
+          toolName: toolUse.name,
+          durationMs,
+        }),
+      );
 
       if (result.terminal && result.agentResult) {
         return result.agentResult;
@@ -234,9 +250,16 @@ interface ToolCallResult {
   agentResult?: AgentLoopResult;
 }
 
+interface ToolCallCtx {
+  dbPath: string;
+  projectDir: string;
+  config: Required<BotholomewConfig>;
+  mcpxClient: McpxClient | null;
+}
+
 async function executeToolCall(
   toolUse: ToolUseBlock,
-  ctx: ToolContext,
+  baseCtx: ToolCallCtx,
 ): Promise<ToolCallResult> {
   const tool = getTool(toolUse.name);
   if (!tool) {
@@ -261,7 +284,10 @@ async function executeToolCall(
 
   let result: unknown;
   try {
-    result = await tool.execute(parsed.data, ctx);
+    result = await withDb(baseCtx.dbPath, (conn) => {
+      const ctx: ToolContext = { ...baseCtx, conn };
+      return tool.execute(parsed.data, ctx);
+    });
   } catch (err) {
     return {
       output: `Tool ${toolUse.name} threw an error: ${err}. You may retry with different parameters or try an alternative approach.`,

package/src/daemon/prompt.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { getBotholomewDir } from "../constants.ts";
 import { embedSingle } from "../context/embedder.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import { hybridSearch } from "../db/embeddings.ts";
 import type { Task } from "../db/tasks.ts";
 import { parseContextFile } from "../utils/frontmatter.ts";
@@ -89,7 +89,7 @@ export function buildMetaHeader(projectDir: string): string[] {
 export async function buildSystemPrompt(
   projectDir: string,
   task?: Task,
-  conn?: DbConnection,
+  dbPath?: string,
   _config?: Required<BotholomewConfig>,
   options?: { hasMcpTools?: boolean },
 ): Promise<string> {
@@ -107,11 +107,13 @@ export async function buildSystemPrompt(
   parts.push(...(await loadPersistentContext(projectDir, taskKeywords)));
 
   // Relevant context from embeddings search
-  if (task && conn && _config?.openai_api_key) {
+  if (task && dbPath && _config?.openai_api_key) {
     try {
       const query = `${task.name} ${task.description}`;
       const queryVec = await embedSingle(query, _config);
-      const results = await hybridSearch(conn, query, queryVec, 5);
+      const results = await withDb(dbPath, (conn) =>
+        hybridSearch(conn, query, queryVec, 5),
+      );
 
       if (results.length > 0) {
         parts.push("## Relevant Context");

package/src/daemon/schedules.ts

@@ -1,6 +1,6 @@
 import Anthropic from "@anthropic-ai/sdk";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
+import { withDb } from "../db/connection.ts";
 import {
   listSchedules,
   markScheduleRun,
@@ -105,14 +105,17 @@ Is this schedule due to run? If yes, what tasks should be created?`;
 }
 
 export async function processSchedules(
-  conn: DbConnection,
+  dbPath: string,
   config: Required<BotholomewConfig>,
 ): Promise<void> {
-  const schedules = await listSchedules(conn, { enabled: true });
+  const schedules = await withDb(dbPath, (conn) =>
+    listSchedules(conn, { enabled: true }),
+  );
   if (schedules.length === 0) return;
 
   for (const schedule of schedules) {
     try {
+      // LLM evaluation does no DB work — no connection held here.
       const evaluation = await evaluateSchedule(config, schedule);
 
       if (!evaluation.isDue) {
@@ -128,16 +131,18 @@ export async function processSchedules(
           .map((i: number) => createdIds[i])
           .filter(Boolean) as string[];
 
-        const task = await createTask(conn, {
-          name: taskDef.name,
-          description: taskDef.description,
-          priority: taskDef.priority,
-          blocked_by: blockedBy,
-        });
+        const task = await withDb(dbPath, (conn) =>
+          createTask(conn, {
+            name: taskDef.name,
+            description: taskDef.description,
+            priority: taskDef.priority,
+            blocked_by: blockedBy,
+          }),
+        );
         createdIds.push(task.id);
       }
 
-      await markScheduleRun(conn, schedule.id);
+      await withDb(dbPath, (conn) => markScheduleRun(conn, schedule.id));
       logger.info(
         `Schedule "${schedule.name}" fired, created ${createdIds.length} task(s)`,
       );