botholomew 0.12.5 → 0.14.0

package/src/tui/components/WorkerPanel.tsx

@@ -1,11 +1,10 @@
 import { Box, Text, useInput, useStdout } from "ink";
 import { memo, useEffect, useMemo, useState } from "react";
-import { withDb } from "../../db/connection.ts";
-import { listWorkers, type Worker } from "../../db/workers.ts";
 import { readLogTail } from "../../worker/log-reader.ts";
+import { listWorkers, type Worker } from "../../workers/store.ts";
 
 interface WorkerPanelProps {
-  dbPath: string;
+  projectDir: string;
   isActive: boolean;
 }
 
@@ -30,7 +29,9 @@ function statusColor(status: Worker["status"]): string {
   }
 }
 
-function formatAge(from: Date, now: Date): string {
+function formatAge(fromIso: string, now: Date): string {
+  const from = new Date(fromIso);
+  if (Number.isNaN(from.getTime())) return "?";
   const secs = Math.max(0, Math.floor((now.getTime() - from.getTime()) / 1000));
   if (secs < 60) return `${secs}s`;
   const mins = Math.floor(secs / 60);
@@ -47,7 +48,7 @@ function formatBytes(n: number): string {
 }
 
 export const WorkerPanel = memo(function WorkerPanel({
-  dbPath,
+  projectDir,
   isActive,
 }: WorkerPanelProps) {
   const { stdout } = useStdout();
@@ -68,15 +69,17 @@ export const WorkerPanel = memo(function WorkerPanel({
 
     const refresh = async () => {
       const status = STATUS_FILTERS[filterIdx] ?? undefined;
-      const result = await withDb(dbPath, (conn) =>
-        listWorkers(conn, status ? { status } : {}),
-      );
-      if (mounted) {
-        setWorkers(result);
-        setNow(new Date());
-        setSelectedIndex((prev) =>
-          Math.min(prev, Math.max(0, result.length - 1)),
-        );
+      try {
+        const result = await listWorkers(projectDir, status ? { status } : {});
+        if (mounted) {
+          setWorkers(result);
+          setNow(new Date());
+          setSelectedIndex((prev) =>
+            Math.min(prev, Math.max(0, result.length - 1)),
+          );
+        }
+      } catch {
+        // ignore — next tick retries
       }
     };
 
@@ -86,7 +89,7 @@ export const WorkerPanel = memo(function WorkerPanel({
       mounted = false;
       clearInterval(interval);
     };
-  }, [dbPath, filterIdx]);
+  }, [projectDir, filterIdx]);
 
   const selected = workers[selectedIndex];
   const selectedLogPath = selected?.log_path ?? null;
@@ -332,18 +335,17 @@ function WorkerDetail({ worker, now }: { worker: Worker; now: Date }) {
         </Text>
         <Text>
           <Text dimColor>Started </Text>
-          {worker.started_at.toISOString()}{" "}
+          {worker.started_at}{" "}
           <Text dimColor>({formatAge(worker.started_at, now)} ago)</Text>
         </Text>
         <Text>
-          <Text dimColor>Heartbeat</Text>{" "}
-          {worker.last_heartbeat_at.toISOString()}{" "}
+          <Text dimColor>Heartbeat</Text> {worker.last_heartbeat_at}{" "}
           <Text dimColor>({formatAge(worker.last_heartbeat_at, now)} ago)</Text>
         </Text>
         {worker.stopped_at && (
           <Text>
             <Text dimColor>Stopped </Text>
-            {worker.stopped_at.toISOString()}
+            {worker.stopped_at}
           </Text>
         )}
         {worker.task_id && (

package/src/tui/markdown.ts

@@ -1,14 +1,8 @@
-import type { ContextItem } from "../db/context.ts";
-
 export function renderMarkdown(text: string): string {
   if (!text) return "";
   return Bun.markdown.ansi(text).trimEnd();
 }
 
-export function isMarkdownItem(
-  item: Pick<ContextItem, "mime_type" | "path">,
-): boolean {
-  if (item.mime_type === "text/markdown") return true;
-  if (item.path.toLowerCase().endsWith(".md")) return true;
-  return false;
+export function isMarkdownPath(path: string): boolean {
+  return path.toLowerCase().endsWith(".md");
 }

package/src/utils/title.ts

@@ -1,18 +1,16 @@
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { withDb } from "../db/connection.ts";
-import { updateThreadTitle } from "../db/threads.ts";
+import { updateThreadTitle } from "../threads/store.ts";
 import { createLlmClient } from "../worker/llm-client.ts";
 import { logger } from "./logger.ts";
 
 /**
  * Generate a short title for a thread using the chunker model (Haiku).
- * Fire-and-forget — errors are logged at debug level and never propagated.
- * Opens its own short-lived DB connection for the write so callers can
- * safely `void`-chain without holding a connection during the LLM call.
+ * Fire-and-forget — errors are logged and never propagated. Writes the
+ * title back to the thread's CSV file by rewriting the thread_meta row.
  */
 export async function generateThreadTitle(
   config: Required<BotholomewConfig>,
-  dbPath: string,
+  projectDir: string,
   threadId: string,
   context: string,
 ): Promise<void> {
@@ -39,7 +37,7 @@ export async function generateThreadTitle(
       .trim();
 
     if (title) {
-      await withDb(dbPath, (conn) => updateThreadTitle(conn, threadId, title));
+      await updateThreadTitle(projectDir, threadId, title);
     }
   } catch (err) {
     logger.warn(`Failed to generate thread title: ${err}`);

package/src/utils/v7-date.ts

@@ -0,0 +1,47 @@
+/**
+ * Helpers for deriving a UTC `YYYY-MM-DD` date string from a uuidv7 id.
+ * Used by the threads store and the worker-log spawn path so file layouts
+ * grouped by creation date can be computed without scanning the disk.
+ */
+
+/**
+ * Format a Date as `YYYY-MM-DD` in UTC. UTC (not local time) keeps file
+ * paths stable across machines and timezone moves: a thread created at
+ * 11pm PT and read the next morning from a different zone resolves to
+ * the same folder either way.
+ */
+export function utcDateString(d: Date): string {
+  const y = d.getUTCFullYear();
+  const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(d.getUTCDate()).padStart(2, "0");
+  return `${y}-${m}-${day}`;
+}
+
+/**
+ * Recover the creation timestamp from a uuidv7. The first 48 bits of v7
+ * are unix-millis. Returns null for non-v7 ids (or any parse failure) so
+ * the caller can fall back — typically to a directory walk for reads, or
+ * to "today" for writes.
+ */
+export function dateFromUuidV7(id: string): string | null {
+  // shape: xxxxxxxx-xxxx-7xxx-yxxx-xxxxxxxxxxxx — the version nibble is
+  // the 13th hex char (position 14 with the dash).
+  if (id.length < 19 || id[14] !== "7") return null;
+  const hex = id.slice(0, 8) + id.slice(9, 13);
+  const ms = Number.parseInt(hex, 16);
+  if (!Number.isFinite(ms) || ms <= 0) return null;
+  const d = new Date(ms);
+  if (Number.isNaN(d.getTime())) return null;
+  return utcDateString(d);
+}
+
+/**
+ * Best-effort: prefer the v7-derived date, else fall back to today's UTC.
+ * Use for write paths where you have an id and need a concrete date dir.
+ */
+export function dateForId(id: string): string {
+  return dateFromUuidV7(id) ?? utcDateString(new Date());
+}
+
+/** Regex that matches `YYYY-MM-DD` directory names. */
+export const DATE_DIR_RE = /^\d{4}-\d{2}-\d{2}$/;

package/src/worker/heartbeat.ts

@@ -1,32 +1,28 @@
-import { withDb } from "../db/connection.ts";
+import { reapOrphanScheduleLocks } from "../schedules/store.ts";
+import { reapOrphanLocks as reapOrphanTaskLocks } from "../tasks/store.ts";
+import { logger } from "../utils/logger.ts";
 import {
   heartbeat,
+  isWorkerRunning,
   pruneStoppedWorkers,
   reapDeadWorkers,
-} from "../db/workers.ts";
-import { logger } from "../utils/logger.ts";
+} from "../workers/store.ts";
 
 /**
- * Start a non-blocking heartbeat interval for a running worker.
- *
- * The heartbeat runs on its own `setInterval` timer so it stays live even
- * while the worker is blocked inside a long LLM call. We `unref` the timer
- * so it doesn't keep the Bun event loop alive on its own — the main tick
- * loop (or the awaited one-shot task) is what keeps the process running.
- *
- * Errors are swallowed with a warning: a transient DB lock shouldn't crash
- * a worker that's otherwise doing useful work. If every heartbeat fails the
- * worker will eventually be reaped, which is the correct outcome.
+ * Start a non-blocking heartbeat interval for a running worker. Each tick
+ * atomically rewrites `<projectDir>/workers/<id>.json` with an updated
+ * `last_heartbeat_at`. The setInterval handle is unref'd so the heartbeat
+ * doesn't keep the Bun event loop alive on its own.
  */
 export function startHeartbeat(
-  dbPath: string,
+  projectDir: string,
   workerId: string,
   intervalSeconds: number,
 ): () => void {
   const ms = Math.max(1_000, intervalSeconds * 1_000);
   const handle = setInterval(async () => {
     try {
-      await withDb(dbPath, (conn) => heartbeat(conn, workerId));
+      await heartbeat(projectDir, workerId);
     } catch (err) {
       logger.warn(`worker heartbeat failed: ${err}`);
     }
@@ -36,12 +32,14 @@ export function startHeartbeat(
 }
 
 /**
- * Start a periodic reaper that marks stale workers dead and releases any
- * tasks / schedule claims they held. Only persist workers need this — a
- * one-shot worker does a single reap pass before claiming its task.
+ * Periodic reaper: walk `workers/`, mark any running worker dead whose
+ * heartbeat is older than `staleAfterSeconds`, then walk `tasks/.locks/`
+ * and `schedules/.locks/` and unlink any lockfile whose holder is no
+ * longer running. Cleanly-stopped worker JSON files older than
+ * `stoppedRetentionSeconds` are pruned.
  */
 export function startReaper(
-  dbPath: string,
+  projectDir: string,
   intervalSeconds: number,
   staleAfterSeconds: number,
   stoppedRetentionSeconds: number,
@@ -49,9 +47,7 @@ export function startReaper(
   const ms = Math.max(1_000, intervalSeconds * 1_000);
   const handle = setInterval(async () => {
     try {
-      const reaped = await withDb(dbPath, (conn) =>
-        reapDeadWorkers(conn, staleAfterSeconds),
-      );
+      const reaped = await reapDeadWorkers(projectDir, staleAfterSeconds);
       if (reaped.length > 0) {
         logger.warn(
           `reaped ${reaped.length} stale worker(s): ${reaped.join(", ")}`,
@@ -60,9 +56,35 @@ export function startReaper(
     } catch (err) {
       logger.warn(`worker reap failed: ${err}`);
     }
+
+    const isAlive = (id: string) => isWorkerRunning(projectDir, id);
+
+    try {
+      const released = await reapOrphanTaskLocks(projectDir, isAlive);
+      if (released.length > 0) {
+        logger.warn(
+          `released ${released.length} orphan task lock(s): ${released.join(", ")}`,
+        );
+      }
+    } catch (err) {
+      logger.warn(`task lock reap failed: ${err}`);
+    }
+
+    try {
+      const released = await reapOrphanScheduleLocks(projectDir, isAlive);
+      if (released.length > 0) {
+        logger.warn(
+          `released ${released.length} orphan schedule lock(s): ${released.join(", ")}`,
+        );
+      }
+    } catch (err) {
+      logger.warn(`schedule lock reap failed: ${err}`);
+    }
+
     try {
-      const pruned = await withDb(dbPath, (conn) =>
-        pruneStoppedWorkers(conn, stoppedRetentionSeconds),
+      const pruned = await pruneStoppedWorkers(
+        projectDir,
+        stoppedRetentionSeconds,
       );
       if (pruned.length > 0) {
         logger.debug(

package/src/worker/index.ts

@@ -5,9 +5,9 @@ import { getDbPath } from "../constants.ts";
 import { withDb } from "../db/connection.ts";
 import { migrate } from "../db/schema.ts";
 import { uuidv7 } from "../db/uuid.ts";
-import { markWorkerStopped, registerWorker } from "../db/workers.ts";
 import { createMcpxClient } from "../mcpx/client.ts";
 import { logger } from "../utils/logger.ts";
+import { markWorkerStopped, registerWorker } from "../workers/store.ts";
 import { startHeartbeat, startReaper } from "./heartbeat.ts";
 import type { WorkerStreamCallbacks } from "./llm.ts";
 import { runSpecificTask, tick } from "./tick.ts";
@@ -100,26 +100,24 @@ export async function startWorker(
   }
 
   const workerId = options.workerId ?? uuidv7();
-  await withDb(dbPath, (conn) =>
-    registerWorker(conn, {
-      id: workerId,
-      pid: process.pid,
-      hostname: hostname(),
-      mode,
-      taskId: taskId ?? null,
-      logPath: options.logPath ?? null,
-    }),
-  );
+  await registerWorker(projectDir, {
+    id: workerId,
+    pid: process.pid,
+    hostname: hostname(),
+    mode,
+    taskId: taskId ?? null,
+    logPath: options.logPath ?? null,
+  });
 
   const stopHeartbeat = startHeartbeat(
-    dbPath,
+    projectDir,
     workerId,
     config.worker_heartbeat_interval_seconds,
   );
   const stopReaper =
     mode === "persist"
       ? startReaper(
-          dbPath,
+          projectDir,
           config.worker_reap_interval_seconds,
           config.worker_dead_after_seconds,
           config.worker_stopped_retention_seconds,
@@ -132,7 +130,7 @@ export async function startWorker(
     stopReaper();
     await mcpxClient?.close();
     try {
-      await withDb(dbPath, (conn) => markWorkerStopped(conn, workerId));
+      await markWorkerStopped(projectDir, workerId);
     } catch (err) {
       logger.warn(`failed to mark worker stopped: ${err}`);
     }
@@ -205,7 +203,7 @@ export async function startWorker(
     stopHeartbeat();
     stopReaper();
     try {
-      await withDb(dbPath, (conn) => markWorkerStopped(conn, workerId));
+      await markWorkerStopped(projectDir, workerId);
     } catch (err) {
       logger.warn(`failed to mark worker stopped: ${err}`);
     }

package/src/worker/llm.ts

@@ -7,8 +7,9 @@ import type {
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { withDb } from "../db/connection.ts";
-import { getTask, type Task } from "../db/tasks.ts";
-import { logInteraction } from "../db/threads.ts";
+import type { Task } from "../tasks/schema.ts";
+import { getTask } from "../tasks/store.ts";
+import { logInteraction } from "../threads/store.ts";
 import { registerAllTools } from "../tools/registry.ts";
 import { getTool, type ToolContext, toAnthropicTools } from "../tools/tool.ts";
 import { logger } from "../utils/logger.ts";
@@ -72,7 +73,7 @@ export async function runAgentLoop(input: {
   if (task.blocked_by.length > 0) {
     const predecessorOutputs: string[] = [];
     for (const blockerId of task.blocked_by) {
-      const blocker = await withDb(dbPath, (conn) => getTask(conn, blockerId));
+      const blocker = await getTask(projectDir, blockerId);
       if (blocker?.output) {
         predecessorOutputs.push(
           `### ${blocker.name} (${blocker.id})\n${blocker.output}`,
@@ -89,13 +90,11 @@ export async function runAgentLoop(input: {
   const messages: MessageParam[] = [{ role: "user", content: userMessage }];
 
   // Log the initial user message
-  await withDb(dbPath, (conn) =>
-    logInteraction(conn, threadId, {
-      role: "user",
-      kind: "message",
-      content: userMessage,
-    }),
-  );
+  await logInteraction(projectDir, threadId, {
+    role: "user",
+    kind: "message",
+    content: userMessage,
+  });
 
   clearLargeResults();
   const workerTools = toAnthropicTools();
@@ -149,15 +148,13 @@ export async function runAgentLoop(input: {
     // Log assistant text blocks
     for (const block of response.content) {
       if (block.type === "text" && block.text) {
-        await withDb(dbPath, (conn) =>
-          logInteraction(conn, threadId, {
-            role: "assistant",
-            kind: "message",
-            content: block.text,
-            durationMs,
-            tokenCount,
-          }),
-        );
+        await logInteraction(projectDir, threadId, {
+          role: "assistant",
+          kind: "message",
+          content: block.text,
+          durationMs,
+          tokenCount,
+        });
         if (!callbacks) {
           logger.phase("assistant", block.text);
         }
@@ -189,15 +186,13 @@ export async function runAgentLoop(input: {
           `${toolUse.name} ${truncate(toolInput, 200)}`,
         );
       }
-      await withDb(dbPath, (conn) =>
-        logInteraction(conn, threadId, {
-          role: "assistant",
-          kind: "tool_use",
-          content: `Calling ${toolUse.name}`,
-          toolName: toolUse.name,
-          toolInput,
-        }),
-      );
+      await logInteraction(projectDir, threadId, {
+        role: "assistant",
+        kind: "tool_use",
+        content: `Calling ${toolUse.name}`,
+        toolName: toolUse.name,
+        toolInput,
+      });
     }
 
     // Execute all tools in parallel. Each tool call opens its own short-lived
@@ -227,15 +222,13 @@ export async function runAgentLoop(input: {
     // Log results and collect tool_result messages
     const toolResults: ToolResultBlockParam[] = [];
     for (const { toolUse, result, durationMs } of execResults) {
-      await withDb(dbPath, (conn) =>
-        logInteraction(conn, threadId, {
-          role: "tool",
-          kind: "tool_result",
-          content: result.output,
-          toolName: toolUse.name,
-          durationMs,
-        }),
-      );
+      await logInteraction(projectDir, threadId, {
+        role: "tool",
+        kind: "tool_result",
+        content: result.output,
+        toolName: toolUse.name,
+        durationMs,
+      });
       if (!callbacks) {
         const seconds = (durationMs / 1000).toFixed(1);
         const status = result.isError ? "err" : "ok";

package/src/worker/prompt.ts

@@ -1,13 +1,9 @@
 import { readdir } from "node:fs/promises";
 import { join } from "node:path";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { getBotholomewDir } from "../constants.ts";
-import { embedSingle } from "../context/embedder.ts";
-import { withDb } from "../db/connection.ts";
-import { hybridSearch } from "../db/embeddings.ts";
-import type { Task } from "../db/tasks.ts";
+import { getPromptsDir } from "../constants.ts";
+import type { Task } from "../tasks/schema.ts";
 import { parseContextFile } from "../utils/frontmatter.ts";
-import { logger } from "../utils/logger.ts";
 
 const pkg = await Bun.file(
   new URL("../../package.json", import.meta.url),
@@ -37,23 +33,23 @@ export function extractKeywords(text: string): Set<string> {
 }
 
 /**
- * Load persistent context files from .botholomew/ directory as a single
- * formatted string. Includes "always" files unconditionally and "contextual"
- * files whose content overlaps the provided taskKeywords.
+ * Load persistent context files from prompts/ as a single formatted
+ * string. Includes "always" files unconditionally and "contextual" files
+ * whose content overlaps the provided taskKeywords.
  */
 export async function loadPersistentContext(
   projectDir: string,
   taskKeywords?: Set<string> | null,
 ): Promise<string> {
-  const dotDir = getBotholomewDir(projectDir);
+  const dir = getPromptsDir(projectDir);
   let out = "";
 
   try {
-    const files = await readdir(dotDir);
+    const files = await readdir(dir);
     const mdFiles = files.filter((f) => f.endsWith(".md"));
 
     for (const filename of mdFiles) {
-      const filePath = join(dotDir, filename);
+      const filePath = join(dir, filename);
       const raw = await Bun.file(filePath).text();
       const { meta, content } = parseContextFile(raw);
 
@@ -70,7 +66,7 @@ export async function loadPersistentContext(
       }
     }
   } catch {
-    // .botholomew dir might not have md files yet
+    // prompts/ might not have md files yet
   }
 
   return out;
@@ -104,30 +100,12 @@ export async function buildSystemPrompt(
 
   prompt += await loadPersistentContext(projectDir, taskKeywords);
 
-  if (task && dbPath && _config) {
-    try {
-      const query = `${task.name} ${task.description}`;
-      const queryVec = await embedSingle(query, _config);
-      const results = await withDb(dbPath, (conn) =>
-        hybridSearch(conn, query, queryVec, 5),
-      );
-
-      if (results.length > 0) {
-        prompt += "## Relevant Context\n";
-        for (const r of results) {
-          const ref =
-            r.drive && r.path ? `${r.drive}:${r.path}` : r.context_item_id;
-          prompt += `### ${r.title} (${ref})\n`;
-          if (r.chunk_content) {
-            prompt += `${r.chunk_content.slice(0, 1000)}\n`;
-          }
-          prompt += "\n";
-        }
-      }
-    } catch (err) {
-      logger.debug(`Failed to load contextual embeddings: ${err}`);
-    }
-  }
+  // The agent finds task-relevant content via the `search` tool on demand
+  // rather than having chunks pre-stuffed into the system prompt — keeps the
+  // prompt small and lets the model decide what it actually needs to read.
+  void task;
+  void dbPath;
+  void _config;
 
   prompt += `## Instructions
 You are Botholomew, a wise-owl worker that works through tasks. Use available tools to complete your assigned task, then call complete_task, fail_task, or wait_task. Use create_task for subtasks and update_task to refine pending tasks. Batch independent tool calls in a single response for parallel execution.
@@ -141,19 +119,19 @@ When calling complete_task, write a summary that captures your key findings, dec
 
 ### Local context first
 
-**Before any MCP read, search local context.** Drive, Gmail, GitHub, URLs, and prior agent runs are usually already ingested — refetching is slower, costs tokens, and risks rate limits.
+**Before any MCP read, search local context.** Files in \`context/\` (Gmail dumps, GitHub fetches, URL ingests, prior agent outputs) are usually already there — refetching is slower, costs tokens, and risks rate limits.
 
 Workflow for any "look up / find / read" intent:
 
-1. \`search\` (hybrid regexp + semantic) or \`context_search\` (keyword), then \`context_read\` / \`context_tree\` to drill in.
-2. If freshness matters, call \`context_info\` and check \`indexed_at\`. To re-pull a single stale item, use \`context_refresh\` rather than going to MCP for the whole document.
+1. \`search\` (hybrid regexp + semantic) over \`context/\`, then \`context_read\` / \`context_tree\` to drill in.
+2. If freshness matters, call \`context_info\` and check the file's mtime. To re-pull stale content, write fresh into \`context/\` (\`pipe_to_context\` from an \`mcp_exec\` call is the typical path) rather than going to MCP for the whole document on every question.
 3. Only call \`mcp_exec\` for reads when the data is genuinely missing locally **or** must be real-time (e.g., "what's on my calendar right now").
 
 Writes always go through MCP — sending an email, creating an issue, posting to Slack. Don't search context first for those.
 
 Examples:
 - "What does doc X say?" → \`search\` first.
-- "Any new emails from Y?" → check the \`gmail\` drive first; only hit Gmail MCP if the freshest indexed item is too old for the question.
+- "Any new emails from Y?" → \`search\` for the sender under \`context/gmail/\` (or wherever you've been ingesting mail) before hitting Gmail MCP.
 - "Send an email to Y" → MCP write directly; no context lookup.
 
 ### Calling MCP tools