botholomew 0.12.5 → 0.14.0

package/src/worker/schedules.ts

@@ -1,14 +1,12 @@
 import Anthropic from "@anthropic-ai/sdk";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { withDb } from "../db/connection.ts";
+import type { Schedule } from "../schedules/schema.ts";
 import {
-  claimSchedule,
   listSchedules,
   markScheduleRun,
-  releaseSchedule,
-  type Schedule,
-} from "../db/schedules.ts";
-import { createTask } from "../db/tasks.ts";
+  withScheduleLock,
+} from "../schedules/store.ts";
+import { createTask } from "../tasks/store.ts";
 import { logger } from "../utils/logger.ts";
 
 interface ScheduleTaskDef {
@@ -55,7 +53,7 @@ The "depends_on" array contains indices of other tasks in the array that must co
   const userMessage = `Schedule: "${schedule.name}"
 Description: ${schedule.description || "(none)"}
 Frequency: ${schedule.frequency}
-Last run: ${schedule.last_run_at?.toISOString() ?? "never"}
+Last run: ${schedule.last_run_at ?? "never"}
 Current time: ${new Date().toISOString()}
 
 Is this schedule due to run? If yes, what tasks should be created?`;
@@ -73,7 +71,6 @@ Is this schedule due to run? If yes, what tasks should be created?`;
       .map((b) => b.text)
       .join("");
 
-    // Strip markdown code fences the LLM may wrap around JSON
     text = text
       .replace(/^```(?:json)?\s*\n?/, "")
       .replace(/\n?```\s*$/, "")
@@ -107,75 +104,57 @@ Is this schedule due to run? If yes, what tasks should be created?`;
 }
 
 export async function processSchedules(
-  dbPath: string,
+  projectDir: string,
   config: Required<BotholomewConfig>,
   workerId: string,
 ): Promise<void> {
-  const schedules = await withDb(dbPath, (conn) =>
-    listSchedules(conn, { enabled: true }),
-  );
+  const schedules = await listSchedules(projectDir, { enabled: true });
   if (schedules.length === 0) return;
 
   logger.phase("evaluating-schedules", `${schedules.length} enabled`);
 
   for (const schedule of schedules) {
-    // Only one worker evaluates a schedule per window. claimSchedule is an
-    // atomic UPDATE ... RETURNING guarded by both a claim-stale window and
-    // a minimum-interval-since-last-run window; if it returns null, another
-    // worker already holds the claim or the schedule ran too recently.
-    const claimed = await withDb(dbPath, (conn) =>
-      claimSchedule(conn, schedule.id, workerId, {
-        staleAfterSeconds: config.schedule_claim_stale_seconds,
-        minIntervalSeconds: config.schedule_min_interval_seconds,
-      }),
+    // Lockfile + min-interval guard prevent two workers (or two ticks) from
+    // evaluating the same schedule too closely.
+    await withScheduleLock(
+      projectDir,
+      schedule.id,
+      workerId,
+      { minIntervalSeconds: config.schedule_min_interval_seconds },
+      async (claimed) => {
+        try {
+          const evaluation = await evaluateSchedule(config, claimed);
+
+          if (!evaluation.isDue) {
+            logger.debug(
+              `Schedule "${claimed.name}" not due: ${evaluation.reasoning}`,
+            );
+            return;
+          }
+
+          const createdIds: string[] = [];
+          for (const taskDef of evaluation.tasksToCreate) {
+            const blockedBy = (taskDef.depends_on ?? [])
+              .map((i: number) => createdIds[i])
+              .filter(Boolean) as string[];
+
+            const task = await createTask(projectDir, {
+              name: taskDef.name,
+              description: taskDef.description,
+              priority: taskDef.priority,
+              blocked_by: blockedBy,
+            });
+            createdIds.push(task.id);
+          }
+
+          await markScheduleRun(projectDir, claimed.id);
+          logger.info(
+            `Schedule "${claimed.name}" fired, created ${createdIds.length} task(s)`,
+          );
+        } catch (err) {
+          logger.error(`Error processing schedule "${claimed.name}": ${err}`);
+        }
+      },
     );
-    if (!claimed) {
-      logger.debug(
-        `Schedule "${schedule.name}" skipped: claimed by another worker or too recent`,
-      );
-      continue;
-    }
-
-    try {
-      const evaluation = await evaluateSchedule(config, claimed);
-
-      if (!evaluation.isDue) {
-        logger.debug(
-          `Schedule "${schedule.name}" not due: ${evaluation.reasoning}`,
-        );
-        continue;
-      }
-
-      const createdIds: string[] = [];
-      for (const taskDef of evaluation.tasksToCreate) {
-        const blockedBy = (taskDef.depends_on ?? [])
-          .map((i: number) => createdIds[i])
-          .filter(Boolean) as string[];
-
-        const task = await withDb(dbPath, (conn) =>
-          createTask(conn, {
-            name: taskDef.name,
-            description: taskDef.description,
-            priority: taskDef.priority,
-            blocked_by: blockedBy,
-          }),
-        );
-        createdIds.push(task.id);
-      }
-
-      await withDb(dbPath, (conn) => markScheduleRun(conn, schedule.id));
-      logger.info(
-        `Schedule "${schedule.name}" fired, created ${createdIds.length} task(s)`,
-      );
-    } catch (err) {
-      logger.error(`Error processing schedule "${schedule.name}": ${err}`);
-    } finally {
-      // Release the claim so other workers (or the next tick) can re-evaluate
-      // once the min-interval window has elapsed. markScheduleRun above
-      // updates last_run_at, which is the actual cooldown.
-      await withDb(dbPath, (conn) =>
-        releaseSchedule(conn, schedule.id, workerId),
-      );
-    }
   }
 }

package/src/worker/spawn.ts

@@ -1,12 +1,9 @@
 import { mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import {
-  getBotholomewDir,
-  getWorkerLogPath,
-  getWorkerLogsDir,
-} from "../constants.ts";
+import { dirname } from "node:path";
+import { getConfigPath, getWorkerLogPath } from "../constants.ts";
 import { uuidv7 } from "../db/uuid.ts";
 import { logger } from "../utils/logger.ts";
+import { dateForId } from "../utils/v7-date.ts";
 import type { WorkerMode } from "./index.ts";
 
 export interface SpawnWorkerOptions {
@@ -26,16 +23,19 @@ export async function spawnWorker(
   projectDir: string,
   options: SpawnWorkerOptions = {},
 ): Promise<{ pid: number; workerId: string; logPath: string }> {
-  const dotDir = getBotholomewDir(projectDir);
-  const dirExists = await Bun.file(join(dotDir, "config.json")).exists();
-  if (!dirExists) {
+  const configPath = getConfigPath(projectDir);
+  const initialized = await Bun.file(configPath).exists();
+  if (!initialized) {
     logger.error("Project not initialized. Run 'botholomew init' first.");
     process.exit(1);
   }
 
   const workerId = uuidv7();
-  await mkdir(getWorkerLogsDir(projectDir), { recursive: true });
-  const logPath = getWorkerLogPath(projectDir, workerId);
+  // Per-worker log path is derived from the worker id's UTC date so the
+  // logs/ tree stays browsable as workers accumulate. Mirrors the threads
+  // layout under <projectDir>/threads/.
+  const logPath = getWorkerLogPath(projectDir, workerId, dateForId(workerId));
+  await mkdir(dirname(logPath), { recursive: true });
   const logFile = Bun.file(logPath);
 
   const workerScript = new URL("./run.ts", import.meta.url).pathname;

package/src/worker/tick.ts

@@ -1,14 +1,14 @@
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { withDb } from "../db/connection.ts";
+import type { Task } from "../tasks/schema.ts";
 import {
   claimNextTask,
   claimSpecificTask,
+  releaseTaskLock,
   resetStaleTasks,
-  type Task,
   updateTaskStatus,
-} from "../db/tasks.ts";
-import { createThread, endThread, logInteraction } from "../db/threads.ts";
+} from "../tasks/store.ts";
+import { createThread, endThread, logInteraction } from "../threads/store.ts";
 import { logger } from "../utils/logger.ts";
 import { generateThreadTitle } from "../utils/title.ts";
 import type { WorkerStreamCallbacks } from "./llm.ts";
@@ -46,9 +46,9 @@ export async function tick(opts: TickOptions): Promise<boolean> {
   const tickStart = Date.now();
   logger.phase("tick-start", `#${tickNum}`);
 
-  // Reset stale tasks stuck in in_progress
-  const resetIds = await withDb(dbPath, (conn) =>
-    resetStaleTasks(conn, config.max_tick_duration_seconds * 3),
+  const resetIds = await resetStaleTasks(
+    projectDir,
+    config.max_tick_duration_seconds * 3,
   );
   if (resetIds.length > 0) {
     logger.warn(
@@ -58,15 +58,14 @@ export async function tick(opts: TickOptions): Promise<boolean> {
 
   if (evalSchedules) {
     try {
-      await processSchedules(dbPath, config, workerId);
+      await processSchedules(projectDir, config, workerId);
     } catch (err) {
       logger.error(`Schedule processing failed: ${err}`);
     }
   }
 
-  // Claim a task
   logger.phase("claiming-task");
-  const task = await withDb(dbPath, (conn) => claimNextTask(conn, workerId));
+  const task = await claimNextTask(projectDir, workerId);
   if (!task) {
     logger.info("No task claimed (queue empty or all blocked)");
     const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);
@@ -90,8 +89,7 @@ export async function tick(opts: TickOptions): Promise<boolean> {
 
 /**
  * Claim and run a single, explicitly-named task. Returns true if the task
- * was claimed and processed, false if it wasn't eligible (already claimed,
- * not pending, or doesn't exist).
+ * was claimed and processed, false if it wasn't eligible.
  */
 export async function runSpecificTask(opts: {
   projectDir: string;
@@ -102,8 +100,10 @@ export async function runSpecificTask(opts: {
   mcpxClient?: McpxClient | null;
   callbacks?: WorkerStreamCallbacks;
 }): Promise<boolean> {
-  const task = await withDb(opts.dbPath, (conn) =>
-    claimSpecificTask(conn, opts.taskId, opts.workerId),
+  const task = await claimSpecificTask(
+    opts.projectDir,
+    opts.taskId,
+    opts.workerId,
   );
   if (!task) {
     logger.warn(
@@ -138,8 +138,11 @@ async function runClaimedTask(opts: {
   }
   callbacks?.onTaskStart(task);
 
-  const threadId = await withDb(dbPath, (conn) =>
-    createThread(conn, "worker_tick", task.id, `Working: ${task.name}`),
+  const threadId = await createThread(
+    projectDir,
+    "worker_tick",
+    task.id,
+    `Working: ${task.name}`,
   );
 
   const systemPrompt = await buildSystemPrompt(
@@ -163,47 +166,40 @@ async function runClaimedTask(opts: {
     });
 
     const isComplete = result.status === "complete";
-    await withDb(dbPath, (conn) =>
-      updateTaskStatus(
-        conn,
-        task.id,
-        result.status,
-        isComplete ? null : result.reason,
-        isComplete ? result.reason : null,
-      ),
+    await updateTaskStatus(
+      projectDir,
+      task.id,
+      result.status,
+      isComplete ? null : result.reason,
+      isComplete ? result.reason : null,
     );
 
-    await withDb(dbPath, (conn) =>
-      logInteraction(conn, threadId, {
-        role: "system",
-        kind: "status_change",
-        content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
-      }),
-    );
+    await logInteraction(projectDir, threadId, {
+      role: "system",
+      kind: "status_change",
+      content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
+    });
 
     logger.info(`Task ${task.id} -> ${result.status}`);
 
     void generateThreadTitle(
       config,
-      dbPath,
+      projectDir,
       threadId,
       `Task: ${task.name}\nDescription: ${task.description}\nOutcome: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`,
     );
   } catch (err) {
-    await withDb(dbPath, (conn) =>
-      updateTaskStatus(conn, task.id, "failed", String(err), null),
-    );
+    await updateTaskStatus(projectDir, task.id, "failed", String(err), null);
 
-    await withDb(dbPath, (conn) =>
-      logInteraction(conn, threadId, {
-        role: "system",
-        kind: "status_change",
-        content: `Task ${task.id} failed: ${err}`,
-      }),
-    );
+    await logInteraction(projectDir, threadId, {
+      role: "system",
+      kind: "status_change",
+      content: `Task ${task.id} failed: ${err}`,
+    });
 
     logger.error(`Task ${task.id} failed: ${err}`);
   } finally {
-    await withDb(dbPath, (conn) => endThread(conn, threadId));
+    await releaseTaskLock(projectDir, task.id);
+    await endThread(projectDir, threadId);
   }
 }

package/src/workers/store.ts

@@ -0,0 +1,247 @@
+import { readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { getWorkersDir } from "../constants.ts";
+import { atomicWrite, readWithMtime } from "../fs/atomic.ts";
+
+export const WORKER_MODES = ["persist", "once"] as const;
+export const WORKER_STATUSES = ["running", "stopped", "dead"] as const;
+
+export type WorkerMode = (typeof WORKER_MODES)[number];
+export type WorkerStatus = (typeof WORKER_STATUSES)[number];
+
+/**
+ * Persistent worker record. One JSON file per worker at
+ * `<projectDir>/workers/<id>.json`. Heartbeats rewrite the file
+ * atomically (write-to-tmp + rename); the file's existence is the pidfile,
+ * `last_heartbeat_at` inside is the liveness signal.
+ */
+export interface Worker {
+  id: string;
+  pid: number;
+  hostname: string;
+  mode: WorkerMode;
+  task_id: string | null;
+  status: WorkerStatus;
+  started_at: string;
+  last_heartbeat_at: string;
+  stopped_at: string | null;
+  log_path: string | null;
+}
+
+function workerFilePath(projectDir: string, id: string): string {
+  return join(getWorkersDir(projectDir), `${id}.json`);
+}
+
+async function readWorker(
+  projectDir: string,
+  id: string,
+): Promise<Worker | null> {
+  const file = await readWithMtime(workerFilePath(projectDir, id));
+  if (!file) return null;
+  try {
+    return JSON.parse(file.content) as Worker;
+  } catch {
+    return null;
+  }
+}
+
+async function writeWorker(projectDir: string, worker: Worker): Promise<void> {
+  await atomicWrite(
+    workerFilePath(projectDir, worker.id),
+    `${JSON.stringify(worker, null, 2)}\n`,
+  );
+}
+
+export async function registerWorker(
+  projectDir: string,
+  params: {
+    id: string;
+    pid: number;
+    hostname: string;
+    mode: WorkerMode;
+    taskId?: string | null;
+    logPath?: string | null;
+  },
+): Promise<Worker> {
+  const now = new Date().toISOString();
+  const worker: Worker = {
+    id: params.id,
+    pid: params.pid,
+    hostname: params.hostname,
+    mode: params.mode,
+    task_id: params.taskId ?? null,
+    status: "running",
+    started_at: now,
+    last_heartbeat_at: now,
+    stopped_at: null,
+    log_path: params.logPath ?? null,
+  };
+  await writeWorker(projectDir, worker);
+  return worker;
+}
+
+/**
+ * Update last_heartbeat_at on a running worker. No-op for stopped/dead
+ * workers so a misbehaving heartbeat doesn't resurrect a worker the reaper
+ * has retired.
+ */
+export async function heartbeat(projectDir: string, id: string): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker || worker.status !== "running") return;
+  worker.last_heartbeat_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+export async function markWorkerStopped(
+  projectDir: string,
+  id: string,
+): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker || worker.status !== "running") return;
+  worker.status = "stopped";
+  worker.stopped_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+export async function markWorkerDead(
+  projectDir: string,
+  id: string,
+): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker) return;
+  if (worker.status === "stopped") return; // don't overwrite a clean stop
+  worker.status = "dead";
+  worker.stopped_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+/**
+ * Walk `workers/`, mark any running worker as dead if its
+ * `last_heartbeat_at` is older than `staleAfterSeconds`. Tasks/schedules
+ * they held are reclaimed via the lockfile reapers driven by
+ * `isWorkerRunning`. Returns the ids that were just marked dead.
+ */
+export async function reapDeadWorkers(
+  projectDir: string,
+  staleAfterSeconds: number,
+): Promise<string[]> {
+  const ids = await listWorkerIds(projectDir);
+  const cutoff = Date.now() - staleAfterSeconds * 1000;
+  const reaped: string[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w || w.status !== "running") continue;
+    const heartbeatMs = Date.parse(w.last_heartbeat_at);
+    if (Number.isFinite(heartbeatMs) && heartbeatMs >= cutoff) continue;
+    w.status = "dead";
+    w.stopped_at = new Date().toISOString();
+    await writeWorker(projectDir, w);
+    reaped.push(id);
+  }
+  return reaped;
+}
+
+export async function isWorkerRunning(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  const w = await readWorker(projectDir, id);
+  return w?.status === "running";
+}
+
+/**
+ * Delete cleanly-stopped worker JSON files whose `stopped_at` is older
+ * than `afterSeconds`. Dead workers are preserved as forensic evidence.
+ */
+export async function pruneStoppedWorkers(
+  projectDir: string,
+  afterSeconds: number,
+): Promise<string[]> {
+  const ids = await listWorkerIds(projectDir);
+  const cutoff = Date.now() - afterSeconds * 1000;
+  const pruned: string[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w || w.status !== "stopped" || !w.stopped_at) continue;
+    const stoppedMs = Date.parse(w.stopped_at);
+    if (Number.isFinite(stoppedMs) && stoppedMs >= cutoff) continue;
+    try {
+      await unlink(workerFilePath(projectDir, id));
+      pruned.push(id);
+    } catch {
+      // ignore — concurrent delete is fine
+    }
+  }
+  return pruned;
+}
+
+export async function listWorkers(
+  projectDir: string,
+  filters?: {
+    status?: WorkerStatus;
+    limit?: number;
+    offset?: number;
+  },
+): Promise<Worker[]> {
+  const ids = await listWorkerIds(projectDir);
+  const out: Worker[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w) continue;
+    if (filters?.status && w.status !== filters.status) continue;
+    out.push(w);
+  }
+  out.sort((a, b) => (a.started_at < b.started_at ? 1 : -1));
+  const offset = filters?.offset ?? 0;
+  const limit = filters?.limit ?? out.length;
+  return out.slice(offset, offset + limit);
+}
+
+export async function getWorker(
+  projectDir: string,
+  id: string,
+): Promise<Worker | null> {
+  return readWorker(projectDir, id);
+}
+
+export async function deleteWorker(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  try {
+    await unlink(workerFilePath(projectDir, id));
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}
+
+async function listWorkerIds(projectDir: string): Promise<string[]> {
+  const dir = getWorkersDir(projectDir);
+  try {
+    const names = await readdir(dir);
+    return names
+      .filter((n) => n.endsWith(".json"))
+      .map((n) => n.slice(0, -".json".length));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return [];
+    throw err;
+  }
+}
+
+/**
+ * For tests / doctor: confirm a worker JSON file exists at the expected path.
+ */
+export async function workerFileExists(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  try {
+    await stat(workerFilePath(projectDir, id));
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}