botholomew 0.12.3 → 0.13.0

package/src/worker/spawn.ts

@@ -1,12 +1,9 @@
 import { mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import {
-  getBotholomewDir,
-  getWorkerLogPath,
-  getWorkerLogsDir,
-} from "../constants.ts";
+import { dirname } from "node:path";
+import { getConfigPath, getWorkerLogPath } from "../constants.ts";
 import { uuidv7 } from "../db/uuid.ts";
 import { logger } from "../utils/logger.ts";
+import { dateForId } from "../utils/v7-date.ts";
 import type { WorkerMode } from "./index.ts";
 
 export interface SpawnWorkerOptions {
@@ -26,16 +23,19 @@ export async function spawnWorker(
   projectDir: string,
   options: SpawnWorkerOptions = {},
 ): Promise<{ pid: number; workerId: string; logPath: string }> {
-  const dotDir = getBotholomewDir(projectDir);
-  const dirExists = await Bun.file(join(dotDir, "config.json")).exists();
-  if (!dirExists) {
+  const configPath = getConfigPath(projectDir);
+  const initialized = await Bun.file(configPath).exists();
+  if (!initialized) {
     logger.error("Project not initialized. Run 'botholomew init' first.");
     process.exit(1);
   }
 
   const workerId = uuidv7();
-  await mkdir(getWorkerLogsDir(projectDir), { recursive: true });
-  const logPath = getWorkerLogPath(projectDir, workerId);
+  // Per-worker log path is derived from the worker id's UTC date so the
+  // logs/ tree stays browsable as workers accumulate. Mirrors the threads
+  // layout under <projectDir>/threads/.
+  const logPath = getWorkerLogPath(projectDir, workerId, dateForId(workerId));
+  await mkdir(dirname(logPath), { recursive: true });
   const logFile = Bun.file(logPath);
 
   const workerScript = new URL("./run.ts", import.meta.url).pathname;

package/src/worker/tick.ts

@@ -1,14 +1,14 @@
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { withDb } from "../db/connection.ts";
+import type { Task } from "../tasks/schema.ts";
 import {
   claimNextTask,
   claimSpecificTask,
+  releaseTaskLock,
   resetStaleTasks,
-  type Task,
   updateTaskStatus,
-} from "../db/tasks.ts";
-import { createThread, endThread, logInteraction } from "../db/threads.ts";
+} from "../tasks/store.ts";
+import { createThread, endThread, logInteraction } from "../threads/store.ts";
 import { logger } from "../utils/logger.ts";
 import { generateThreadTitle } from "../utils/title.ts";
 import type { WorkerStreamCallbacks } from "./llm.ts";
@@ -46,9 +46,9 @@ export async function tick(opts: TickOptions): Promise<boolean> {
   const tickStart = Date.now();
   logger.phase("tick-start", `#${tickNum}`);
 
-  // Reset stale tasks stuck in in_progress
-  const resetIds = await withDb(dbPath, (conn) =>
-    resetStaleTasks(conn, config.max_tick_duration_seconds * 3),
+  const resetIds = await resetStaleTasks(
+    projectDir,
+    config.max_tick_duration_seconds * 3,
   );
   if (resetIds.length > 0) {
     logger.warn(
@@ -58,15 +58,14 @@ export async function tick(opts: TickOptions): Promise<boolean> {
 
   if (evalSchedules) {
     try {
-      await processSchedules(dbPath, config, workerId);
+      await processSchedules(projectDir, config, workerId);
     } catch (err) {
       logger.error(`Schedule processing failed: ${err}`);
     }
   }
 
-  // Claim a task
   logger.phase("claiming-task");
-  const task = await withDb(dbPath, (conn) => claimNextTask(conn, workerId));
+  const task = await claimNextTask(projectDir, workerId);
   if (!task) {
     logger.info("No task claimed (queue empty or all blocked)");
     const elapsed = ((Date.now() - tickStart) / 1000).toFixed(1);
@@ -90,8 +89,7 @@ export async function tick(opts: TickOptions): Promise<boolean> {
 
 /**
  * Claim and run a single, explicitly-named task. Returns true if the task
- * was claimed and processed, false if it wasn't eligible (already claimed,
- * not pending, or doesn't exist).
+ * was claimed and processed, false if it wasn't eligible.
  */
 export async function runSpecificTask(opts: {
   projectDir: string;
@@ -102,8 +100,10 @@ export async function runSpecificTask(opts: {
   mcpxClient?: McpxClient | null;
   callbacks?: WorkerStreamCallbacks;
 }): Promise<boolean> {
-  const task = await withDb(opts.dbPath, (conn) =>
-    claimSpecificTask(conn, opts.taskId, opts.workerId),
+  const task = await claimSpecificTask(
+    opts.projectDir,
+    opts.taskId,
+    opts.workerId,
   );
   if (!task) {
     logger.warn(
@@ -138,8 +138,11 @@ async function runClaimedTask(opts: {
   }
   callbacks?.onTaskStart(task);
 
-  const threadId = await withDb(dbPath, (conn) =>
-    createThread(conn, "worker_tick", task.id, `Working: ${task.name}`),
+  const threadId = await createThread(
+    projectDir,
+    "worker_tick",
+    task.id,
+    `Working: ${task.name}`,
   );
 
   const systemPrompt = await buildSystemPrompt(
@@ -163,47 +166,40 @@ async function runClaimedTask(opts: {
     });
 
     const isComplete = result.status === "complete";
-    await withDb(dbPath, (conn) =>
-      updateTaskStatus(
-        conn,
-        task.id,
-        result.status,
-        isComplete ? null : result.reason,
-        isComplete ? result.reason : null,
-      ),
+    await updateTaskStatus(
+      projectDir,
+      task.id,
+      result.status,
+      isComplete ? null : result.reason,
+      isComplete ? result.reason : null,
     );
 
-    await withDb(dbPath, (conn) =>
-      logInteraction(conn, threadId, {
-        role: "system",
-        kind: "status_change",
-        content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
-      }),
-    );
+    await logInteraction(projectDir, threadId, {
+      role: "system",
+      kind: "status_change",
+      content: `Task ${task.id} -> ${result.status}${result.reason ? `: ${result.reason}` : ""}`,
+    });
 
     logger.info(`Task ${task.id} -> ${result.status}`);
 
     void generateThreadTitle(
       config,
-      dbPath,
+      projectDir,
       threadId,
       `Task: ${task.name}\nDescription: ${task.description}\nOutcome: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`,
     );
   } catch (err) {
-    await withDb(dbPath, (conn) =>
-      updateTaskStatus(conn, task.id, "failed", String(err), null),
-    );
+    await updateTaskStatus(projectDir, task.id, "failed", String(err), null);
 
-    await withDb(dbPath, (conn) =>
-      logInteraction(conn, threadId, {
-        role: "system",
-        kind: "status_change",
-        content: `Task ${task.id} failed: ${err}`,
-      }),
-    );
+    await logInteraction(projectDir, threadId, {
+      role: "system",
+      kind: "status_change",
+      content: `Task ${task.id} failed: ${err}`,
+    });
 
     logger.error(`Task ${task.id} failed: ${err}`);
   } finally {
-    await withDb(dbPath, (conn) => endThread(conn, threadId));
+    await releaseTaskLock(projectDir, task.id);
+    await endThread(projectDir, threadId);
   }
 }

package/src/workers/store.ts

@@ -0,0 +1,247 @@
+import { readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { getWorkersDir } from "../constants.ts";
+import { atomicWrite, readWithMtime } from "../fs/atomic.ts";
+
+export const WORKER_MODES = ["persist", "once"] as const;
+export const WORKER_STATUSES = ["running", "stopped", "dead"] as const;
+
+export type WorkerMode = (typeof WORKER_MODES)[number];
+export type WorkerStatus = (typeof WORKER_STATUSES)[number];
+
+/**
+ * Persistent worker record. One JSON file per worker at
+ * `<projectDir>/workers/<id>.json`. Heartbeats rewrite the file
+ * atomically (write-to-tmp + rename); the file's existence is the pidfile,
+ * `last_heartbeat_at` inside is the liveness signal.
+ */
+export interface Worker {
+  id: string;
+  pid: number;
+  hostname: string;
+  mode: WorkerMode;
+  task_id: string | null;
+  status: WorkerStatus;
+  started_at: string;
+  last_heartbeat_at: string;
+  stopped_at: string | null;
+  log_path: string | null;
+}
+
+function workerFilePath(projectDir: string, id: string): string {
+  return join(getWorkersDir(projectDir), `${id}.json`);
+}
+
+async function readWorker(
+  projectDir: string,
+  id: string,
+): Promise<Worker | null> {
+  const file = await readWithMtime(workerFilePath(projectDir, id));
+  if (!file) return null;
+  try {
+    return JSON.parse(file.content) as Worker;
+  } catch {
+    return null;
+  }
+}
+
+async function writeWorker(projectDir: string, worker: Worker): Promise<void> {
+  await atomicWrite(
+    workerFilePath(projectDir, worker.id),
+    `${JSON.stringify(worker, null, 2)}\n`,
+  );
+}
+
+export async function registerWorker(
+  projectDir: string,
+  params: {
+    id: string;
+    pid: number;
+    hostname: string;
+    mode: WorkerMode;
+    taskId?: string | null;
+    logPath?: string | null;
+  },
+): Promise<Worker> {
+  const now = new Date().toISOString();
+  const worker: Worker = {
+    id: params.id,
+    pid: params.pid,
+    hostname: params.hostname,
+    mode: params.mode,
+    task_id: params.taskId ?? null,
+    status: "running",
+    started_at: now,
+    last_heartbeat_at: now,
+    stopped_at: null,
+    log_path: params.logPath ?? null,
+  };
+  await writeWorker(projectDir, worker);
+  return worker;
+}
+
+/**
+ * Update last_heartbeat_at on a running worker. No-op for stopped/dead
+ * workers so a misbehaving heartbeat doesn't resurrect a worker the reaper
+ * has retired.
+ */
+export async function heartbeat(projectDir: string, id: string): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker || worker.status !== "running") return;
+  worker.last_heartbeat_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+export async function markWorkerStopped(
+  projectDir: string,
+  id: string,
+): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker || worker.status !== "running") return;
+  worker.status = "stopped";
+  worker.stopped_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+export async function markWorkerDead(
+  projectDir: string,
+  id: string,
+): Promise<void> {
+  const worker = await readWorker(projectDir, id);
+  if (!worker) return;
+  if (worker.status === "stopped") return; // don't overwrite a clean stop
+  worker.status = "dead";
+  worker.stopped_at = new Date().toISOString();
+  await writeWorker(projectDir, worker);
+}
+
+/**
+ * Walk `workers/`, mark any running worker as dead if its
+ * `last_heartbeat_at` is older than `staleAfterSeconds`. Tasks/schedules
+ * they held are reclaimed via the lockfile reapers driven by
+ * `isWorkerRunning`. Returns the ids that were just marked dead.
+ */
+export async function reapDeadWorkers(
+  projectDir: string,
+  staleAfterSeconds: number,
+): Promise<string[]> {
+  const ids = await listWorkerIds(projectDir);
+  const cutoff = Date.now() - staleAfterSeconds * 1000;
+  const reaped: string[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w || w.status !== "running") continue;
+    const heartbeatMs = Date.parse(w.last_heartbeat_at);
+    if (Number.isFinite(heartbeatMs) && heartbeatMs >= cutoff) continue;
+    w.status = "dead";
+    w.stopped_at = new Date().toISOString();
+    await writeWorker(projectDir, w);
+    reaped.push(id);
+  }
+  return reaped;
+}
+
+export async function isWorkerRunning(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  const w = await readWorker(projectDir, id);
+  return w?.status === "running";
+}
+
+/**
+ * Delete cleanly-stopped worker JSON files whose `stopped_at` is older
+ * than `afterSeconds`. Dead workers are preserved as forensic evidence.
+ */
+export async function pruneStoppedWorkers(
+  projectDir: string,
+  afterSeconds: number,
+): Promise<string[]> {
+  const ids = await listWorkerIds(projectDir);
+  const cutoff = Date.now() - afterSeconds * 1000;
+  const pruned: string[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w || w.status !== "stopped" || !w.stopped_at) continue;
+    const stoppedMs = Date.parse(w.stopped_at);
+    if (Number.isFinite(stoppedMs) && stoppedMs >= cutoff) continue;
+    try {
+      await unlink(workerFilePath(projectDir, id));
+      pruned.push(id);
+    } catch {
+      // ignore — concurrent delete is fine
+    }
+  }
+  return pruned;
+}
+
+export async function listWorkers(
+  projectDir: string,
+  filters?: {
+    status?: WorkerStatus;
+    limit?: number;
+    offset?: number;
+  },
+): Promise<Worker[]> {
+  const ids = await listWorkerIds(projectDir);
+  const out: Worker[] = [];
+  for (const id of ids) {
+    const w = await readWorker(projectDir, id);
+    if (!w) continue;
+    if (filters?.status && w.status !== filters.status) continue;
+    out.push(w);
+  }
+  out.sort((a, b) => (a.started_at < b.started_at ? 1 : -1));
+  const offset = filters?.offset ?? 0;
+  const limit = filters?.limit ?? out.length;
+  return out.slice(offset, offset + limit);
+}
+
+export async function getWorker(
+  projectDir: string,
+  id: string,
+): Promise<Worker | null> {
+  return readWorker(projectDir, id);
+}
+
+export async function deleteWorker(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  try {
+    await unlink(workerFilePath(projectDir, id));
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}
+
+async function listWorkerIds(projectDir: string): Promise<string[]> {
+  const dir = getWorkersDir(projectDir);
+  try {
+    const names = await readdir(dir);
+    return names
+      .filter((n) => n.endsWith(".json"))
+      .map((n) => n.slice(0, -".json".length));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return [];
+    throw err;
+  }
+}
+
+/**
+ * For tests / doctor: confirm a worker JSON file exists at the expected path.
+ */
+export async function workerFileExists(
+  projectDir: string,
+  id: string,
+): Promise<boolean> {
+  try {
+    await stat(workerFilePath(projectDir, id));
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}