botholomew 0.12.3 → 0.13.0

package/src/commands/worker.ts

@@ -1,6 +1,7 @@
 import ansis from "ansis";
 import type { Command } from "commander";
 import { loadConfig } from "../config/loader.ts";
+import { logger } from "../utils/logger.ts";
 import {
   getWorker,
   listWorkers,
@@ -10,11 +11,10 @@ import {
   reapDeadWorkers,
   WORKER_STATUSES,
   type Worker,
-} from "../db/workers.ts";
-import { logger } from "../utils/logger.ts";
-import { withDb } from "./with-db.ts";
+} from "../workers/store.ts";
 
-function formatAge(from: Date, to = new Date()): string {
+function formatAge(fromIso: string, to = new Date()): string {
+  const from = new Date(fromIso);
   const secs = Math.max(0, Math.floor((to.getTime() - from.getTime()) / 1000));
   if (secs < 60) return `${secs}s ago`;
   const mins = Math.floor(secs / 60);
@@ -40,11 +40,11 @@ function printWorker(w: Worker) {
   const short = w.id.slice(0, 8);
   const lines = [
     `${ansis.bold(short)}  pid=${w.pid}  mode=${w.mode}  ${statusColor(w.status)}  host=${w.hostname}`,
-    `  started: ${w.started_at.toISOString()} (${formatAge(w.started_at)})`,
-    `  heartbeat: ${w.last_heartbeat_at.toISOString()} (${formatAge(w.last_heartbeat_at)})`,
+    `  started: ${w.started_at} (${formatAge(w.started_at)})`,
+    `  heartbeat: ${w.last_heartbeat_at} (${formatAge(w.last_heartbeat_at)})`,
   ];
   if (w.task_id) lines.push(`  task: ${w.task_id}`);
-  if (w.stopped_at) lines.push(`  stopped: ${w.stopped_at.toISOString()}`);
+  if (w.stopped_at) lines.push(`  stopped: ${w.stopped_at}`);
   console.log(lines.join("\n"));
 }
 
@@ -105,7 +105,7 @@ export function registerWorkerCommand(program: Command) {
 
   worker
     .command("list")
-    .description("List workers registered in this project's database")
+    .description("List workers (one JSON record per file under workers/)")
     .option(
       "-s, --status <status>",
       `filter by status (${WORKER_STATUSES.join("|")})`,
@@ -113,106 +113,106 @@ export function registerWorkerCommand(program: Command) {
     .option("-l, --limit <n>", "max number of workers", Number.parseInt)
     .option("-o, --offset <n>", "skip first N workers", Number.parseInt)
     .action(
-      (opts: { status?: Worker["status"]; limit?: number; offset?: number }) =>
-        withDb(program, async (conn) => {
-          if (opts.status && !WORKER_STATUSES.includes(opts.status)) {
-            logger.error(
-              `Unknown status: ${opts.status}. Use one of: ${WORKER_STATUSES.join(", ")}`,
-            );
-            process.exit(1);
-          }
-          const workers = await listWorkers(conn, {
-            status: opts.status,
-            limit: opts.limit,
-            offset: opts.offset,
-          });
-          if (workers.length === 0) {
-            logger.dim("No workers found.");
-            return;
-          }
-          for (const w of workers) {
-            printWorker(w);
-            console.log("");
-          }
-        }),
+      async (opts: {
+        status?: Worker["status"];
+        limit?: number;
+        offset?: number;
+      }) => {
+        if (opts.status && !WORKER_STATUSES.includes(opts.status)) {
+          logger.error(
+            `Unknown status: ${opts.status}. Use one of: ${WORKER_STATUSES.join(", ")}`,
+          );
+          process.exit(1);
+        }
+        const dir = program.opts().dir;
+        const workers = await listWorkers(dir, {
+          status: opts.status,
+          limit: opts.limit,
+          offset: opts.offset,
+        });
+        if (workers.length === 0) {
+          logger.dim("No workers found.");
+          return;
+        }
+        for (const w of workers) {
+          printWorker(w);
+          console.log("");
+        }
+      },
     );
 
   worker
     .command("status <id>")
     .description("Show details for a single worker")
-    .action((id: string) =>
-      withDb(program, async (conn) => {
-        const w = await getWorker(conn, id);
-        if (!w) {
-          logger.error(`No worker found with id ${id}.`);
-          process.exit(1);
-        }
-        printWorker(w);
-      }),
-    );
+    .action(async (id: string) => {
+      const dir = program.opts().dir;
+      const w = await getWorker(dir, id);
+      if (!w) {
+        logger.error(`No worker found with id ${id}.`);
+        process.exit(1);
+      }
+      printWorker(w);
+    });
 
   worker
     .command("stop <id>")
     .description("SIGTERM the worker's process (graceful) and mark stopped")
-    .action((id: string) =>
-      withDb(program, async (conn) => {
-        const w = await getWorker(conn, id);
-        if (!w) {
-          logger.error(`No worker found with id ${id}.`);
-          process.exit(1);
-        }
-        signalWorker(w, "SIGTERM");
-        await markWorkerStopped(conn, id);
-        logger.success(`Worker ${id} signaled (SIGTERM) and marked stopped.`);
-      }),
-    );
+    .action(async (id: string) => {
+      const dir = program.opts().dir;
+      const w = await getWorker(dir, id);
+      if (!w) {
+        logger.error(`No worker found with id ${id}.`);
+        process.exit(1);
+      }
+      signalWorker(w, "SIGTERM");
+      await markWorkerStopped(dir, id);
+      logger.success(`Worker ${id} signaled (SIGTERM) and marked stopped.`);
+    });
 
   worker
     .command("kill <id>")
     .description("SIGKILL the worker's process and mark dead")
-    .action((id: string) =>
-      withDb(program, async (conn) => {
-        const w = await getWorker(conn, id);
-        if (!w) {
-          logger.error(`No worker found with id ${id}.`);
-          process.exit(1);
-        }
-        signalWorker(w, "SIGKILL");
-        await markWorkerDead(conn, id);
-        logger.success(`Worker ${id} killed (SIGKILL) and marked dead.`);
-      }),
-    );
+    .action(async (id: string) => {
+      const dir = program.opts().dir;
+      const w = await getWorker(dir, id);
+      if (!w) {
+        logger.error(`No worker found with id ${id}.`);
+        process.exit(1);
+      }
+      signalWorker(w, "SIGKILL");
+      await markWorkerDead(dir, id);
+      logger.success(`Worker ${id} killed (SIGKILL) and marked dead.`);
+    });
 
   worker
     .command("reap")
     .description(
       "Mark stale workers dead (releasing their tasks/schedule claims) and prune cleanly-stopped workers older than the retention window",
     )
-    .action(() =>
-      withDb(program, async (conn, dir) => {
-        const config = await loadConfig(dir);
-        const reaped = await reapDeadWorkers(
-          conn,
-          config.worker_dead_after_seconds,
+    .action(async () => {
+      const dir = program.opts().dir;
+      const config = await loadConfig(dir);
+      const reaped = await reapDeadWorkers(
+        dir,
+        config.worker_dead_after_seconds,
+      );
+      if (reaped.length === 0) {
+        logger.dim("No stale workers to reap.");
+      } else {
+        logger.success(
+          `Reaped ${reaped.length} worker(s): ${reaped.join(", ")}`,
         );
-        if (reaped.length === 0) {
-          logger.dim("No stale workers to reap.");
-        } else {
-          logger.success(
-            `Reaped ${reaped.length} worker(s): ${reaped.join(", ")}`,
-          );
-        }
-        const pruned = await pruneStoppedWorkers(
-          conn,
-          config.worker_stopped_retention_seconds,
+      }
+      const pruned = await pruneStoppedWorkers(
+        dir,
+        config.worker_stopped_retention_seconds,
+      );
+      if (pruned.length > 0) {
+        logger.success(
+          `Pruned ${pruned.length} stopped worker(s) older than retention window.`,
         );
-        if (pruned.length > 0) {
-          logger.success(
-            `Pruned ${pruned.length} stopped worker(s) older than retention window.`,
-          );
-        }
-      }),
-    );
+      }
+    });
 }
 
 function signalWorker(w: Worker, signal: "SIGTERM" | "SIGKILL"): void {
@@ -225,7 +225,7 @@ function signalWorker(w: Worker, signal: "SIGTERM" | "SIGKILL"): void {
     process.kill(w.pid, signal);
   } catch (err) {
     logger.warn(
-      `Could not send ${signal} to PID ${w.pid}: ${err instanceof Error ? err.message : err}. Marking DB state only.`,
+      `Could not send ${signal} to PID ${w.pid}: ${err instanceof Error ? err.message : err}. Marking state only.`,
     );
   }
 }

package/src/constants.ts

@@ -1,7 +1,26 @@
 import { homedir } from "node:os";
 import { join } from "node:path";
 
-export const BOTHOLOMEW_DIR = ".botholomew";
+/**
+ * Project layout (rooted at `projectDir`, typically the user's cwd):
+ *
+ *   <projectDir>/
+ *     config/config.json
+ *     prompts/{soul,beliefs,goals,capabilities}.md
+ *     skills/*.md
+ *     mcpx/servers.json
+ *     models/                          embedding model cache
+ *     context/                         user-curated knowledge tree
+ *     tasks/<id>.md                    tasks (status in frontmatter)
+ *     tasks/.locks/<id>.lock           O_EXCL claim files
+ *     schedules/<id>.md                schedules
+ *     schedules/.locks/<id>.lock
+ *     threads/<YYYY-MM-DD>/<id>.csv    conversation history
+ *     workers/<id>.json                pidfile + heartbeat
+ *     logs/                            worker logs
+ *     index.duckdb                     search index (rebuildable from disk)
+ */
+
 export const HOME_CONFIG_DIR = join(homedir(), ".botholomew");
 
 export const ENV = {
@@ -12,47 +31,105 @@ export const DEFAULTS = {
   UPDATE_CHECK_INTERVAL_MS: 24 * 60 * 60 * 1000, // 24 hours
   UPDATE_CHECK_TIMEOUT_MS: 5_000,
 } as const;
-export const DB_FILENAME = "data.duckdb";
-export const LOGS_DIR = "logs";
+
+export const INDEX_DB_FILENAME = "index.duckdb";
+export const CONFIG_DIR = "config";
 export const CONFIG_FILENAME = "config.json";
+export const PROMPTS_DIR = "prompts";
+export const SKILLS_DIR = "skills";
 export const MCPX_DIR = "mcpx";
 export const MODELS_DIR = "models";
-export const SKILLS_DIR = "skills";
+export const CONTEXT_DIR = "context";
+export const TASKS_DIR = "tasks";
+export const SCHEDULES_DIR = "schedules";
+export const LOCKS_SUBDIR = ".locks";
+export const LOGS_DIR = "logs";
+export const WORKERS_DIR = "workers";
+export const THREADS_DIR = "threads";
 export const MCPX_SERVERS_FILENAME = "servers.json";
 export const EMBEDDING_DIMENSION = 384;
 export const EMBEDDING_MODEL = "Xenova/bge-small-en-v1.5";
 
-export function getBotholomewDir(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR);
-}
+/**
+ * Top-level areas tools must never touch directly. Use as a safelist when
+ * validating tool path arguments — most file/dir tools pin to CONTEXT_DIR.
+ */
+export const PROTECTED_AREAS: ReadonlySet<string> = new Set([
+  MODELS_DIR,
+  LOGS_DIR,
+  `${TASKS_DIR}/${LOCKS_SUBDIR}`,
+  `${SCHEDULES_DIR}/${LOCKS_SUBDIR}`,
+]);
 
 export function getDbPath(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, DB_FILENAME);
+  return join(projectDir, INDEX_DB_FILENAME);
 }
 
 export function getWorkerLogsDir(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, LOGS_DIR);
+  return join(projectDir, LOGS_DIR);
 }
 
-export function getWorkerLogPath(projectDir: string, workerId: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, LOGS_DIR, `${workerId}.log`);
+/**
+ * Per-worker log file at `<logs>/<YYYY-MM-DD>/<workerId>.log`. The date
+ * subdir keeps the logs directory browsable as workers accumulate.
+ * Callers derive `date` from the worker's uuidv7 timestamp via
+ * `src/utils/v7-date.ts::dateForId` so the path is a pure function of
+ * the id and survives a process restart.
+ */
+export function getWorkerLogPath(
+  projectDir: string,
+  workerId: string,
+  date: string,
+): string {
+  return join(projectDir, LOGS_DIR, date, `${workerId}.log`);
 }
 
 export function getConfigPath(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, CONFIG_FILENAME);
+  return join(projectDir, CONFIG_DIR, CONFIG_FILENAME);
 }
 
 export function getMcpxDir(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, MCPX_DIR);
+  return join(projectDir, MCPX_DIR);
 }
 
 export function getModelsDir(projectDir: string): string {
   return (
-    process.env.BOTHOLOMEW_MODELS_DIR_OVERRIDE ??
-    join(projectDir, BOTHOLOMEW_DIR, MODELS_DIR)
+    process.env.BOTHOLOMEW_MODELS_DIR_OVERRIDE ?? join(projectDir, MODELS_DIR)
   );
 }
 
 export function getSkillsDir(projectDir: string): string {
-  return join(projectDir, BOTHOLOMEW_DIR, SKILLS_DIR);
+  return join(projectDir, SKILLS_DIR);
+}
+
+export function getPromptsDir(projectDir: string): string {
+  return join(projectDir, PROMPTS_DIR);
+}
+
+export function getContextDir(projectDir: string): string {
+  return join(projectDir, CONTEXT_DIR);
+}
+
+export function getTasksDir(projectDir: string): string {
+  return join(projectDir, TASKS_DIR);
+}
+
+export function getTasksLockDir(projectDir: string): string {
+  return join(projectDir, TASKS_DIR, LOCKS_SUBDIR);
+}
+
+export function getSchedulesDir(projectDir: string): string {
+  return join(projectDir, SCHEDULES_DIR);
+}
+
+export function getSchedulesLockDir(projectDir: string): string {
+  return join(projectDir, SCHEDULES_DIR, LOCKS_SUBDIR);
+}
+
+export function getWorkersDir(projectDir: string): string {
+  return join(projectDir, WORKERS_DIR);
+}
+
+export function getThreadsDir(projectDir: string): string {
+  return join(projectDir, THREADS_DIR);
 }

package/src/context/capabilities.ts

@@ -2,7 +2,7 @@ import { join } from "node:path";
 import Anthropic from "@anthropic-ai/sdk";
 import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { getBotholomewDir } from "../constants.ts";
+import { getPromptsDir } from "../constants.ts";
 import { getAllTools, type ToolDefinition } from "../tools/tool.ts";
 import {
   type ContextFileMeta,
@@ -151,7 +151,7 @@ const SUMMARIZE_TOOL = {
       internal_themes: {
         type: "array",
         description:
-          "Themes covering the agent's built-in tools (task queue, virtual filesystem, search, threads, MCPX meta-tools, workers, self-reflection, etc.).",
+          "Themes covering the agent's built-in tools (task queue, files & sandbox, search, threads, MCPX meta-tools, workers, self-reflection, etc.).",
         items: {
           type: "object",
           properties: {
@@ -241,12 +241,12 @@ Rules:
 - Do NOT list specific tool names. The agent discovers exact names via the MCPX meta-tools (mcp_search, mcp_list_tools, mcp_info) when it actually needs to invoke one.
 - Group tools into natural themes.
 - For MCPX tools, one theme usually = one external service (Gmail, Google Calendar, GitHub, Linear, Slack, Google Docs, Google Drive, Google Sheets, Apple Notes, etc.). Split a single server into multiple themes when it clearly exposes distinct services.
-- For internal tools, use coarse buckets aligned with the provided groups (task management, virtual filesystem, search, threads, MCPX meta-tools, workers, self-reflection, capabilities). Merge overlapping groups if natural.
+- For internal tools, use coarse buckets aligned with the provided groups (task management, files & sandbox, search, threads, MCPX meta-tools, workers, self-reflection, capabilities). Merge overlapping groups if natural.
 - Each summary is ONE sentence with concrete action verbs. Present-tense imperative, no preamble.
 
 GOOD examples:
   "Gmail — read, send, draft, search, and reply to emails; manage labels and threads"
-  "Virtual filesystem — read, write, edit, move, copy, delete, and navigate items in the agent's persistent memory store"
+  "Files & sandbox — read, write, edit, move, copy, delete, and navigate files under the agent's context/ tree"
   "GitHub — read and write repositories, branches, files, issues, pull requests, reviews, and labels"
 
 BAD examples (do not produce):
@@ -368,8 +368,8 @@ function renderFallback(inv: RawInventory, now: Date): string {
     schedule:
       "create and list recurring schedules that automatically generate tasks",
     context:
-      "read, write, edit, move, copy, delete, and navigate items in the agent's persistent memory store; update beliefs and goals; read large tool results",
-    search: "keyword and semantic search over the virtual filesystem",
+      "read, write, edit, move, copy, delete, and navigate files in the agent's context/ tree; update beliefs and goals; read large tool results",
+    search: "keyword, semantic, and regexp search over files in context/",
     thread: "list and view past conversation threads and tool interactions",
     mcp: "search, list, inspect, and execute tools exposed by configured MCPX servers",
     worker: "spawn background workers to run tasks asynchronously",
@@ -461,9 +461,9 @@ export interface WriteResult {
 }
 
 /**
- * Regenerate and write `.botholomew/capabilities.md`. Preserves any existing
- * frontmatter (so a human-edited `loading:` flag survives). On first write
- * the default frontmatter is `loading: always`, `agent-modification: true`.
+ * Regenerate and write `prompts/capabilities.md`. Preserves any
+ * existing frontmatter (so a human-edited `loading:` flag survives). On first
+ * write the default frontmatter is `loading: always`, `agent-modification: true`.
  */
 export async function writeCapabilitiesFile(
   projectDir: string,
@@ -471,7 +471,7 @@ export async function writeCapabilitiesFile(
   config: Required<BotholomewConfig>,
   onPhase?: ProgressCallback,
 ): Promise<WriteResult> {
-  const filePath = join(getBotholomewDir(projectDir), CAPABILITIES_FILENAME);
+  const filePath = join(getPromptsDir(projectDir), CAPABILITIES_FILENAME);
   const file = Bun.file(filePath);
 
   let meta: ContextFileMeta = {

package/src/context/fetcher.ts

@@ -15,7 +15,6 @@ import { mcpSearchTool } from "../tools/mcp/search.ts";
 import type { ToolContext } from "../tools/tool.ts";
 import { type AnyToolDefinition, toAnthropicTool } from "../tools/tool.ts";
 import { logger } from "../utils/logger.ts";
-import { detectDriveFromUrl } from "./drives.ts";
 import { stripHtmlTags } from "./url-utils.ts";
 
 const MAX_CONTENT_BYTES = 500_000;
@@ -29,8 +28,12 @@ export interface FetchedContent {
   content: string;
   mimeType: string;
   sourceUrl: string;
-  drive: string;
-  path: string;
+  /**
+   * MCP server that produced the content (e.g. "google-docs", "github",
+   * "firecrawl"), or null when we fell back to a plain HTTP fetch. Useful
+   * for `bothy context import` to pick a default destination subdirectory.
+   */
+  source: string | null;
 }
 
 export class FetchFailureError extends Error {
@@ -138,7 +141,7 @@ export async function fetchUrl(
 ): Promise<FetchedContent> {
   if (!config.anthropic_api_key) {
     throw new Error(
-      "Anthropic API key is required for URL fetching. Set ANTHROPIC_API_KEY or configure it in .botholomew/config.json",
+      "Anthropic API key is required for URL fetching. Set ANTHROPIC_API_KEY or configure it in config/config.json",
     );
   }
 
@@ -293,14 +296,12 @@ async function runFetcherLoop(
       logger.dim(
         `  turn ${turn + 1}: accept_content: "${input.title}" (${cached.content.length} chars, ${mimeType}, from ${cached.server}/${cached.tool})`,
       );
-      const { drive, path } = detectDriveFromUrl(url, cached.server);
       return {
         title: input.title,
         content: cached.content.slice(0, MAX_CONTENT_BYTES),
         mimeType,
         sourceUrl: url,
-        drive,
-        path,
+        source: cached.server,
       };
     }
 
@@ -435,13 +436,11 @@ export async function httpFallback(url: string): Promise<FetchedContent> {
     ? "text/markdown"
     : contentType.split(";")[0] || "text/plain";
 
-  const { drive, path } = detectDriveFromUrl(url);
   return {
     title,
     content: text,
     mimeType,
     sourceUrl: url,
-    drive,
-    path,
+    source: null,
   };
 }