botholomew 0.16.4 → 0.18.0

package/src/worker/index.ts

@@ -1,12 +1,10 @@
 import { hostname } from "node:os";
 import ansis from "ansis";
 import { loadConfig } from "../config/loader.ts";
-import { getDbPath } from "../constants.ts";
-import { withDb } from "../db/connection.ts";
-import { migrate } from "../db/schema.ts";
-import { uuidv7 } from "../db/uuid.ts";
-import { createMcpxClient } from "../mcpx/client.ts";
+import { createMcpxClient, resolveMcpxDir } from "../mcpx/client.ts";
+import { openMembot, resolveMembotDir } from "../mem/client.ts";
 import { logger } from "../utils/logger.ts";
+import { uuidv7 } from "../utils/uuid.ts";
 import { markWorkerStopped, registerWorker } from "../workers/store.ts";
 import { startHeartbeat, startReaper } from "./heartbeat.ts";
 import type { WorkerStreamCallbacks } from "./llm.ts";
@@ -89,12 +87,12 @@ export async function startWorker(
   const evalSchedules = options.evalSchedules ?? !taskId;
 
   const config = await loadConfig(projectDir);
-  const dbPath = getDbPath(projectDir);
+  const mem = openMembot(resolveMembotDir(projectDir, config));
+  // Surface init-time failures (bad config, locked DB) up front rather than
+  // letting the first tool call do it.
+  await mem.connect();
 
-  // One short-lived connection to apply migrations, then release the lock.
-  await withDb(dbPath, (conn) => migrate(conn));
-
-  const mcpxClient = await createMcpxClient(projectDir);
+  const mcpxClient = await createMcpxClient(resolveMcpxDir(projectDir, config));
   if (mcpxClient) {
     logger.info("MCPX client initialized with external tools");
   }
@@ -129,6 +127,7 @@ export async function startWorker(
     stopHeartbeat();
     stopReaper();
     await mcpxClient?.close();
+    await mem.close();
     try {
       await markWorkerStopped(projectDir, workerId);
     } catch (err) {
@@ -151,7 +150,7 @@ export async function startWorker(
       if (taskId) {
         await runSpecificTask({
           projectDir,
-          dbPath,
+          mem,
           config,
           workerId,
           taskId,
@@ -161,7 +160,7 @@ export async function startWorker(
       } else {
         await tick({
           projectDir,
-          dbPath,
+          mem,
           config,
           workerId,
           mcpxClient,
@@ -182,7 +181,7 @@ export async function startWorker(
       try {
         didWork = await tick({
           projectDir,
-          dbPath,
+          mem,
           config,
           workerId,
           mcpxClient,
@@ -208,5 +207,6 @@ export async function startWorker(
       logger.warn(`failed to mark worker stopped: ${err}`);
     }
     await mcpxClient?.close();
+    await mem.close();
   }
 }

package/src/worker/llm.ts

@@ -5,8 +5,8 @@ import type {
   ToolUseBlock,
 } from "@anthropic-ai/sdk/resources/messages";
 import type { McpxClient } from "@evantahler/mcpx";
+import type { MembotClient } from "membot";
 import type { BotholomewConfig } from "../config/schemas.ts";
-import { withDb } from "../db/connection.ts";
 import type { Task } from "../tasks/schema.ts";
 import { getTask } from "../tasks/store.ts";
 import { logInteraction } from "../threads/store.ts";
@@ -50,7 +50,7 @@ export async function runAgentLoop(input: {
   systemPrompt: string;
   task: Task;
   config: Required<BotholomewConfig>;
-  dbPath: string;
+  mem: MembotClient;
   threadId: string;
   projectDir: string;
   workerId?: string;
@@ -61,7 +61,7 @@ export async function runAgentLoop(input: {
     systemPrompt,
     task,
     config,
-    dbPath,
+    mem,
     threadId,
     projectDir,
     workerId,
@@ -205,7 +205,7 @@ export async function runAgentLoop(input: {
       toolUseBlocks.map(async (toolUse) => {
         const start = Date.now();
         const result = await executeToolCall(toolUse, {
-          dbPath,
+          mem,
           projectDir,
           config,
           mcpxClient: input.mcpxClient ?? null,
@@ -264,7 +264,7 @@ interface ToolCallResult {
 }
 
 interface ToolCallCtx {
-  dbPath: string;
+  mem: MembotClient;
   projectDir: string;
   config: Required<BotholomewConfig>;
   mcpxClient: McpxClient | null;
@@ -298,10 +298,8 @@ async function executeToolCall(
 
   let result: unknown;
   try {
-    result = await withDb(baseCtx.dbPath, (conn) => {
-      const ctx: ToolContext = { ...baseCtx, conn };
-      return tool.execute(parsed.data, ctx);
-    });
+    const ctx: ToolContext = baseCtx;
+    result = await tool.execute(parsed.data, ctx);
   } catch (err) {
     return {
       output: `Tool ${toolUse.name} threw an error: ${err}. You may retry with different parameters or try an alternative approach.`,

package/src/worker/prompt.ts

@@ -1,10 +1,22 @@
 import { readdir } from "node:fs/promises";
 import { join } from "node:path";
+import { SERVER_INSTRUCTIONS as MEMBOT_INSTRUCTIONS } from "membot";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { getPromptsDir } from "../constants.ts";
 import type { Task } from "../tasks/schema.ts";
 import { parsePromptFile } from "../utils/frontmatter.ts";
 
+/**
+ * Section header rendered above membot's upstream {@link MEMBOT_INSTRUCTIONS}
+ * blob in every system prompt. Pulling the body verbatim from the SDK keeps
+ * the agent's mental model of `membot_*` tools aligned with whatever the
+ * pinned membot version ships, with no per-bump prose edits on our side.
+ */
+export const MEMBOT_PROMPT_SECTION = `## Knowledge store (membot)
+
+${MEMBOT_INSTRUCTIONS}
+`;
+
 const pkg = await Bun.file(
   new URL("../../package.json", import.meta.url),
 ).json();
@@ -103,7 +115,6 @@ User: ${process.env.USER || process.env.USERNAME || "unknown"}
 export async function buildSystemPrompt(
   projectDir: string,
   task?: Task,
-  dbPath?: string,
   _config?: Required<BotholomewConfig>,
   options?: { hasMcpTools?: boolean },
 ): Promise<string> {
@@ -115,11 +126,10 @@ export async function buildSystemPrompt(
 
   prompt += await loadPersistentContext(projectDir, taskKeywords);
 
-  // The agent finds task-relevant content via the `search` tool on demand
-  // rather than having chunks pre-stuffed into the system prompt — keeps the
-  // prompt small and lets the model decide what it actually needs to read.
+  // The agent finds task-relevant content via the `membot_search` tool on
+  // demand rather than having chunks pre-stuffed into the system prompt —
+  // keeps the prompt small and lets the model decide what to read.
   void task;
-  void dbPath;
   void _config;
 
   prompt += `## Instructions
@@ -128,26 +138,28 @@ You are Botholomew, a wise-owl worker that works through tasks. Use available to
 When calling complete_task, write a summary that captures your key findings, decisions, and outputs. This summary becomes the task's output and is provided to any downstream tasks that depend on this one. Include specific results (data, names, paths, conclusions) rather than vague descriptions of what you did — downstream tasks will rely on this information to do their work.
 `;
 
+  prompt += `\n${MEMBOT_PROMPT_SECTION}`;
+
   if (options?.hasMcpTools) {
     prompt += `
 ## External Tools (MCP)
 
-### Local context first
+### Local knowledge store first
 
-**Before any MCP read, search local context.** Files in \`context/\` (Gmail dumps, GitHub fetches, URL ingests, prior agent outputs) are usually already there — refetching is slower, costs tokens, and risks rate limits.
+**Before any MCP read, search the membot knowledge store.** Prior ingests (Gmail dumps, GitHub fetches, URL captures, prior agent outputs) are usually already there — refetching is slower, costs tokens, and risks rate limits.
 
 Workflow for any "look up / find / read" intent:
 
-1. \`search\` (hybrid regexp + semantic) over \`context/\`, then \`context_read\` / \`context_tree\` to drill in.
-2. If freshness matters, call \`context_info\` and check the file's mtime. To re-pull stale content, write fresh into \`context/\` (\`pipe_to_context\` from an \`mcp_exec\` call is the typical path) rather than going to MCP for the whole document on every question.
+1. \`membot_search\` (hybrid semantic + BM25) over the store, then \`membot_read\` / \`membot_tree\` to drill in.
+2. If freshness matters, call \`membot_info\` and check the source mtime / refresh status. To re-pull stale content, call \`membot_refresh\` for URL-backed entries, or \`membot_pipe\` from an \`mcp_exec\` call for fresh captures.
 3. Only call \`mcp_exec\` for reads when the data is genuinely missing locally **or** must be real-time (e.g., "what's on my calendar right now").
 
-Writes always go through MCP — sending an email, creating an issue, posting to Slack. Don't search context first for those.
+Writes to external systems always go through MCP — sending an email, creating an issue, posting to Slack. Don't search membot first for those.
 
 Examples:
-- "What does doc X say?" → \`search\` first.
-- "Any new emails from Y?" → \`search\` for the sender under \`context/gmail/\` (or wherever you've been ingesting mail) before hitting Gmail MCP.
-- "Send an email to Y" → MCP write directly; no context lookup.
+- "What does doc X say?" → \`membot_search\` first.
+- "Any new emails from Y?" → \`membot_search\` for the sender's name before hitting Gmail MCP.
+- "Send an email to Y" → MCP write directly; no membot lookup.
 
 ### Calling MCP tools
 

package/src/worker/spawn.ts

@@ -1,8 +1,8 @@
 import { mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import { getConfigPath, getWorkerLogPath } from "../constants.ts";
-import { uuidv7 } from "../db/uuid.ts";
 import { logger } from "../utils/logger.ts";
+import { uuidv7 } from "../utils/uuid.ts";
 import { dateForId } from "../utils/v7-date.ts";
 import type { WorkerMode } from "./index.ts";
 

package/src/worker/tick.ts

@@ -1,4 +1,5 @@
 import type { McpxClient } from "@evantahler/mcpx";
+import type { MembotClient } from "membot";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import type { Task } from "../tasks/schema.ts";
 import {
@@ -18,7 +19,7 @@ import { processSchedules } from "./schedules.ts";
 
 export interface TickOptions {
   projectDir: string;
-  dbPath: string;
+  mem: MembotClient;
   config: Required<BotholomewConfig>;
   workerId: string;
   mcpxClient?: McpxClient | null;
@@ -34,7 +35,7 @@ export interface TickOptions {
 export async function tick(opts: TickOptions): Promise<boolean> {
   const {
     projectDir,
-    dbPath,
+    mem,
     config,
     workerId,
     mcpxClient,
@@ -75,7 +76,7 @@ export async function tick(opts: TickOptions): Promise<boolean> {
 
   await runClaimedTask({
     projectDir,
-    dbPath,
+    mem,
     config,
     workerId,
     mcpxClient,
@@ -94,7 +95,7 @@ export async function tick(opts: TickOptions): Promise<boolean> {
  */
 export async function runSpecificTask(opts: {
   projectDir: string;
-  dbPath: string;
+  mem: MembotClient;
   config: Required<BotholomewConfig>;
   workerId: string;
   taskId: string;
@@ -114,7 +115,7 @@ export async function runSpecificTask(opts: {
   }
   await runClaimedTask({
     projectDir: opts.projectDir,
-    dbPath: opts.dbPath,
+    mem: opts.mem,
     config: opts.config,
     workerId: opts.workerId,
     mcpxClient: opts.mcpxClient,
@@ -126,14 +127,14 @@ export async function runSpecificTask(opts: {
 
 async function runClaimedTask(opts: {
   projectDir: string;
-  dbPath: string;
+  mem: MembotClient;
   config: Required<BotholomewConfig>;
   workerId: string;
   mcpxClient?: McpxClient | null;
   callbacks?: WorkerStreamCallbacks;
   task: Task;
 }): Promise<void> {
-  const { projectDir, dbPath, config, workerId, mcpxClient, callbacks, task } =
+  const { projectDir, mem, config, workerId, mcpxClient, callbacks, task } =
     opts;
 
   logger.info(`Claimed task: ${task.name} (${task.id})`);
@@ -151,7 +152,7 @@ async function runClaimedTask(opts: {
 
   let systemPrompt: string;
   try {
-    systemPrompt = await buildSystemPrompt(projectDir, task, dbPath, config, {
+    systemPrompt = await buildSystemPrompt(projectDir, task, config, {
       hasMcpTools: mcpxClient != null,
     });
   } catch (err) {
@@ -171,7 +172,7 @@ async function runClaimedTask(opts: {
       systemPrompt,
       task,
       config,
-      dbPath,
+      mem,
       threadId,
       projectDir,
       workerId,

package/src/commands/db.ts

@@ -1,119 +0,0 @@
-import ansis from "ansis";
-import type { Command } from "commander";
-import { getDbPath } from "../constants.ts";
-import {
-  isPidAlive,
-  type ProbeResult,
-  probeAllTables,
-  repairDatabase,
-} from "../db/doctor.ts";
-import { logger } from "../utils/logger.ts";
-import { listWorkers, type Worker } from "../workers/store.ts";
-
-function statusBadge(status: ProbeResult["status"]): string {
-  switch (status) {
-    case "ok":
-      return ansis.green("ok");
-    case "empty":
-      return ansis.dim("empty");
-    case "missing":
-      return ansis.dim("missing");
-    case "corrupt":
-      return ansis.red.bold("corrupt");
-  }
-}
-
-function printResults(results: ProbeResult[]) {
-  const nameWidth = Math.max(...results.map((r) => r.table.length));
-  for (const r of results) {
-    const name = r.table.padEnd(nameWidth + 2);
-    const detail = r.message ? ansis.dim(`  ${r.message.slice(0, 200)}`) : "";
-    console.log(`  ${name}${statusBadge(r.status)}${detail}`);
-  }
-}
-
-export function registerDbCommand(program: Command) {
-  const db = program
-    .command("db")
-    .description("Inspect and repair the project database");
-
-  db.command("doctor")
-    .description(
-      "Probe every table for primary-key index corruption and optionally repair via EXPORT/IMPORT",
-    )
-    .option(
-      "-r, --repair",
-      "Rebuild the database file from an export when corruption is detected",
-    )
-    .action((opts) => doctor(program, opts.repair === true));
-}
-
-async function doctor(program: Command, repair: boolean): Promise<void> {
-  const dir = program.opts().dir as string;
-  const dbPath = getDbPath(dir);
-
-  logger.info(`Probing tables in ${dbPath}`);
-  const results = await probeAllTables(dbPath);
-  printResults(results);
-
-  const corrupt = results.filter((r) => r.status === "corrupt");
-  if (corrupt.length === 0) {
-    logger.success("No corruption detected.");
-    return;
-  }
-
-  logger.error(
-    `${corrupt.length} table(s) have corrupted indexes: ${corrupt
-      .map((r) => r.table)
-      .join(", ")}`,
-  );
-
-  if (!repair) {
-    console.log("");
-    console.log(
-      ansis.yellow(
-        "Re-run with --repair to rebuild the database file (creates a timestamped backup).",
-      ),
-    );
-    process.exit(1);
-  }
-
-  // Repair requires exclusive access — refuse if any worker is actually
-  // running, otherwise the EXPORT would race with the worker's writes.
-  // Stale 'running' worker JSON files whose PID is dead are reported but
-  // don't block repair.
-  let running: Worker[];
-  try {
-    running = await listWorkers(dir, { status: "running" });
-  } catch {
-    running = [];
-  }
-  const live = running.filter((w) => isPidAlive(w.pid));
-  const stale = running.filter((w) => !isPidAlive(w.pid));
-  if (live.length > 0) {
-    logger.error(
-      `${live.length} worker(s) actually running. Stop them first: botholomew worker stop <id>`,
-    );
-    for (const w of live) {
-      logger.dim(`  ${w.id} (pid ${w.pid}, mode=${w.mode})`);
-    }
-    process.exit(1);
-  }
-  if (stale.length > 0) {
-    logger.warn(
-      `${stale.length} worker row(s) marked 'running' but PID is dead — proceeding (rows will be carried through repair, then reapable):`,
-    );
-    for (const w of stale) {
-      logger.dim(`  ${w.id} (pid ${w.pid}, mode=${w.mode})`);
-    }
-  }
-
-  logger.phase("repair", "EXPORT DATABASE → swap files → IMPORT DATABASE");
-  const result = await repairDatabase(dbPath);
-  logger.success(
-    `Repaired in ${result.durationMs}ms. Backup: ${result.backupDbPath}`,
-  );
-  logger.dim(
-    "  Re-run `botholomew db doctor` to confirm. Delete the backup once you're sure.",
-  );
-}

package/src/commands/with-db.ts

@@ -1,22 +0,0 @@
-import type { Command } from "commander";
-import { getDbPath } from "../constants.ts";
-import type { DbConnection } from "../db/connection.ts";
-import { withDb as coreWithDb } from "../db/connection.ts";
-import { migrate } from "../db/schema.ts";
-
-/**
- * Open a migrated DB connection from the CLI --dir flag, run the callback,
- * and guarantee the connection is closed afterward. Retries on lock
- * conflicts so CLI invocations cooperate with running workers or chat.
- */
-export async function withDb<T>(
-  program: Command,
-  fn: (conn: DbConnection, dir: string) => Promise<T>,
-): Promise<T> {
-  const dir = program.opts().dir;
-  const dbPath = getDbPath(dir);
-  return coreWithDb(dbPath, async (conn) => {
-    await migrate(conn);
-    return fn(conn, dir);
-  });
-}