botholomew 0.7.2 → 0.7.4

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "botholomew",
-  "version": "0.7.2",
-  "description": "An AI agent for knowledge work",
+  "version": "0.7.4",
+  "description": "Local, autonomous AI agent for knowledge work — works your task queue while you sleep.",
   "type": "module",
   "bin": {
     "botholomew": "./src/cli.ts"

package/src/chat/agent.ts

@@ -33,6 +33,8 @@ const CHAT_TOOL_NAMES = new Set([
   "list_tasks",
   "view_task",
   "context_search",
+  "context_info",
+  "context_refresh",
   "search_grep",
   "search_semantic",
   "list_threads",
@@ -103,7 +105,7 @@ export async function buildChatSystemPrompt(
     "You do NOT execute long-running work directly — enqueue tasks for the daemon instead using create_task.",
   );
   parts.push(
-    "Use the available tools to look up tasks, threads, schedules, and context when the user asks about them.",
+    "Use the available tools to look up tasks, threads, schedules, and context when the user asks about them. Context items can be looked up by virtual path or by UUID via `context_info` and refreshed via `context_refresh`.",
   );
   parts.push(
     "When multiple tool calls are independent of each other (i.e., one does not depend on the result of another), call them all in a single response. They will be executed in parallel, which is faster than calling them one at a time.",

package/src/commands/context.ts

@@ -14,6 +14,7 @@ import {
   prepareIngestion,
   storeIngestion,
 } from "../context/ingest.ts";
+import { refreshContextItems } from "../context/refresh.ts";
 import { isUrl, urlToContextPath } from "../context/url-utils.ts";
 import type { DbConnection } from "../db/connection.ts";
 import {
@@ -22,7 +23,6 @@ import {
   listContextItems,
   listContextItemsByPrefix,
   resolveContextItem,
-  updateContextItem,
   upsertContextItem,
 } from "../db/context.ts";
 import { getEmbeddingsForItem, hybridSearch } from "../db/embeddings.ts";
@@ -444,95 +444,55 @@ export function registerContextCommand(program: Command) {
         const hasUrls = sourced.some((i) => i.source_type === "url");
         const mcpxClient = hasUrls ? await createMcpxClient(dir) : null;
 
-        // Phase 1: Read files / fetch URLs, compare, and update DB
-        const spinner = createSpinner(
+        const refreshSpinner = createSpinner(
           `Refreshing 0/${sourced.length} items...`,
         ).start();
-        let updated = 0;
-        let unchanged = 0;
-        let missing = 0;
-        const toReembed: string[] = [];
-
-        for (const [idx, item] of sourced.entries()) {
-          spinner.update({
-            text: `Refreshing ${idx + 1}/${sourced.length} items...`,
-          });
-          try {
-            const sourcePath = item.source_path as string;
-            let content: string;
-
-            if (item.source_type === "url") {
-              const fetched = await fetchUrl(sourcePath, config, mcpxClient);
-              content = fetched.content;
-            } else {
-              const bunFile = Bun.file(sourcePath);
-              if (!(await bunFile.exists())) {
-                missing++;
-                logger.warn(`  Missing: ${item.source_path}`);
-                continue;
-              }
-              content = await bunFile.text();
-            }
+        const embedSpinner = createSpinner("Embedding 0 item(s)...");
+
+        const result = await refreshContextItems(
+          conn,
+          sourced,
+          config,
+          mcpxClient,
+          {
+            onItemProgress: (done, total) => {
+              refreshSpinner.update({
+                text: `Refreshing ${done}/${total} items...`,
+              });
+            },
+            onEmbedProgress: (done, total) => {
+              if (done === 1) embedSpinner.start();
+              embedSpinner.update({
+                text: `Embedding ${done}/${total} item(s)...`,
+              });
+            },
+          },
+        );
 
-            if (content === item.content) {
-              unchanged++;
-              continue;
-            }
-            await updateContextItem(conn, item.id, { content });
-            updated++;
-            toReembed.push(item.id);
-          } catch (err) {
-            logger.warn(`  Error refreshing ${item.source_path}: ${err}`);
-          }
-        }
-        spinner.success({
-          text: `Checked ${sourced.length} item(s): ${updated} updated, ${unchanged} unchanged, ${missing} missing.`,
+        refreshSpinner.success({
+          text: `Checked ${result.checked} item(s): ${result.updated} updated, ${result.unchanged} unchanged, ${result.missing} missing.`,
         });
 
-        // Phase 2: Re-embed changed items
-        if (toReembed.length === 0 || !config.openai_api_key) {
-          if (toReembed.length > 0 && !config.openai_api_key) {
-            logger.dim("Skipping embeddings (no OpenAI API key configured).");
+        for (const item of result.items) {
+          if (item.status === "missing") {
+            logger.warn(`  Missing: ${item.source_path}`);
+          } else if (item.status === "error") {
+            logger.warn(
+              `  Error refreshing ${item.source_path}: ${item.error}`,
+            );
           }
-          return;
         }
 
-        const CONCURRENCY = 10;
-        let completed = 0;
-        const embedSpinner = createSpinner(
-          `Embedding 0/${toReembed.length} item(s)...`,
-        ).start();
-
-        const prepared: PreparedIngestion[] = [];
-        for (let i = 0; i < toReembed.length; i += CONCURRENCY) {
-          const batch = toReembed.slice(i, i + CONCURRENCY);
-          const results = await Promise.all(
-            batch.map(async (id) => {
-              const result = await prepareIngestion(conn, id, config);
-              completed++;
-              embedSpinner.update({
-                text: `Embedding ${completed}/${toReembed.length} item(s)...`,
-              });
-              return result;
-            }),
+        if (result.reembedded > 0) {
+          embedSpinner.success({
+            text: `Embedded ${result.reembedded} item(s).`,
+          });
+          logger.success(
+            `Refreshed ${result.updated} item(s), ${result.chunks} chunk(s) re-indexed.`,
           );
-          for (const r of results) {
-            if (r) prepared.push(r);
-          }
+        } else if (result.embeddings_skipped) {
+          logger.dim("Skipping embeddings (no OpenAI API key configured).");
         }
-        embedSpinner.success({
-          text: `Embedded ${prepared.length} item(s).`,
-        });
-
-        let chunks = 0;
-        for (const p of prepared) {
-          const result = await storeIngestion(conn, p);
-          chunks += result.chunks;
-        }
-
-        logger.success(
-          `Refreshed ${updated} item(s), ${chunks} chunk(s) re-indexed.`,
-        );
       }),
     );
 

package/src/context/refresh.ts

@@ -0,0 +1,159 @@
+import type { McpxClient } from "@evantahler/mcpx";
+import type { BotholomewConfig } from "../config/schemas.ts";
+import type { DbConnection } from "../db/connection.ts";
+import { type ContextItem, updateContextItem } from "../db/context.ts";
+import { fetchUrl } from "./fetcher.ts";
+import {
+  type PreparedIngestion,
+  prepareIngestion,
+  storeIngestion,
+} from "./ingest.ts";
+
+export type RefreshItemStatus = "updated" | "unchanged" | "missing" | "error";
+
+export interface RefreshItemResult {
+  id: string;
+  context_path: string;
+  source_path: string;
+  source_type: "file" | "url";
+  status: RefreshItemStatus;
+  error?: string;
+}
+
+export interface RefreshResult {
+  checked: number;
+  updated: number;
+  unchanged: number;
+  missing: number;
+  reembedded: number;
+  chunks: number;
+  embeddings_skipped: boolean;
+  items: RefreshItemResult[];
+}
+
+export interface RefreshOptions {
+  concurrency?: number;
+  onItemProgress?: (done: number, total: number) => void;
+  onEmbedProgress?: (done: number, total: number) => void;
+}
+
+type IngestEmbedFn = (texts: string[]) => Promise<number[][]>;
+
+/**
+ * Refresh a batch of context items: re-read source (file or URL), diff, update
+ * content, and re-embed only the items that changed. Side-effect free on the
+ * outside world — the caller owns logging and spinners.
+ */
+export async function refreshContextItems(
+  conn: DbConnection,
+  items: ContextItem[],
+  config: Required<BotholomewConfig>,
+  mcpxClient: McpxClient | null,
+  opts: RefreshOptions = {},
+  embedFn?: IngestEmbedFn,
+): Promise<RefreshResult> {
+  const sourced = items.filter(
+    (i): i is ContextItem & { source_path: string } => !!i.source_path,
+  );
+
+  const results: RefreshItemResult[] = [];
+  const toReembed: string[] = [];
+
+  // Phase 1: read each source, diff against stored content, update when changed.
+  for (const [idx, item] of sourced.entries()) {
+    opts.onItemProgress?.(idx, sourced.length);
+    const base = {
+      id: item.id,
+      context_path: item.context_path,
+      source_path: item.source_path,
+      source_type: item.source_type,
+    };
+
+    try {
+      let content: string;
+
+      if (item.source_type === "url") {
+        const fetched = await fetchUrl(item.source_path, config, mcpxClient);
+        content = fetched.content;
+      } else {
+        const bunFile = Bun.file(item.source_path);
+        if (!(await bunFile.exists())) {
+          results.push({ ...base, status: "missing" });
+          continue;
+        }
+        content = await bunFile.text();
+      }
+
+      if (content === item.content) {
+        results.push({ ...base, status: "unchanged" });
+        continue;
+      }
+
+      await updateContextItem(conn, item.id, { content });
+      results.push({ ...base, status: "updated" });
+      toReembed.push(item.id);
+    } catch (err) {
+      results.push({
+        ...base,
+        status: "error",
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+  opts.onItemProgress?.(sourced.length, sourced.length);
+
+  const updated = results.filter((r) => r.status === "updated").length;
+  const unchanged = results.filter((r) => r.status === "unchanged").length;
+  const missing = results.filter((r) => r.status === "missing").length;
+
+  // Phase 2: re-embed changed items. Skip cleanly if no OpenAI key.
+  const hasEmbedder = !!embedFn || !!config.openai_api_key;
+  if (toReembed.length === 0 || !hasEmbedder) {
+    return {
+      checked: sourced.length,
+      updated,
+      unchanged,
+      missing,
+      reembedded: 0,
+      chunks: 0,
+      embeddings_skipped: toReembed.length > 0 && !hasEmbedder,
+      items: results,
+    };
+  }
+
+  const concurrency = opts.concurrency ?? 10;
+  const prepared: PreparedIngestion[] = [];
+  let completed = 0;
+
+  for (let i = 0; i < toReembed.length; i += concurrency) {
+    const batch = toReembed.slice(i, i + concurrency);
+    const batchResults = await Promise.all(
+      batch.map(async (id) => {
+        const r = await prepareIngestion(conn, id, config, embedFn);
+        completed++;
+        opts.onEmbedProgress?.(completed, toReembed.length);
+        return r;
+      }),
+    );
+    for (const r of batchResults) {
+      if (r) prepared.push(r);
+    }
+  }
+
+  let chunks = 0;
+  for (const p of prepared) {
+    const result = await storeIngestion(conn, p);
+    chunks += result.chunks;
+  }
+
+  return {
+    checked: sourced.length,
+    updated,
+    unchanged,
+    missing,
+    reembedded: prepared.length,
+    chunks,
+    embeddings_skipped: false,
+    items: results,
+  };
+}

package/src/db/context.ts

@@ -290,6 +290,10 @@ export async function getDistinctDirectories(
 
 // --- Mutations ---
 
+// `UPDATE context_items ... RETURNING *` can crash @duckdb/node-api via a C++
+// exception when the table's unique index is in a violated state. Update +
+// separate SELECT is equivalent here (single-connection, no concurrent writers)
+// and avoids the crash entirely.
 export async function updateContextItem(
   db: DbConnection,
   id: string,
@@ -307,14 +311,13 @@ export async function updateContextItem(
   setClauses.push("updated_at = current_timestamp::VARCHAR");
   params.push(id);
 
-  const row = await db.queryGet<ContextItemRow>(
+  await db.queryRun(
     `UPDATE context_items
      SET ${setClauses.join(", ")}
-     WHERE id = ?${params.length}
-     RETURNING *`,
+     WHERE id = ?${params.length}`,
     ...params,
   );
-  return row ? rowToContextItem(row) : null;
+  return getContextItem(db, id);
 }
 
 export async function updateContextItemContent(
@@ -322,15 +325,14 @@ export async function updateContextItemContent(
   contextPath: string,
   content: string,
 ): Promise<ContextItem | null> {
-  const row = await db.queryGet<ContextItemRow>(
+  await db.queryRun(
     `UPDATE context_items
      SET content = ?1, updated_at = current_timestamp::VARCHAR
-     WHERE context_path = ?2
-     RETURNING *`,
+     WHERE context_path = ?2`,
     content,
     contextPath,
   );
-  return row ? rowToContextItem(row) : null;
+  return getContextItemByPath(db, contextPath);
 }
 
 export async function applyPatchesToContextItem(

package/src/db/sql/10-dedupe_context_items.sql

@@ -0,0 +1,26 @@
+-- Older DBs could accumulate duplicate rows in context_items with the same
+-- context_path: migration 4's CREATE UNIQUE INDEX IF NOT EXISTS silently left
+-- the index metadata in place without enforcing it when duplicates predated
+-- the migration. The resulting "corrupt" unique index triggers a native
+-- crash in @duckdb/node-api on UPDATE ... RETURNING. Rebuild cleanly.
+DROP INDEX IF EXISTS idx_context_items_context_path;
+
+DELETE FROM embeddings WHERE context_item_id IN (
+  SELECT id FROM (
+    SELECT id, ROW_NUMBER() OVER (
+      PARTITION BY context_path
+      ORDER BY updated_at DESC, id DESC
+    ) AS rn FROM context_items
+  ) WHERE rn > 1
+);
+
+DELETE FROM context_items WHERE id IN (
+  SELECT id FROM (
+    SELECT id, ROW_NUMBER() OVER (
+      PARTITION BY context_path
+      ORDER BY updated_at DESC, id DESC
+    ) AS rn FROM context_items
+  ) WHERE rn > 1
+);
+
+CREATE UNIQUE INDEX idx_context_items_context_path ON context_items(context_path);

package/src/db/sql/11-rebuild_hnsw.sql

@@ -0,0 +1,9 @@
+-- The HNSW index from migration 6 can end up in an internally-inconsistent
+-- state after a native-side crash during embedding writes: the buffered WAL
+-- replay tries to re-insert a node that HNSW's high-level wrapper already has,
+-- and search_semantic then fails with "Duplicate keys not allowed in
+-- high-level wrappers". Dropping and recreating the index rebuilds it cleanly
+-- from the current contents of the embeddings table.
+DROP INDEX IF EXISTS idx_embeddings_cosine;
+
+CREATE INDEX idx_embeddings_cosine ON embeddings USING HNSW (embedding) WITH (metric = 'cosine');

package/src/tools/context/refresh.ts

@@ -0,0 +1,136 @@
+import { z } from "zod";
+import { refreshContextItems } from "../../context/refresh.ts";
+import {
+  type ContextItem,
+  listContextItems,
+  listContextItemsByPrefix,
+  resolveContextItem,
+} from "../../db/context.ts";
+import type { ToolDefinition } from "../tool.ts";
+
+const inputSchema = z.object({
+  path: z
+    .string()
+    .optional()
+    .describe(
+      "Context path or ID of a single item, or a path prefix to refresh a subtree. Mutually exclusive with `all`.",
+    ),
+  all: z
+    .boolean()
+    .optional()
+    .describe(
+      "Refresh every item that has a source_path (file or URL). Mutually exclusive with `path`.",
+    ),
+});
+
+const outputSchema = z.object({
+  checked: z.number(),
+  updated: z.number(),
+  unchanged: z.number(),
+  missing: z.number(),
+  reembedded: z.number(),
+  chunks: z.number(),
+  embeddings_skipped: z.boolean(),
+  items: z.array(
+    z.object({
+      id: z.string(),
+      context_path: z.string(),
+      source_path: z.string(),
+      source_type: z.enum(["file", "url"]),
+      status: z.enum(["updated", "unchanged", "missing", "error"]),
+      error: z.string().optional(),
+    }),
+  ),
+  message: z.string(),
+  is_error: z.boolean(),
+});
+
+const empty = {
+  checked: 0,
+  updated: 0,
+  unchanged: 0,
+  missing: 0,
+  reembedded: 0,
+  chunks: 0,
+  embeddings_skipped: false,
+  items: [],
+};
+
+export const contextRefreshTool = {
+  name: "context_refresh",
+  description:
+    "Re-read source files from disk / re-fetch source URLs, update stored content if it changed, and re-embed only changed items. Use `path` for a single item or subtree, or `all: true` for every sourced item. Items without a source_path are skipped. URL fetches use the project's MCPX client when available and fall back to plain HTTP.",
+  group: "context",
+  inputSchema,
+  outputSchema,
+  execute: async (input, ctx) => {
+    if (!input.path && !input.all) {
+      return {
+        ...empty,
+        message: "Provide a `path` or set `all: true`.",
+        is_error: true,
+      };
+    }
+    if (input.path && input.all) {
+      return {
+        ...empty,
+        message: "`path` and `all` are mutually exclusive.",
+        is_error: true,
+      };
+    }
+
+    let items: ContextItem[];
+    if (input.all) {
+      items = await listContextItems(ctx.conn);
+    } else {
+      const exact = await resolveContextItem(ctx.conn, input.path as string);
+      items = exact
+        ? [exact]
+        : await listContextItemsByPrefix(ctx.conn, input.path as string, {
+            recursive: true,
+          });
+    }
+
+    if (items.length === 0) {
+      return {
+        ...empty,
+        message: `No context items match \`${input.path ?? "all"}\`.`,
+        is_error: true,
+      };
+    }
+
+    const sourced = items.filter((i) => i.source_path);
+    if (sourced.length === 0) {
+      return {
+        ...empty,
+        message:
+          "No matching items have a source_path — nothing to refresh. (Items created via `context write` are not sourced.)",
+        is_error: false,
+      };
+    }
+
+    const result = await refreshContextItems(
+      ctx.conn,
+      sourced,
+      ctx.config,
+      ctx.mcpxClient,
+    );
+
+    const parts = [
+      `Checked ${result.checked}`,
+      `${result.updated} updated`,
+      `${result.unchanged} unchanged`,
+      `${result.missing} missing`,
+      `${result.reembedded} re-embedded (${result.chunks} chunks)`,
+    ];
+    if (result.embeddings_skipped) {
+      parts.push("embeddings skipped (no OpenAI API key configured)");
+    }
+
+    return {
+      ...result,
+      message: parts.join(", "),
+      is_error: false,
+    };
+  },
+} satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;

package/src/tools/dir/tree.ts

@@ -66,7 +66,7 @@ type TreeEntry = DirNode | FileNode;
 export const contextTreeTool = {
   name: "context_tree",
   description:
-    "Render a directory as a markdown-style tree. Use max_depth and items_per_dir to bound output; drill in by passing a deeper path.",
+    "Explore your context filesystem with a bird's-eye view — shows many paths across nested directories in one call. Reach for this first when you need to discover what content exists before reading a specific file (context_read) or running a keyword search (context_search). Returns a markdown-style tree; tune max_depth and items_per_dir to bound output, or pass a deeper path to drill into a subtree.",
   group: "context",
   inputSchema,
   outputSchema,

package/src/tools/registry.ts

@@ -1,6 +1,7 @@
 // Context tools
 
 import { readLargeResultTool } from "./context/read-large-result.ts";
+import { contextRefreshTool } from "./context/refresh.ts";
 import { contextSearchTool } from "./context/search.ts";
 import { updateBeliefsTool } from "./context/update-beliefs.ts";
 import { updateGoalsTool } from "./context/update-goals.ts";
@@ -68,6 +69,7 @@ export function registerAllTools(): void {
   registerTool(contextExistsTool);
   registerTool(contextCountLinesTool);
   registerTool(contextSearchTool);
+  registerTool(contextRefreshTool);
   registerTool(updateBeliefsTool);
   registerTool(updateGoalsTool);
   registerTool(readLargeResultTool);