botholomew 0.13.0 → 0.14.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.13.0",
+  "version": "0.14.2",
   "description": "An autonomous AI agent for knowledge work — works your task queue while you sleep.",
   "type": "module",
   "bin": {

package/src/chat/agent.ts

@@ -62,6 +62,7 @@ const CHAT_TOOL_NAMES = new Set([
   "skill_edit",
   "skill_search",
   "skill_delete",
+  "sleep",
 ]);
 
 export function getChatTools() {
@@ -364,6 +365,7 @@ export async function runChatTurn(input: {
           projectDir,
           config,
           mcpxClient,
+          shouldAbort: session ? () => session.aborted : undefined,
         });
         const durationMs = Date.now() - start;
         const stored = maybeStoreResult(toolUse.name, result.output);
@@ -411,6 +413,7 @@ interface ChatToolCallCtx {
   projectDir: string;
   config: Required<BotholomewConfig>;
   mcpxClient: McpxClient | null;
+  shouldAbort?: () => boolean;
 }
 
 async function executeChatToolCall(
@@ -434,10 +437,20 @@ async function executeChatToolCall(
   }
 
   try {
-    const result = await withDb(baseCtx.dbPath, (conn) => {
-      const ctx: ToolContext = { ...baseCtx, conn };
-      return tool.execute(parsed.data, ctx);
-    });
+    // `sleep` deliberately yields for up to an hour; opening a DuckDB
+    // connection for that whole window would hold the instance-level file
+    // lock and block any worker that also wants the DB. Run it without a
+    // connection — the tool doesn't touch the DB.
+    const runWithoutDb = tool.name === "sleep";
+    const result = runWithoutDb
+      ? await tool.execute(parsed.data, {
+          ...baseCtx,
+          conn: undefined as unknown as ToolContext["conn"],
+        })
+      : await withDb(baseCtx.dbPath, (conn) => {
+          const ctx: ToolContext = { ...baseCtx, conn };
+          return tool.execute(parsed.data, ctx);
+        });
     const isError =
       typeof result === "object" && result !== null && "is_error" in result
         ? (result as { is_error: boolean }).is_error

package/src/commands/context.ts

@@ -18,6 +18,10 @@ import { withDb } from "../db/connection.ts";
 import { indexStats } from "../db/embeddings.ts";
 import { migrate } from "../db/schema.ts";
 import { createMcpxClient } from "../mcpx/client.ts";
+import {
+  type ContextFileMeta,
+  serializeContextFile,
+} from "../utils/frontmatter.ts";
 import { logger } from "../utils/logger.ts";
 
 export function registerContextCommand(program: Command) {
@@ -46,21 +50,42 @@ export function registerContextCommand(program: Command) {
       const dir = program.opts().dir;
       const config = await loadConfig(dir);
       const mcpxClient = await createMcpxClient(dir);
-      const spinner = createSpinner(`fetching ${url}`).start();
+      logger.info(`importing ${url}`);
       try {
         const fetched = await fetchUrl(url, config, mcpxClient, opts.prompt);
-        spinner.update({ text: "writing to context/" });
         const dest = opts.path ?? deriveContextPath(url, fetched.source);
-        await writeContextFile(dir, dest, fetched.content, {
+        const meta: ContextFileMeta = {
+          source_url: url,
+          imported_at: new Date().toISOString(),
+        };
+        // Title falls back to the URL when fetcher couldn't extract one —
+        // skip it in that case to avoid duplicating source_url.
+        if (fetched.title && fetched.title !== url) {
+          meta.title = fetched.title;
+        }
+        const body = serializeContextFile(meta, fetched.content);
+        await writeContextFile(dir, dest, body, {
           onConflict: opts.overwrite ? "overwrite" : "error",
         });
-        spinner.success({
-          text: `imported ${fetched.content.length} bytes → ${ansis.bold(`context/${dest}`)} (source: ${fetched.source ?? "http"})`,
+        logger.success(
+          `imported ${body.length} bytes → ${ansis.bold(`context/${dest}`)} (source: ${fetched.source ?? "http"})`,
+        );
+
+        // Reindex so the new file is searchable. reindexContext is
+        // incremental — files whose content_hash matches the index are
+        // skipped, so this only embeds the file we just wrote.
+        const dbPath = getDbPath(dir);
+        await withDb(dbPath, migrate);
+        const summary = await reindexContext(dir, config, dbPath, {
+          onProgress: (msg) => logger.dim(`  ${msg}`),
         });
+        logger.success(
+          `indexed: ${summary.added} added, ${summary.updated} updated, ${summary.unchanged} unchanged, ${summary.chunksWritten} chunks written`,
+        );
       } catch (err) {
-        spinner.error({
-          text: `import failed: ${err instanceof Error ? err.message : String(err)}`,
-        });
+        logger.error(
+          `import failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
         process.exit(1);
       } finally {
         await mcpxClient?.close();
@@ -196,9 +221,10 @@ function renderTreeAnsi(
 ): string {
   const lines: string[] = [];
   const connector = isRoot ? "" : isLast ? "└── " : "├── ";
-  const label = node.is_directory
+  const base = node.is_directory
     ? ansis.blue(node.name === "." ? "context/" : `${node.name}/`)
     : node.name;
+  const label = node.is_symlink ? `${base} ${ansis.cyan("→")}` : base;
   lines.push(`${prefix}${connector}${label}`);
   if (node.is_directory && node.children) {
     const childPrefix = isRoot ? "" : prefix + (isLast ? "    " : "│   ");

package/src/context/fetcher-errors.ts

@@ -0,0 +1,8 @@
+export class FetchFailureError extends Error {
+  readonly userMessage: string;
+  constructor(message: string) {
+    super(message);
+    this.name = "FetchFailureError";
+    this.userMessage = message;
+  }
+}

package/src/context/fetcher.ts

@@ -15,8 +15,16 @@ import { mcpSearchTool } from "../tools/mcp/search.ts";
 import type { ToolContext } from "../tools/tool.ts";
 import { type AnyToolDefinition, toAnthropicTool } from "../tools/tool.ts";
 import { logger } from "../utils/logger.ts";
+import { FetchFailureError } from "./fetcher-errors.ts";
+import {
+  convertToMarkdown,
+  isMarkdownMimeType,
+  resolveEffectiveMimeType,
+} from "./markdown-converter.ts";
 import { stripHtmlTags } from "./url-utils.ts";
 
+export { FetchFailureError } from "./fetcher-errors.ts";
+
 const MAX_CONTENT_BYTES = 500_000;
 const MAX_TURNS = 10;
 const MAX_RESPONSE_TOKENS = 4_096;
@@ -36,29 +44,23 @@ export interface FetchedContent {
   source: string | null;
 }
 
-export class FetchFailureError extends Error {
-  readonly userMessage: string;
-  constructor(message: string) {
-    super(message);
-    this.name = "FetchFailureError";
-    this.userMessage = message;
-  }
-}
-
 const FETCHER_SYSTEM_PROMPT = `You are a content fetcher. Your job is to find the right MCP tool to retrieve the content at the given URL, run it, and tell the harness which result to save.
 
 **Important: the harness captures the full result of every mcp_exec call automatically.** You only see a short preview of each result so you can verify it looks reasonable. You do NOT need to read or copy the full content — you just identify which exec call to save.
 
-Strongly prefer markdown output. Most MCP tools support a markdown/format parameter — use it when available.
+**Format preference: markdown, in order of preference.**
+1. When searching with mcp_search or mcp_list_tools, prefer tools whose names indicate markdown output: anything containing "markdown", "md", "AsMarkdown", "AsMd", "AsDocmd", or similar. For example, prefer "GoogleDocs_GetDocumentAsDocmd" over "GoogleDocs_GetDocumentAsHtml".
+2. If no markdown-named variant exists, use mcp_info to inspect the tool's input schema for a "format", "mime_type", "output_format", or similar parameter and request "markdown" (or "md") when available.
+3. If neither is possible, run the tool anyway. The harness will convert the captured content to markdown via a separate LLM call before saving — markdown-native tools are still preferred because they're cheaper and higher fidelity, but you do not have to find one.
 
 Workflow:
-1. Use mcp_search or mcp_list_tools to find the best tool for this URL (e.g., Google Docs tools for docs.google.com, Firecrawl for generic web pages, GitHub tools for github.com).
+1. Use mcp_search or mcp_list_tools to find the best tool for this URL (e.g., Google Docs tools for docs.google.com, Firecrawl for generic web pages, GitHub tools for github.com). Apply the format preference above.
 2. Use mcp_info to inspect the tool's input schema.
 3. Call mcp_exec with the right arguments — request markdown format when supported.
-4. Look at the preview returned by mcp_exec. If it looks like the right content, call accept_content with the exec_call_id (the tool_use_id of the mcp_exec call) and a sensible title.
+4. Look at the preview returned by mcp_exec. If it looks like the right content, call accept_content with the exec_call_id (the tool_use_id of the mcp_exec call), a sensible title, and the actual mime_type the tool returned (so the harness knows whether to convert).
 
 Terminal tools:
-- accept_content(exec_call_id, title, mime_type?) — save the full content captured from a previous mcp_exec call. The harness has the full content; you just supply the id, title, and optional mime_type (defaults to text/markdown).
+- accept_content(exec_call_id, title, mime_type?) — save the content captured from a previous mcp_exec call. The harness has the full content; you supply the id, title, and the source mime_type (e.g., "text/html", "application/json", "text/markdown"). The harness converts to markdown before storage when needed.
 - request_http_fallback() — fall back to a basic HTTP fetch. Use only when no MCP tool can handle the URL after a genuine attempt. Tools like Firecrawl can handle most URLs, so don't give up on the first try.
 - report_failure(message) — surface an actionable message to the user (e.g., "this Google Doc is private — share it with your service account", "Firecrawl is not authenticated"). Use only when there is a specific next step the user must take.`;
 
@@ -147,14 +149,14 @@ export async function fetchUrl(
 
   if (!mcpxClient) {
     logger.dim("  no MCPX client — using HTTP fallback");
-    return httpFallback(url);
+    return httpFallback(url, config);
   }
 
   const result = await runFetcherLoop(url, config, mcpxClient, promptAddition);
   if (result) return result;
 
   logger.dim("  agent signaled fallback — using HTTP");
-  return httpFallback(url);
+  return httpFallback(url, config);
 }
 
 async function runFetcherLoop(
@@ -292,14 +294,26 @@ async function runFetcherLoop(
         });
         continue;
       }
-      const mimeType = input.mime_type || cached.mimeType;
+      const claimedMimeType = input.mime_type || cached.mimeType;
       logger.dim(
-        `  turn ${turn + 1}: accept_content: "${input.title}" (${cached.content.length} chars, ${mimeType}, from ${cached.server}/${cached.tool})`,
+        `  turn ${turn + 1}: accept_content: "${input.title}" (${cached.content.length} chars, claimed ${claimedMimeType}, from ${cached.server}/${cached.tool})`,
+      );
+      const truncated = cached.content.slice(0, MAX_CONTENT_BYTES);
+      // Always normalize via the converter. MCP tools frequently mislabel
+      // format — e.g. Google Docs' "Docmd" tool claims text/markdown but
+      // returns a structured `[H1 ...]` annotation format. The converter
+      // prompt handles already-clean markdown by echoing it unchanged.
+      logger.dim(`  normalizing → markdown`);
+      const finalContent = await convertToMarkdown(
+        truncated,
+        claimedMimeType,
+        url,
+        config,
       );
       return {
         title: input.title,
-        content: cached.content.slice(0, MAX_CONTENT_BYTES),
-        mimeType,
+        content: finalContent,
+        mimeType: "text/markdown",
         sourceUrl: url,
         source: cached.server,
       };
@@ -405,7 +419,10 @@ async function runFetcherLoop(
   return null;
 }
 
-export async function httpFallback(url: string): Promise<FetchedContent> {
+export async function httpFallback(
+  url: string,
+  config: Required<BotholomewConfig> | null = null,
+): Promise<FetchedContent> {
   const response = await fetch(url, {
     headers: { "User-Agent": "Botholomew/1.0" },
     signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
@@ -416,7 +433,8 @@ export async function httpFallback(url: string): Promise<FetchedContent> {
   }
 
   const contentType = response.headers.get("content-type") || "";
-  const isHtml = contentType.includes("text/html");
+  const baseMimeType = contentType.split(";")[0]?.trim() || "text/plain";
+  const isHtml = baseMimeType === "text/html";
   let text = await response.text();
 
   let title = url;
@@ -425,21 +443,72 @@ export async function httpFallback(url: string): Promise<FetchedContent> {
     if (titleMatch?.[1]) {
       title = titleMatch[1].trim();
     }
-    text = stripHtmlTags(text);
   }
 
   if (text.length > MAX_CONTENT_BYTES) {
     text = text.slice(0, MAX_CONTENT_BYTES);
   }
 
-  const mimeType = isHtml
-    ? "text/markdown"
-    : contentType.split(";")[0] || "text/plain";
+  // No API key: we can't honestly produce markdown. Strip HTML tags so the
+  // saved file is at least readable, and label it text/plain so downstream
+  // consumers know it isn't real markdown. Other content types pass through.
+  if (!config?.anthropic_api_key) {
+    if (isHtml) {
+      return {
+        title,
+        content: stripHtmlTags(text),
+        mimeType: "text/plain",
+        sourceUrl: url,
+        source: null,
+      };
+    }
+    return {
+      title,
+      content: text,
+      mimeType: baseMimeType,
+      sourceUrl: url,
+      source: null,
+    };
+  }
+
+  // With an API key: convert anything non-text/non-markdown to markdown.
+  // Plain text short-circuits to avoid burning a conversion call on what's
+  // probably already a readable README/log/etc. text/markdown short-circuits
+  // too — but only after verifying the body actually looks like markdown.
+  // Some servers mislabel HTML as text/markdown.
+  const { mimeType: effectiveMimeType, sniffed } = resolveEffectiveMimeType(
+    baseMimeType,
+    text,
+  );
+  if (sniffed) {
+    logger.warn(
+      `server claimed ${baseMimeType} but body looks like ${effectiveMimeType} — converting anyway`,
+    );
+  }
+  if (
+    effectiveMimeType === "text/plain" ||
+    isMarkdownMimeType(effectiveMimeType)
+  ) {
+    return {
+      title,
+      content: text,
+      mimeType: effectiveMimeType,
+      sourceUrl: url,
+      source: null,
+    };
+  }
 
+  logger.dim(`  converting ${effectiveMimeType} → markdown`);
+  const converted = await convertToMarkdown(
+    text,
+    effectiveMimeType,
+    url,
+    config,
+  );
   return {
     title,
-    content: text,
-    mimeType,
+    content: converted,
+    mimeType: "text/markdown",
     sourceUrl: url,
     source: null,
   };

package/src/context/markdown-converter.ts

@@ -0,0 +1,186 @@
+import Anthropic from "@anthropic-ai/sdk";
+import type { BotholomewConfig } from "../config/schemas.ts";
+import { logger } from "../utils/logger.ts";
+import { FetchFailureError } from "./fetcher-errors.ts";
+
+const CONVERTER_MAX_TOKENS = 16_384;
+
+const CONVERTER_SYSTEM_PROMPT = `You normalize documents to clean, well-structured Markdown.
+
+**If the input is already clean, valid Markdown, return it verbatim with no edits.** Look for ATX headings (#, ##), bullet/numbered lists, fenced code blocks, inline code, links in [text](url) form, blockquotes, GFM tables. If the structure is consistently markdown-shaped, echo it back unchanged.
+
+Otherwise, convert it. The input mime_type is a hint, not a guarantee — verify the actual content. Common non-markdown formats to recognize and convert:
+- **HTML** — strip tags, scripts, styles, navigation/footer chrome; preserve headings, paragraphs, lists, tables, links, code.
+- **JSON / XML / YAML** — render the structure as readable Markdown (headings/lists for objects, tables where appropriate, fenced code blocks for inline values).
+- **DocMD (Google Docs structured format)** — lines like \`[H1 1-31 HEADING_1 tabId=t.0 ...] Title text\` or \`[P5 884-937 PARAGRAPH ...] Body text\`. Strip the bracket annotations entirely; map H1→#, H2→##, H3→###, P→paragraph; preserve the trailing text content.
+- **RTF, plain text with mixed structure, ad-hoc formats** — extract the semantic content, drop the noise.
+
+Rules for the output:
+- Preserve all semantic content: headings, paragraphs, lists, tables, links, inline code, code blocks, blockquotes.
+- Use ATX headings (#, ##, ###), fenced code blocks (\`\`\`lang), GFM-style tables, and reference- or inline-style links — whichever is cleanest.
+- Strip metadata headers/IDs that aren't part of the document body (e.g. \`@document_id: ...\`, \`@revision_id: ...\`).
+- Output **only** the Markdown. No preamble ("Here is the converted markdown:"), no trailing commentary, no wrapping the entire output in a code fence.`;
+
+const MARKDOWN_MIME_TYPES = new Set([
+  "text/markdown",
+  "text/x-markdown",
+  "text/md",
+]);
+
+export function isMarkdownMimeType(mimeType: string): boolean {
+  const base = mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return MARKDOWN_MIME_TYPES.has(base);
+}
+
+/**
+ * Sniff content for a non-markdown structure. Returns a mime type when the
+ * content has unmistakable markers of HTML / XML / JSON / etc., otherwise
+ * null. Used to verify a tool's claim of `text/markdown` — if the agent (or
+ * a defaulted mime type) lies about the format, we want to convert anyway.
+ *
+ * Markdown is a superset of plain text, so a null return ≠ "definitely
+ * markdown". It just means we found no strong contradicting signal.
+ */
+export function sniffNonMarkdownMimeType(content: string): string | null {
+  const head = content.trimStart().slice(0, 4096);
+  if (!head) return null;
+
+  if (/^<!doctype\s+html/i.test(head)) return "text/html";
+  if (/^<html[\s>]/i.test(head)) return "text/html";
+  if (/^<\?xml[\s?]/i.test(head)) return "application/xml";
+
+  // JSON: parses as JSON top-to-bottom (use the full content, not the head).
+  const trimmed = content.trim();
+  if (
+    (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
+    (trimmed.startsWith("[") && trimmed.endsWith("]"))
+  ) {
+    try {
+      JSON.parse(trimmed);
+      return "application/json";
+    } catch {
+      // fall through
+    }
+  }
+
+  // Heuristic HTML: dense tag markup. Markdown can contain occasional inline
+  // HTML, so we only flag it when tags dominate the sample.
+  const tagMatches = head.match(/<\/?[a-z][a-z0-9]*[\s/>]/gi) ?? [];
+  if (tagMatches.length >= 10) {
+    const charsPerTag = head.length / tagMatches.length;
+    if (charsPerTag < 80) return "text/html";
+  }
+
+  return null;
+}
+
+/**
+ * Decide the effective mime type for a piece of content. If the claim is
+ * markdown but the content sniffs as something else, trust the sniff so we
+ * convert instead of saving mislabeled garbage.
+ */
+export function resolveEffectiveMimeType(
+  claimedMimeType: string,
+  content: string,
+): { mimeType: string; sniffed: boolean } {
+  if (!isMarkdownMimeType(claimedMimeType)) {
+    return { mimeType: claimedMimeType, sniffed: false };
+  }
+  const sniffed = sniffNonMarkdownMimeType(content);
+  if (sniffed) return { mimeType: sniffed, sniffed: true };
+  return { mimeType: claimedMimeType, sniffed: false };
+}
+
+function stripLeadingMarkdownFence(text: string): string {
+  const trimmed = text.trim();
+  const fenceMatch = trimmed.match(
+    /^```(?:markdown|md)?\s*\n([\s\S]*?)\n```\s*$/,
+  );
+  if (fenceMatch?.[1]) return fenceMatch[1];
+  return text;
+}
+
+/**
+ * Convert arbitrary content to Markdown via a single-shot LLM call.
+ *
+ * Does **not** short-circuit on `mimeType === "text/markdown"` — tools
+ * frequently mislabel their output (e.g. Google Docs' "DocMD" tool returns
+ * structured `[H1 ...]` annotations, not real markdown). The mime type is
+ * passed in as a hint for the model; the model decides whether the content
+ * is already markdown (echo unchanged) or needs converting.
+ *
+ * - Throws FetchFailureError when the response hits max_tokens (silently
+ *   truncating the saved file would be worse than failing loudly).
+ * - On transient API errors, logs a warning and returns the raw content so
+ *   the import still produces *something* the user can edit.
+ */
+export async function convertToMarkdown(
+  content: string,
+  mimeType: string,
+  sourceUrl: string,
+  config: Required<BotholomewConfig>,
+): Promise<string> {
+  if (!config.anthropic_api_key) return content;
+
+  const client = new Anthropic({ apiKey: config.anthropic_api_key });
+  // Conversion is mechanical text-shaping — Haiku (the chunker model) is
+  // plenty smart for this and ~5x faster than Opus on long documents.
+  const model = config.chunker_model || config.model;
+
+  try {
+    const stream = client.messages.stream({
+      model,
+      max_tokens: CONVERTER_MAX_TOKENS,
+      system: CONVERTER_SYSTEM_PROMPT,
+      messages: [
+        {
+          role: "user",
+          content: `Convert this ${mimeType} content to Markdown. Source URL: ${sourceUrl}\n\n${content}`,
+        },
+      ],
+    });
+
+    let charsReceived = 0;
+    let lastLogged = 0;
+    const PROGRESS_INTERVAL_CHARS = 2_000;
+    for await (const event of stream) {
+      if (
+        event.type === "content_block_delta" &&
+        event.delta.type === "text_delta"
+      ) {
+        charsReceived += event.delta.text.length;
+        if (charsReceived - lastLogged >= PROGRESS_INTERVAL_CHARS) {
+          logger.dim(`  ...converted ${charsReceived} chars`);
+          lastLogged = charsReceived;
+        }
+      }
+    }
+
+    const final = await stream.finalMessage();
+
+    if (final.stop_reason === "max_tokens") {
+      throw new FetchFailureError(
+        `Markdown conversion exceeded token budget (max_tokens=${CONVERTER_MAX_TOKENS}). The source document is too large to convert in one pass — try fetching a smaller section or a tool that supports pagination.`,
+      );
+    }
+
+    const text = final.content
+      .flatMap((block) => (block.type === "text" ? [block.text] : []))
+      .join("");
+
+    if (!text.trim()) {
+      logger.warn(
+        "markdown conversion returned empty output — saving raw content",
+      );
+      return content;
+    }
+
+    return stripLeadingMarkdownFence(text);
+  } catch (err) {
+    if (err instanceof FetchFailureError) throw err;
+    logger.warn(
+      `markdown conversion failed (${err instanceof Error ? err.message : String(err)}) — saving raw content`,
+    );
+    return content;
+  }
+}