botholomew 0.6.2 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "An AI agent for knowledge work",
   "type": "module",
   "bin": {
@@ -24,7 +24,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.88.0",
     "@duckdb/node-api": "^1.5.2-r.1",
-    "@evantahler/mcpx": "0.18.3",
+    "@evantahler/mcpx": "0.18.6",
     "ansis": "^4.2.0",
     "commander": "^14.0.0",
     "gray-matter": "^4.0.3",

package/src/commands/context.ts

@@ -8,11 +8,13 @@ import { loadConfig } from "../config/loader.ts";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import { generateDescription } from "../context/describer.ts";
 import { embedSingle } from "../context/embedder.ts";
+import { FetchFailureError, fetchUrl } from "../context/fetcher.ts";
 import {
   type PreparedIngestion,
   prepareIngestion,
   storeIngestion,
 } from "../context/ingest.ts";
+import { isUrl, urlToContextPath } from "../context/url-utils.ts";
 import type { DbConnection } from "../db/connection.ts";
 import {
   type ContextItem,
@@ -24,6 +26,7 @@ import {
   upsertContextItem,
 } from "../db/context.ts";
 import { getEmbeddingsForItem, hybridSearch } from "../db/embeddings.ts";
+import { createMcpxClient } from "../mcpx/client.ts";
 import { logger } from "../utils/logger.ts";
 import {
   registerContextToolSubcommands,
@@ -63,7 +66,7 @@ export function registerContextCommand(program: Command) {
           return;
         }
 
-        const header = `${ansis.bold("Path".padEnd(35))} ${"Title".padEnd(20)} ${"Description".padEnd(30)} ${"Type".padEnd(15)} ${"Updated".padEnd(18)} Indexed`;
+        const header = `${ansis.bold("ID".padEnd(36))} ${ansis.bold("Path".padEnd(35))} ${"Title".padEnd(20)} ${"Description".padEnd(30)} ${"Source".padEnd(6)} ${"Type".padEnd(15)} ${"Updated".padEnd(18)} Indexed`;
         console.log(header);
         console.log("-".repeat(header.length));
 
@@ -75,8 +78,13 @@ export function registerContextCommand(program: Command) {
           const desc = item.description
             ? ansis.dim(item.description.slice(0, 29).padEnd(30))
             : ansis.dim("".padEnd(30));
+          const source =
+            item.source_type === "url"
+              ? ansis.cyan("url".padEnd(6))
+              : ansis.dim("file".padEnd(6));
+          const id = ansis.dim(item.id.padEnd(36));
           console.log(
-            `${item.context_path.slice(0, 34).padEnd(35)} ${item.title.slice(0, 19).padEnd(20)} ${desc} ${item.mime_type.slice(0, 14).padEnd(15)} ${updated} ${indexed}`,
+            `${id} ${item.context_path.slice(0, 34).padEnd(35)} ${item.title.slice(0, 19).padEnd(20)} ${desc} ${source} ${item.mime_type.slice(0, 14).padEnd(15)} ${updated} ${indexed}`,
           );
         }
 
@@ -86,87 +94,178 @@ export function registerContextCommand(program: Command) {
 
   ctx
     .command("add <paths...>")
-    .description("Add files or directories to context")
+    .description("Add files, directories, or URLs to context")
     .option("--prefix <prefix>", "virtual path prefix", "/")
+    .option("--name <path>", "custom context path (single URL only)")
+    .option(
+      "--prompt-addition <text>",
+      "extra guidance for the URL fetcher agent (e.g., auth notes, tool hints)",
+    )
     .action((paths: string[], opts) =>
       withDb(program, async (conn, dir) => {
-        // Phase 1: Scan all paths and validate they exist
+        // Phase 1: Scan all paths — separate URLs from local files
         const filesToAdd: { filePath: string; contextPath: string }[] = [];
-        const spinner = createSpinner("Scanning files...").start();
+        const urlsToAdd: { url: string; contextPath: string }[] = [];
+        const spinner = createSpinner("Scanning paths...").start();
+
+        // Validate --name: only valid with a single URL
+        if (opts.name && (paths.length > 1 || !paths[0] || !isUrl(paths[0]))) {
+          spinner.error({
+            text: "--name can only be used with a single URL",
+          });
+          process.exit(1);
+        }
 
         for (const path of paths) {
-          const resolvedPath = resolve(path);
-          let info: Awaited<ReturnType<typeof stat>>;
-          try {
-            info = await stat(resolvedPath);
-          } catch {
-            spinner.error({ text: `Path not found: ${resolvedPath}` });
-            process.exit(1);
-          }
+          if (isUrl(path)) {
+            const contextPath =
+              opts.name || urlToContextPath(path, opts.prefix);
+            urlsToAdd.push({ url: path, contextPath });
+          } else {
+            const resolvedPath = resolve(path);
+            let info: Awaited<ReturnType<typeof stat>>;
+            try {
+              info = await stat(resolvedPath);
+            } catch {
+              spinner.error({ text: `Path not found: ${resolvedPath}` });
+              process.exit(1);
+            }
 
-          if (info.isDirectory()) {
-            const entries = await walkDirectory(resolvedPath);
-            for (const filePath of entries) {
-              const relativePath = filePath.slice(resolvedPath.length);
+            if (info.isDirectory()) {
+              const entries = await walkDirectory(resolvedPath);
+              for (const filePath of entries) {
+                const relativePath = filePath.slice(resolvedPath.length);
+                filesToAdd.push({
+                  filePath,
+                  contextPath: join(opts.prefix, relativePath),
+                });
+              }
+            } else {
               filesToAdd.push({
-                filePath,
-                contextPath: join(opts.prefix, relativePath),
+                filePath: resolvedPath,
+                contextPath: join(opts.prefix, basename(resolvedPath)),
               });
             }
-          } else {
-            filesToAdd.push({
-              filePath: resolvedPath,
-              contextPath: join(opts.prefix, basename(resolvedPath)),
-            });
           }
         }
 
+        const totalCount = filesToAdd.length + urlsToAdd.length;
         spinner.success({
-          text: `Found ${filesToAdd.length} file(s) to add.`,
+          text: `Found ${totalCount} item(s) to add (${filesToAdd.length} file(s), ${urlsToAdd.length} URL(s)).`,
         });
 
         // Phase 2: Load config and upsert DB records (batched, parallel LLM descriptions)
         const config = await loadConfig(dir);
         const CONCURRENCY = 10;
         let addCompleted = 0;
-        const upsertSpinner = createSpinner(
-          `Adding and describing 0/${filesToAdd.length} files...`,
-        ).start();
         const itemIds: { id: string; contextPath: string }[] = [];
 
-        for (let i = 0; i < filesToAdd.length; i += CONCURRENCY) {
-          const batch = filesToAdd.slice(i, i + CONCURRENCY);
-          const results = await Promise.all(
-            batch.map(async ({ filePath, contextPath }) => {
-              const result = await addFile(conn, filePath, contextPath, config);
-              addCompleted++;
-              upsertSpinner.update({
-                text: `Adding and describing ${addCompleted}/${filesToAdd.length} files...`,
-              });
-              return result ? { id: result, contextPath } : null;
-            }),
-          );
-          for (const r of results) {
-            if (r) itemIds.push(r);
+        // Process local files (with spinner — these are quick, no chatty logs)
+        if (filesToAdd.length > 0) {
+          const fileSpinner = createSpinner(
+            `Adding and describing 0/${filesToAdd.length} file(s)...`,
+          ).start();
+
+          for (let i = 0; i < filesToAdd.length; i += CONCURRENCY) {
+            const batch = filesToAdd.slice(i, i + CONCURRENCY);
+            const results = await Promise.all(
+              batch.map(async ({ filePath, contextPath }) => {
+                const result = await addFile(
+                  conn,
+                  filePath,
+                  contextPath,
+                  config,
+                );
+                addCompleted++;
+                fileSpinner.update({
+                  text: `Adding and describing ${addCompleted}/${filesToAdd.length} file(s)...`,
+                });
+                return result ? { id: result, contextPath } : null;
+              }),
+            );
+            for (const r of results) {
+              if (r) itemIds.push(r);
+            }
           }
+
+          fileSpinner.success({
+            text: `Added and described ${addCompleted} file(s).`,
+          });
         }
 
-        upsertSpinner.success({
-          text: `Added and described ${itemIds.length} file(s).`,
-        });
+        // Process URLs (no spinner — agent logs would interleave; render cleanly instead)
+        if (urlsToAdd.length > 0) {
+          const mcpxClient = await createMcpxClient(dir);
+          if (!mcpxClient) {
+            logger.dim(
+              "No MCP servers configured — remote fetches will use basic HTTP.",
+            );
+          }
+
+          let urlIdx = 0;
+          let urlAdded = 0;
+          for (const { url, contextPath } of urlsToAdd) {
+            urlIdx++;
+            console.log(
+              `\n${ansis.bold(`[${urlIdx}/${urlsToAdd.length}]`)} ${ansis.cyan(url)}`,
+            );
+            const result = await addUrl(
+              conn,
+              config,
+              url,
+              contextPath,
+              mcpxClient,
+              opts.promptAddition,
+            );
+            if (result.ok) {
+              urlAdded++;
+              itemIds.push({ id: result.id, contextPath });
+              console.log(`  ${ansis.green("✔")} stored at ${contextPath}`);
+            } else if (result.actionable) {
+              console.log(
+                `  ${ansis.red("✗")} ${ansis.bold("action required:")}`,
+              );
+              for (const line of result.error.split("\n")) {
+                console.log(`      ${ansis.yellow(line)}`);
+              }
+            } else {
+              console.log(
+                `  ${ansis.red("✗")} failed to fetch: ${result.error}`,
+              );
+            }
+          }
+
+          const urlSummary = `Added ${urlAdded}/${urlsToAdd.length} URL(s).`;
+          if (urlAdded === urlsToAdd.length) {
+            console.log(`\n${ansis.green("✔")} ${urlSummary}`);
+          } else if (urlAdded === 0) {
+            console.log(`\n${ansis.red("✗")} ${urlSummary}`);
+          } else {
+            console.log(`\n${ansis.yellow("⚠")} ${urlSummary}`);
+          }
+        }
 
         // Phase 3: Chunk + embed in parallel (network I/O)
         if (itemIds.length === 0 || !config.openai_api_key) {
           if (!config.openai_api_key) {
             logger.dim("Skipping embeddings (no OpenAI API key configured).");
           }
-          logger.success(`Added ${itemIds.length} file(s), 0 chunks indexed.`);
-          process.exit(0);
+          const msg = `Added ${itemIds.length}/${totalCount} item(s), 0 chunks indexed.`;
+          if (itemIds.length === totalCount) {
+            logger.success(msg);
+            process.exit(0);
+          } else if (itemIds.length === 0) {
+            logger.error(msg);
+            process.exit(1);
+          } else {
+            logger.warn(msg);
+            process.exit(1);
+          }
         }
 
         let completed = 0;
         const embedSpinner = createSpinner(
-          `Embedding 0/${itemIds.length} files...`,
+          `Embedding 0/${itemIds.length} items...`,
         ).start();
 
         const prepared: PreparedIngestion[] = [];
@@ -177,7 +276,7 @@ export function registerContextCommand(program: Command) {
               const result = await prepareIngestion(conn, id, config);
               completed++;
               embedSpinner.update({
-                text: `Embedding ${completed}/${itemIds.length} files...`,
+                text: `Embedding ${completed}/${itemIds.length} items...`,
               });
               return result;
             }),
@@ -187,7 +286,7 @@ export function registerContextCommand(program: Command) {
           }
         }
         embedSpinner.success({
-          text: `Embedded ${prepared.length} file(s).`,
+          text: `Embedded ${prepared.length} item(s).`,
         });
 
         // Phase 4: Store embeddings (sequential, fast DB writes)
@@ -204,8 +303,14 @@ export function registerContextCommand(program: Command) {
         const parts: string[] = [];
         if (filesAdded > 0) parts.push(`${filesAdded} added`);
         if (filesUpdated > 0) parts.push(`${filesUpdated} updated`);
-        logger.success(`${parts.join(", ")} — ${chunks} chunk(s) indexed.`);
-        process.exit(0);
+        const summary = `${parts.join(", ")} — ${chunks} chunk(s) indexed (${itemIds.length}/${totalCount} item(s)).`;
+        if (itemIds.length === totalCount) {
+          logger.success(summary);
+          process.exit(0);
+        } else {
+          logger.warn(summary);
+          process.exit(1);
+        }
       }),
     );
 
@@ -310,7 +415,9 @@ export function registerContextCommand(program: Command) {
 
   ctx
     .command("refresh [path]")
-    .description("Re-import files from disk and re-embed if content changed")
+    .description(
+      "Re-import files from disk / re-fetch URLs and re-embed if content changed",
+    )
     .option("--all", "refresh all items with a source path")
     .action((path: string | undefined, opts: { all?: boolean }) =>
       withDb(program, async (conn, dir) => {
@@ -333,7 +440,11 @@ export function registerContextCommand(program: Command) {
 
         const config = await loadConfig(dir);
 
-        // Phase 1: Read files from disk, compare, and update DB
+        // Init MCPX client if any URL items need refreshing
+        const hasUrls = sourced.some((i) => i.source_type === "url");
+        const mcpxClient = hasUrls ? await createMcpxClient(dir) : null;
+
+        // Phase 1: Read files / fetch URLs, compare, and update DB
         const spinner = createSpinner(
           `Refreshing 0/${sourced.length} items...`,
         ).start();
@@ -348,13 +459,21 @@ export function registerContextCommand(program: Command) {
           });
           try {
             const sourcePath = item.source_path as string;
-            const bunFile = Bun.file(sourcePath);
-            if (!(await bunFile.exists())) {
-              missing++;
-              logger.warn(`  Missing: ${item.source_path}`);
-              continue;
+            let content: string;
+
+            if (item.source_type === "url") {
+              const fetched = await fetchUrl(sourcePath, config, mcpxClient);
+              content = fetched.content;
+            } else {
+              const bunFile = Bun.file(sourcePath);
+              if (!(await bunFile.exists())) {
+                missing++;
+                logger.warn(`  Missing: ${item.source_path}`);
+                continue;
+              }
+              content = await bunFile.text();
             }
-            const content = await bunFile.text();
+
             if (content === item.content) {
               unchanged++;
               continue;
@@ -363,11 +482,11 @@ export function registerContextCommand(program: Command) {
             updated++;
             toReembed.push(item.id);
           } catch (err) {
-            logger.warn(`  Error reading ${item.source_path}: ${err}`);
+            logger.warn(`  Error refreshing ${item.source_path}: ${err}`);
           }
         }
         spinner.success({
-          text: `Checked ${sourced.length} file(s): ${updated} updated, ${unchanged} unchanged, ${missing} missing.`,
+          text: `Checked ${sourced.length} item(s): ${updated} updated, ${unchanged} unchanged, ${missing} missing.`,
         });
 
         // Phase 2: Re-embed changed items
@@ -381,7 +500,7 @@ export function registerContextCommand(program: Command) {
         const CONCURRENCY = 10;
         let completed = 0;
         const embedSpinner = createSpinner(
-          `Embedding 0/${toReembed.length} files...`,
+          `Embedding 0/${toReembed.length} item(s)...`,
         ).start();
 
         const prepared: PreparedIngestion[] = [];
@@ -392,7 +511,7 @@ export function registerContextCommand(program: Command) {
               const result = await prepareIngestion(conn, id, config);
               completed++;
               embedSpinner.update({
-                text: `Embedding ${completed}/${toReembed.length} files...`,
+                text: `Embedding ${completed}/${toReembed.length} item(s)...`,
               });
               return result;
             }),
@@ -402,7 +521,7 @@ export function registerContextCommand(program: Command) {
           }
         }
         embedSpinner.success({
-          text: `Embedded ${prepared.length} file(s).`,
+          text: `Embedded ${prepared.length} item(s).`,
         });
 
         let chunks = 0;
@@ -412,7 +531,7 @@ export function registerContextCommand(program: Command) {
         }
 
         logger.success(
-          `Refreshed ${updated} file(s), ${chunks} chunk(s) re-indexed.`,
+          `Refreshed ${updated} item(s), ${chunks} chunk(s) re-indexed.`,
         );
       }),
     );
@@ -476,6 +595,48 @@ async function addFile(
   }
 }
 
+/** Fetch a URL and upsert into context. Returns the item ID, or null on failure. */
+type AddUrlResult =
+  | { ok: true; id: string }
+  | { ok: false; error: string; actionable: boolean };
+
+async function addUrl(
+  conn: DbConnection,
+  config: Required<BotholomewConfig>,
+  url: string,
+  contextPath: string,
+  mcpxClient: Awaited<ReturnType<typeof createMcpxClient>>,
+  promptAddition?: string,
+): Promise<AddUrlResult> {
+  try {
+    const fetched = await fetchUrl(url, config, mcpxClient, promptAddition);
+
+    const description = await generateDescription(config, {
+      filename: new URL(url).hostname,
+      mimeType: fetched.mimeType,
+      content: fetched.content,
+    });
+
+    const item = await upsertContextItem(conn, {
+      title: fetched.title,
+      description,
+      content: fetched.content,
+      mimeType: fetched.mimeType,
+      sourceType: "url",
+      sourcePath: url,
+      contextPath,
+      isTextual: true,
+    });
+
+    return { ok: true, id: item.id };
+  } catch (err) {
+    if (err instanceof FetchFailureError) {
+      return { ok: false, error: err.userMessage, actionable: true };
+    }
+    return { ok: false, error: String(err), actionable: false };
+  }
+}
+
 async function walkDirectory(dirPath: string): Promise<string[]> {
   const files: string[] = [];
   const entries = await readdir(dirPath, { withFileTypes: true });

package/src/context/chunker.ts

@@ -9,6 +9,13 @@ export interface Chunk {
 const SHORT_CONTENT_THRESHOLD = 200;
 const LLM_TIMEOUT_MS = 10_000;
 const DEFAULT_OVERLAP_LINES = 2;
+// OpenAI's embedding endpoint caps inputs at 8192 tokens. The cl100k_base
+// tokenizer averages ~4 chars/token on plain English but can drop to ~2
+// chars/token on dense/code/non-ASCII content. We cap at 15k chars so even
+// at the worst-case ~2.5 chars/token (~6k tokens) we stay well under the
+// 8192-token limit, leaving headroom for the title/description prefix
+// prepended at embed time.
+const MAX_CHUNK_CHARS = 15_000;
 
 const CHUNKER_TOOL_NAME = "return_chunks";
 const CHUNKER_TOOL = {
@@ -41,6 +48,90 @@ const CHUNKER_TOOL = {
   },
 };
 
+/**
+ * Split text into pieces no larger than `maxChars`, preferring paragraph,
+ * line, and finally hard-character boundaries.
+ */
+function splitText(text: string, maxChars: number): string[] {
+  if (text.length <= maxChars) return [text];
+
+  // Try paragraph splits first.
+  const paragraphs = text.split(/\n\n+/);
+  if (paragraphs.length > 1) {
+    const out: string[] = [];
+    let buf = "";
+    for (const p of paragraphs) {
+      const candidate = buf ? `${buf}\n\n${p}` : p;
+      if (candidate.length <= maxChars) {
+        buf = candidate;
+      } else {
+        if (buf) out.push(buf);
+        if (p.length <= maxChars) {
+          buf = p;
+        } else {
+          out.push(...splitText(p, maxChars));
+          buf = "";
+        }
+      }
+    }
+    if (buf) out.push(buf);
+    return out;
+  }
+
+  // Fall back to line splits.
+  const lines = text.split("\n");
+  if (lines.length > 1) {
+    const out: string[] = [];
+    let buf = "";
+    for (const line of lines) {
+      const candidate = buf ? `${buf}\n${line}` : line;
+      if (candidate.length <= maxChars) {
+        buf = candidate;
+      } else {
+        if (buf) out.push(buf);
+        if (line.length <= maxChars) {
+          buf = line;
+        } else {
+          // Single line longer than maxChars — slice it.
+          for (let i = 0; i < line.length; i += maxChars) {
+            out.push(line.slice(i, i + maxChars));
+          }
+          buf = "";
+        }
+      }
+    }
+    if (buf) out.push(buf);
+    return out;
+  }
+
+  // Last resort: hard slice.
+  const out: string[] = [];
+  for (let i = 0; i < text.length; i += maxChars) {
+    out.push(text.slice(i, i + maxChars));
+  }
+  return out;
+}
+
+/**
+ * Re-chunk any chunks larger than `maxChars`, preserving order and reindexing.
+ */
+export function enforceMaxChunkSize(
+  chunks: Chunk[],
+  maxChars = MAX_CHUNK_CHARS,
+): Chunk[] {
+  const out: Chunk[] = [];
+  for (const c of chunks) {
+    if (c.content.length <= maxChars) {
+      out.push({ index: out.length, content: c.content });
+      continue;
+    }
+    for (const piece of splitText(c.content, maxChars)) {
+      out.push({ index: out.length, content: piece });
+    }
+  }
+  return out;
+}
+
 /**
  * Add overlapping lines from the end of each chunk to the start of the next.
  * Improves retrieval when concepts span chunk boundaries.
@@ -137,5 +228,11 @@ export async function chunk(
   }
 
   const chunks = await chunkWithLLM(content, mimeType, config);
-  return addOverlapToChunks(chunks);
+  // Enforce a hard size cap before AND after overlap. The first pass handles
+  // oversize chunks from the LLM (common for docs with very long lines); the
+  // second pass handles the rare case where added overlap pushes a near-limit
+  // chunk over.
+  const sized = enforceMaxChunkSize(chunks);
+  const withOverlap = addOverlapToChunks(sized);
+  return enforceMaxChunkSize(withOverlap);
 }

package/src/context/fetcher.ts

@@ -0,0 +1,436 @@
+import Anthropic from "@anthropic-ai/sdk";
+import type {
+  Tool as AnthropicTool,
+  MessageParam,
+  ToolResultBlockParam,
+  ToolUseBlock,
+} from "@anthropic-ai/sdk/resources/messages";
+import type { McpxClient } from "@evantahler/mcpx";
+import type { BotholomewConfig } from "../config/schemas.ts";
+import type { DbConnection } from "../db/connection.ts";
+import { mcpExecTool } from "../tools/mcp/exec.ts";
+import { mcpInfoTool } from "../tools/mcp/info.ts";
+import { mcpListToolsTool } from "../tools/mcp/list-tools.ts";
+import { mcpSearchTool } from "../tools/mcp/search.ts";
+import type { ToolContext } from "../tools/tool.ts";
+import { type AnyToolDefinition, toAnthropicTool } from "../tools/tool.ts";
+import { logger } from "../utils/logger.ts";
+import { stripHtmlTags } from "./url-utils.ts";
+
+const MAX_CONTENT_BYTES = 500_000;
+const MAX_TURNS = 10;
+const MAX_RESPONSE_TOKENS = 4_096;
+const PREVIEW_CHARS = 2_000;
+const HTTP_TIMEOUT_MS = 30_000;
+
+export interface FetchedContent {
+  title: string;
+  content: string;
+  mimeType: string;
+  sourceUrl: string;
+}
+
+export class FetchFailureError extends Error {
+  readonly userMessage: string;
+  constructor(message: string) {
+    super(message);
+    this.name = "FetchFailureError";
+    this.userMessage = message;
+  }
+}
+
+const FETCHER_SYSTEM_PROMPT = `You are a content fetcher. Your job is to find the right MCP tool to retrieve the content at the given URL, run it, and tell the harness which result to save.
+
+**Important: the harness captures the full result of every mcp_exec call automatically.** You only see a short preview of each result so you can verify it looks reasonable. You do NOT need to read or copy the full content — you just identify which exec call to save.
+
+Strongly prefer markdown output. Most MCP tools support a markdown/format parameter — use it when available.
+
+Workflow:
+1. Use mcp_search or mcp_list_tools to find the best tool for this URL (e.g., Google Docs tools for docs.google.com, Firecrawl for generic web pages, GitHub tools for github.com).
+2. Use mcp_info to inspect the tool's input schema.
+3. Call mcp_exec with the right arguments — request markdown format when supported.
+4. Look at the preview returned by mcp_exec. If it looks like the right content, call accept_content with the exec_call_id (the tool_use_id of the mcp_exec call) and a sensible title.
+
+Terminal tools:
+- accept_content(exec_call_id, title, mime_type?) — save the full content captured from a previous mcp_exec call. The harness has the full content; you just supply the id, title, and optional mime_type (defaults to text/markdown).
+- request_http_fallback() — fall back to a basic HTTP fetch. Use only when no MCP tool can handle the URL after a genuine attempt. Tools like Firecrawl can handle most URLs, so don't give up on the first try.
+- report_failure(message) — surface an actionable message to the user (e.g., "this Google Doc is private — share it with your service account", "Firecrawl is not authenticated"). Use only when there is a specific next step the user must take.`;
+
+const acceptContentTool: AnthropicTool = {
+  name: "accept_content",
+  description:
+    "Save the full content captured by the harness from a previous mcp_exec call. You only need to supply the exec_call_id (the tool_use_id of that mcp_exec call) and a title — the harness already has the full content. Do NOT paste content here.",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      exec_call_id: {
+        type: "string",
+        description:
+          "The tool_use_id of the mcp_exec call whose result should be saved (the harness lists captured ids in mcp_exec previews).",
+      },
+      title: {
+        type: "string",
+        description:
+          "A human-readable title for the content (e.g., the document title, or derived from the URL).",
+      },
+      mime_type: {
+        type: "string",
+        description: "MIME type of the content (defaults to text/markdown).",
+      },
+    },
+    required: ["exec_call_id", "title"],
+  },
+};
+
+interface AcceptContentInput {
+  exec_call_id: string;
+  title: string;
+  mime_type?: string;
+}
+
+const requestHttpFallbackTool: AnthropicTool = {
+  name: "request_http_fallback",
+  description:
+    "Fall back to a basic HTTP fetch. Use only when no MCP tool can handle the URL after a genuine attempt.",
+  input_schema: {
+    type: "object" as const,
+    properties: {},
+    required: [],
+  },
+};
+
+const reportFailureTool: AnthropicTool = {
+  name: "report_failure",
+  description:
+    "Report a fetch failure with an actionable message for the user (e.g., 'this Google Doc is private — share it with your service account'). Use only when there is a clear next step the user must take.",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      message: {
+        type: "string",
+        description:
+          "A clear, actionable, user-facing message explaining what the user needs to do to make this URL fetchable.",
+      },
+    },
+    required: ["message"],
+  },
+};
+
+interface ReportFailureInput {
+  message: string;
+}
+
+const mcpTools: AnyToolDefinition[] = [
+  mcpListToolsTool as unknown as AnyToolDefinition,
+  mcpSearchTool as unknown as AnyToolDefinition,
+  mcpInfoTool as unknown as AnyToolDefinition,
+  mcpExecTool as unknown as AnyToolDefinition,
+];
+
+export async function fetchUrl(
+  url: string,
+  config: Required<BotholomewConfig>,
+  mcpxClient: McpxClient | null,
+  promptAddition?: string,
+): Promise<FetchedContent> {
+  if (!config.anthropic_api_key) {
+    throw new Error(
+      "Anthropic API key is required for URL fetching. Set ANTHROPIC_API_KEY or configure it in .botholomew/config.json",
+    );
+  }
+
+  if (!mcpxClient) {
+    logger.dim("  no MCPX client — using HTTP fallback");
+    return httpFallback(url);
+  }
+
+  const result = await runFetcherLoop(url, config, mcpxClient, promptAddition);
+  if (result) return result;
+
+  logger.dim("  agent signaled fallback — using HTTP");
+  return httpFallback(url);
+}
+
+async function runFetcherLoop(
+  url: string,
+  config: Required<BotholomewConfig>,
+  mcpxClient: McpxClient,
+  promptAddition?: string,
+): Promise<FetchedContent | null> {
+  const client = new Anthropic({ apiKey: config.anthropic_api_key });
+
+  const toolCtx: ToolContext = {
+    conn: null as unknown as DbConnection,
+    projectDir: "",
+    config,
+    mcpxClient,
+  };
+
+  const tools: AnthropicTool[] = [
+    ...mcpTools.map(toAnthropicTool),
+    acceptContentTool,
+    requestHttpFallbackTool,
+    reportFailureTool,
+  ];
+
+  // Cache of full mcp_exec results keyed by tool_use_id.
+  // The LLM only sees a truncated preview; on accept_content it references
+  // the id and the harness saves the captured content.
+  const execResults = new Map<
+    string,
+    { server: string; tool: string; content: string; mimeType: string }
+  >();
+
+  const userPrompt = promptAddition
+    ? `Fetch the content at: ${url}\n\nAdditional guidance:\n${promptAddition}`
+    : `Fetch the content at: ${url}`;
+  const messages: MessageParam[] = [{ role: "user", content: userPrompt }];
+
+  for (let turn = 0; turn < MAX_TURNS; turn++) {
+    const response = await client.messages.create({
+      model: config.model,
+      max_tokens: MAX_RESPONSE_TOKENS,
+      system: FETCHER_SYSTEM_PROMPT,
+      messages,
+      tools,
+    });
+
+    // Log assistant text reasoning
+    for (const block of response.content) {
+      if (block.type === "text" && block.text.trim()) {
+        logger.dim(`  turn ${turn + 1}: ${block.text.trim()}`);
+      }
+    }
+
+    if (response.stop_reason === "max_tokens") {
+      throw new FetchFailureError(
+        `The fetched document is too large to return in a single LLM response (hit max_tokens=${MAX_RESPONSE_TOKENS}). Try fetching a smaller section, a specific page, or a tool that supports pagination.`,
+      );
+    }
+
+    const toolUseBlocks = response.content.filter(
+      (block): block is ToolUseBlock => block.type === "tool_use",
+    );
+
+    if (toolUseBlocks.length === 0) {
+      logger.dim(`  turn ${turn + 1}: no tool calls — signaling fallback`);
+      return null;
+    }
+
+    messages.push({ role: "assistant", content: response.content });
+
+    // Check for report_failure first (terminal — surfaces actionable user message)
+    const failureCall = toolUseBlocks.find((b) => b.name === "report_failure");
+    if (failureCall) {
+      const input = failureCall.input as Partial<ReportFailureInput>;
+      const message =
+        typeof input.message === "string" && input.message.trim()
+          ? input.message
+          : "Fetch failed but the agent did not provide a message.";
+      logger.dim(`  turn ${turn + 1}: report_failure: ${message}`);
+      throw new FetchFailureError(message);
+    }
+
+    // Check for request_http_fallback (terminal)
+    const fallbackCall = toolUseBlocks.find(
+      (b) => b.name === "request_http_fallback",
+    );
+    if (fallbackCall) {
+      logger.dim(`  turn ${turn + 1}: agent requested HTTP fallback`);
+      return null;
+    }
+
+    // Check for accept_content (terminal — looks up captured exec result)
+    const acceptCall = toolUseBlocks.find((b) => b.name === "accept_content");
+    if (acceptCall) {
+      const input = acceptCall.input as Partial<AcceptContentInput>;
+      if (
+        typeof input.exec_call_id !== "string" ||
+        typeof input.title !== "string"
+      ) {
+        logger.dim(
+          `  turn ${turn + 1}: accept_content missing required fields — asking agent to retry`,
+        );
+        messages.push({
+          role: "user",
+          content: [
+            {
+              type: "tool_result" as const,
+              tool_use_id: acceptCall.id,
+              content:
+                "Invalid accept_content call: both 'exec_call_id' and 'title' are required strings.",
+              is_error: true,
+            },
+          ],
+        });
+        continue;
+      }
+      const cached = execResults.get(input.exec_call_id);
+      if (!cached) {
+        const validIds = [...execResults.keys()];
+        logger.dim(
+          `  turn ${turn + 1}: accept_content: unknown exec_call_id "${input.exec_call_id}"`,
+        );
+        messages.push({
+          role: "user",
+          content: [
+            {
+              type: "tool_result" as const,
+              tool_use_id: acceptCall.id,
+              content: `No mcp_exec call with id "${input.exec_call_id}" was captured. Captured ids: ${validIds.length ? validIds.join(", ") : "(none yet — run mcp_exec first)"}.`,
+              is_error: true,
+            },
+          ],
+        });
+        continue;
+      }
+      const mimeType = input.mime_type || cached.mimeType;
+      logger.dim(
+        `  turn ${turn + 1}: accept_content: "${input.title}" (${cached.content.length} chars, ${mimeType}, from ${cached.server}/${cached.tool})`,
+      );
+      return {
+        title: input.title,
+        content: cached.content.slice(0, MAX_CONTENT_BYTES),
+        mimeType,
+        sourceUrl: url,
+      };
+    }
+
+    // Execute non-terminal MCP tools in parallel
+    const toolResults: ToolResultBlockParam[] = await Promise.all(
+      toolUseBlocks.map(async (toolUse) => {
+        // Log which tool the agent selected (and the underlying MCP server/tool for mcp_exec)
+        const toolInput = toolUse.input as Record<string, unknown>;
+        if (toolUse.name === "mcp_exec") {
+          logger.dim(
+            `  turn ${turn + 1}: mcp_exec → ${toolInput.server}/${toolInput.tool}`,
+          );
+        } else {
+          const args = JSON.stringify(toolInput).slice(0, 80);
+          logger.dim(`  turn ${turn + 1}: ${toolUse.name}(${args})`);
+        }
+
+        const toolDef = mcpTools.find((t) => t.name === toolUse.name);
+        if (!toolDef) {
+          return {
+            type: "tool_result" as const,
+            tool_use_id: toolUse.id,
+            content: `Unknown tool: ${toolUse.name}`,
+            is_error: true,
+          };
+        }
+
+        try {
+          const parsed = toolDef.inputSchema.safeParse(toolUse.input);
+          if (!parsed.success) {
+            return {
+              type: "tool_result" as const,
+              tool_use_id: toolUse.id,
+              content: `Invalid input: ${parsed.error.message}`,
+              is_error: true,
+            };
+          }
+          const result = await toolDef.execute(parsed.data, toolCtx);
+          if (result.is_error) {
+            logger.dim(
+              `         → error: ${JSON.stringify(result).slice(0, 160)}`,
+            );
+            return {
+              type: "tool_result" as const,
+              tool_use_id: toolUse.id,
+              content: JSON.stringify(result),
+              is_error: true,
+            };
+          }
+
+          // For successful mcp_exec calls, capture the full content in the
+          // harness and send only a preview to the LLM. The LLM accepts the
+          // result by referring to its tool_use_id.
+          if (toolUse.name === "mcp_exec") {
+            const execResult = result as {
+              result: string;
+              is_error: boolean;
+            };
+            const content = execResult.result;
+            execResults.set(toolUse.id, {
+              server: String(toolInput.server),
+              tool: String(toolInput.tool),
+              content,
+              mimeType: "text/markdown",
+            });
+            const preview =
+              content.length > PREVIEW_CHARS
+                ? `${content.slice(0, PREVIEW_CHARS)}\n\n[... ${content.length - PREVIEW_CHARS} more chars truncated. Full content (${content.length} chars total) is captured by the harness with exec_call_id="${toolUse.id}". Call accept_content with this id to save it.]`
+                : `${content}\n\n[Full content (${content.length} chars) captured by the harness with exec_call_id="${toolUse.id}". Call accept_content with this id to save it.]`;
+            logger.dim(
+              `         → captured ${content.length} chars (id=${toolUse.id})`,
+            );
+            return {
+              type: "tool_result" as const,
+              tool_use_id: toolUse.id,
+              content: preview,
+            };
+          }
+
+          return {
+            type: "tool_result" as const,
+            tool_use_id: toolUse.id,
+            content: JSON.stringify(result),
+          };
+        } catch (err) {
+          logger.dim(`         → exception: ${err}`);
+          return {
+            type: "tool_result" as const,
+            tool_use_id: toolUse.id,
+            content: `Error: ${err}`,
+            is_error: true,
+          };
+        }
+      }),
+    );
+
+    messages.push({ role: "user", content: toolResults });
+  }
+
+  logger.dim(`  max turns (${MAX_TURNS}) exceeded — signaling fallback`);
+  return null;
+}
+
+export async function httpFallback(url: string): Promise<FetchedContent> {
+  const response = await fetch(url, {
+    headers: { "User-Agent": "Botholomew/1.0" },
+    signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText}: ${url}`);
+  }
+
+  const contentType = response.headers.get("content-type") || "";
+  const isHtml = contentType.includes("text/html");
+  let text = await response.text();
+
+  let title = url;
+  if (isHtml) {
+    const titleMatch = text.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (titleMatch?.[1]) {
+      title = titleMatch[1].trim();
+    }
+    text = stripHtmlTags(text);
+  }
+
+  if (text.length > MAX_CONTENT_BYTES) {
+    text = text.slice(0, MAX_CONTENT_BYTES);
+  }
+
+  const mimeType = isHtml
+    ? "text/markdown"
+    : contentType.split(";")[0] || "text/plain";
+
+  return {
+    title,
+    content: text,
+    mimeType,
+    sourceUrl: url,
+  };
+}

package/src/context/url-utils.ts

@@ -0,0 +1,48 @@
+/**
+ * Attempts to parse the input as a URL and returns true if the protocol is http or https.
+ */
+export function isUrl(input: string): boolean {
+  try {
+    const url = new URL(input);
+    return url.protocol === "http:" || url.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Derives a virtual context path from a URL.
+ * Example: `https://docs.google.com/document/d/abc123/edit` → `/{prefix}/docs.google.com/document-d-abc123.md`
+ */
+export function urlToContextPath(url: string, prefix: string): string {
+  const parsed = new URL(url);
+  const hostname = parsed.hostname;
+  const pathname = parsed.pathname
+    .replace(/\/+$/, "") // strip trailing slashes
+    .replace(/^\/+/, "") // strip leading slashes
+    .replace(/[^a-zA-Z0-9\-_.]/g, "-") // slugify
+    .replace(/-{2,}/g, "-"); // collapse repeated dashes
+
+  const slug = pathname ? `${hostname}/${pathname}` : hostname;
+  const full = `${prefix.replace(/\/+$/, "")}/${slug}.md`;
+
+  if (full.length > 120) {
+    return `${full.slice(0, 117 - 3)}.md`;
+  }
+
+  return full;
+}
+
+/**
+ * Strips HTML tags from a string, removing script/style blocks first,
+ * then all remaining tags, and collapsing whitespace.
+ */
+export function stripHtmlTags(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, "") // remove script blocks
+    .replace(/<style[\s\S]*?<\/style>/gi, "") // remove style blocks
+    .replace(/<[^>]*>/g, "") // remove all remaining tags
+    .replace(/[ \t]+/g, " ") // collapse horizontal whitespace
+    .replace(/\n{3,}/g, "\n\n") // collapse excessive newlines
+    .trim();
+}

package/src/db/context.ts

@@ -9,6 +9,7 @@ export interface ContextItem {
   content: string | null;
   mime_type: string;
   is_textual: boolean;
+  source_type: "file" | "url";
   source_path: string | null;
   context_path: string;
   indexed_at: Date | null;
@@ -30,6 +31,7 @@ interface ContextItemRow {
   content_blob: unknown;
   mime_type: string;
   is_textual: boolean;
+  source_type: string;
   source_path: string | null;
   context_path: string;
   indexed_at: string | null;
@@ -45,6 +47,7 @@ function rowToContextItem(row: ContextItemRow): ContextItem {
     content: row.content,
     mime_type: row.mime_type,
     is_textual: !!row.is_textual,
+    source_type: row.source_type as "file" | "url",
     source_path: row.source_path,
     context_path: row.context_path,
     indexed_at: row.indexed_at ? new Date(row.indexed_at) : null,
@@ -61,6 +64,7 @@ export async function createContextItem(
     title: string;
     content?: string;
     mimeType?: string;
+    sourceType?: "file" | "url";
     sourcePath?: string;
     contextPath: string;
     description?: string;
@@ -69,8 +73,8 @@ export async function createContextItem(
 ): Promise<ContextItem> {
   const id = uuidv7();
   const row = await db.queryGet<ContextItemRow>(
-    `INSERT INTO context_items (id, title, description, content, mime_type, is_textual, source_path, context_path)
-     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
+    `INSERT INTO context_items (id, title, description, content, mime_type, is_textual, source_type, source_path, context_path)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
      RETURNING *`,
     id,
     params.title,
@@ -78,6 +82,7 @@ export async function createContextItem(
     params.content ?? null,
     params.mimeType ?? "text/plain",
     params.isTextual !== false,
+    params.sourceType ?? "file",
     params.sourcePath ?? null,
     params.contextPath,
   );
@@ -99,6 +104,7 @@ export async function upsertContextItem(
     title: string;
     content?: string;
     mimeType?: string;
+    sourceType?: "file" | "url";
     sourcePath?: string;
     contextPath: string;
     description?: string;

package/src/db/sql/9-source-type.sql

@@ -0,0 +1 @@
+ALTER TABLE context_items ADD COLUMN source_type TEXT DEFAULT 'file';