botholomew 0.8.10 → 0.9.4

package/src/commands/tools.ts

@@ -3,6 +3,10 @@ import type { Command } from "commander";
 import { z } from "zod";
 import { loadConfig } from "../config/loader.ts";
 import { getDbPath } from "../constants.ts";
+import { parseDriveRef } from "../context/drives.ts";
+import type { DbConnection } from "../db/connection.ts";
+import { getContextItemById } from "../db/context.ts";
+import { isUuid } from "../db/uuid.ts";
 import { registerAllTools } from "../tools/registry.ts";
 import {
   type AnyToolDefinition,
@@ -43,7 +47,7 @@ export function registerSearchToolSubcommands(parent: Command) {
   }
 }
 
-/** Derive CLI subcommand name from tool name: "context_read" → "read", "context_list_dir" → "list-dir" */
+/** Derive CLI subcommand name from tool name: "context_read" → "read", "context_create_dir" → "create-dir" */
 function deriveSubName(toolName: string): string {
   return toolName.replace(/^[^_]+_/, "").replace(/_/g, "-");
 }
@@ -64,7 +68,7 @@ function registerToolAsCLI(parent: Command, tool: AnyToolDefinition) {
   for (const [key, schema] of Object.entries(shape)) {
     const desc = schema.description ?? key;
     const isOptional = schema.isOptional();
-    const unwrapped = unwrapOptional(schema);
+    const unwrapped = unwrapSchema(schema);
 
     if (isPositionalArg(key, tool.name)) {
       positionals.push(isOptional ? `[${key}]` : `<${key}>`);
@@ -109,7 +113,14 @@ function registerToolAsCLI(parent: Command, tool: AnyToolDefinition) {
     while (root.parent) root = root.parent;
     return withDb(root, async (conn, dir) => {
       try {
-        const input = buildInput(tool, positionals, options, shape, args);
+        const input = await buildInput(
+          tool,
+          positionals,
+          options,
+          shape,
+          args,
+          conn,
+        );
 
         const ctx: ToolContext = {
           conn,
@@ -129,7 +140,7 @@ function registerToolAsCLI(parent: Command, tool: AnyToolDefinition) {
   });
 }
 
-function buildInput(
+async function buildInput(
   tool: AnyToolDefinition,
   positionals: string[],
   options: {
@@ -140,14 +151,36 @@ function buildInput(
   }[],
   shape: Record<string, z.ZodType>,
   args: unknown[],
-): Record<string, unknown> {
+  conn: DbConnection,
+): Promise<Record<string, unknown>> {
   const input: Record<string, unknown> = {};
 
-  // Positional args come first in Commander's action callback
+  // Positional args come first in Commander's action callback. Context tools
+  // carry `(drive, path)` or `(src_drive, src_path, …)` in their schema but
+  // accept a friendlier `drive:/path` or bare-UUID form as a single positional
+  // on the CLI.
   for (let i = 0; i < positionals.length; i++) {
     const key = positionals[i]?.replace(/[<>[\]]/g, "");
     const value = args[i];
-    if (key !== undefined && value !== undefined) input[key] = value;
+    if (key === undefined || value === undefined) continue;
+    const splitTargets = driveRefSplitTargets(key, shape);
+    if (splitTargets && typeof value === "string") {
+      const parsed = parseDriveRef(value);
+      if (parsed) {
+        input[splitTargets.drive] = parsed.drive;
+        input[splitTargets.path] = parsed.path;
+        continue;
+      }
+      if (isUuid(value)) {
+        const item = await getContextItemById(conn, value);
+        if (item) {
+          input[splitTargets.drive] = item.drive;
+          input[splitTargets.path] = item.path;
+          continue;
+        }
+      }
+    }
+    input[key] = value;
   }
 
   // Options object is the last argument before the Command object
@@ -163,7 +196,7 @@ function buildInput(
 
     const schemaForKey = shape[opt.key];
     if (!schemaForKey) continue;
-    const unwrapped = unwrapOptional(schemaForKey);
+    const unwrapped = unwrapSchema(schemaForKey);
 
     // Parse JSON for array types
     if (opt.isArray && typeof value === "string") {
@@ -192,6 +225,19 @@ function formatOutput(result: unknown, _toolName: string) {
   if (typeof result === "object") {
     const obj = result as Record<string, unknown>;
 
+    // Structured error shape: { is_error: true, message, next_action_hint? }
+    if (obj.is_error === true) {
+      const msg = typeof obj.message === "string" ? obj.message : "Error";
+      logger.error(msg);
+      if (
+        typeof obj.next_action_hint === "string" &&
+        obj.next_action_hint.length > 0
+      ) {
+        console.log(ansis.dim(obj.next_action_hint));
+      }
+      process.exit(1);
+    }
+
     // Special formatting for known output shapes
     if ("tree" in obj && typeof obj.tree === "string") {
       console.log(obj.tree);
@@ -217,6 +263,26 @@ function formatOutput(result: unknown, _toolName: string) {
       return;
     }
 
+    if ("drives" in obj && Array.isArray(obj.drives)) {
+      const drives = obj.drives as { drive: string; count: number }[];
+      if (drives.length === 0) {
+        if (typeof obj.hint === "string") console.log(ansis.dim(obj.hint));
+        return;
+      }
+      const widest = Math.max(...drives.map((d) => d.drive.length));
+      for (const d of drives) {
+        const label = `${d.drive}:/`.padEnd(widest + 2);
+        const plural = d.count === 1 ? "item" : "items";
+        console.log(
+          `  ${ansis.cyan(label)} ${ansis.dim(`(${d.count} ${plural})`)}`,
+        );
+      }
+      if (typeof obj.hint === "string") {
+        console.log(`\n${ansis.dim(obj.hint)}`);
+      }
+      return;
+    }
+
     if ("matches" in obj && Array.isArray(obj.matches)) {
       for (const match of obj.matches) {
         if (typeof match === "string") {
@@ -263,7 +329,6 @@ function isPositionalArg(key: string, toolName: string): boolean {
   // These keys are treated as positional arguments
   const positionalKeys: Record<string, string[]> = {
     context_create_dir: ["path"],
-    context_list_dir: ["path"],
     context_tree: ["path"],
     context_dir_size: ["path"],
     context_read: ["path"],
@@ -282,9 +347,33 @@ function isPositionalArg(key: string, toolName: string): boolean {
   return positionalKeys[toolName]?.includes(key) ?? false;
 }
 
-function unwrapOptional(schema: z.ZodType): z.ZodType {
+function unwrapSchema(schema: z.ZodType): z.ZodType {
   if (schema instanceof z.ZodOptional) {
-    return schema.unwrap() as z.ZodType;
+    return unwrapSchema(schema.unwrap() as z.ZodType);
+  }
+  if (schema instanceof z.ZodDefault) {
+    return unwrapSchema(schema.unwrap() as z.ZodType);
   }
   return schema;
 }
+
+/**
+ * Decide how to expand a positional `path`/`src`/`dst` value into the tool's
+ * schema when it carries a `drive:/path` prefix. Returns the drive+path field
+ * names in the schema, or null if the schema has no matching drive field.
+ */
+function driveRefSplitTargets(
+  positionalKey: string,
+  shape: Record<string, z.ZodType>,
+): { drive: string; path: string } | null {
+  if (positionalKey === "path" && "drive" in shape && "path" in shape) {
+    return { drive: "drive", path: "path" };
+  }
+  if (positionalKey === "src" && "src_drive" in shape && "src_path" in shape) {
+    return { drive: "src_drive", path: "src_path" };
+  }
+  if (positionalKey === "dst" && "dst_drive" in shape && "dst_path" in shape) {
+    return { drive: "dst_drive", path: "dst_path" };
+  }
+  return null;
+}

package/src/context/describer.ts

@@ -3,7 +3,6 @@ import type { BotholomewConfig } from "../config/schemas.ts";
 import { logger } from "../utils/logger.ts";
 
 const DESCRIBE_TOOL_NAME = "return_description";
-const DESCRIBE_AND_PLACE_TOOL_NAME = "return_description_and_path";
 
 const DESCRIBE_TOOL = {
   name: DESCRIBE_TOOL_NAME,
@@ -21,28 +20,6 @@ const DESCRIBE_TOOL = {
   },
 };
 
-const DESCRIBE_AND_PLACE_TOOL = {
-  name: DESCRIBE_AND_PLACE_TOOL_NAME,
-  description:
-    "Return a one-sentence description AND a suggested absolute folder path for this file.",
-  input_schema: {
-    type: "object" as const,
-    properties: {
-      description: {
-        type: "string",
-        description:
-          "A concise one-sentence summary of what this content is about.",
-      },
-      suggested_path: {
-        type: "string",
-        description:
-          "Absolute virtual-filesystem path (starts with /) where this file should live, including the filename. Prefer existing folders. Include a project/source disambiguator (e.g. /projects/<source-dir>/README.md) when the basename is likely to collide.",
-      },
-    },
-    required: ["description", "suggested_path"],
-  },
-};
-
 const TIMEOUT_MS = 10_000;
 const MAX_CONTENT_CHARS = 8000;
 const MAX_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
@@ -56,35 +33,11 @@ const IMAGE_TYPES = new Set([
 
 type ImageMediaType = "image/jpeg" | "image/png" | "image/gif" | "image/webp";
 
-/**
- * Build the message content array for the LLM description request.
- * Attaches the file as an image or document block when possible.
- */
 async function buildMessageContent(
   opts: DescriberOpts,
-  includePlacement: boolean,
 ): Promise<Anthropic.Messages.ContentBlockParam[]> {
-  const placementBlock = includePlacement
-    ? [
-        "",
-        "Also suggest an absolute folder path where this file should live in the virtual filesystem. Rules:",
-        "- Start with /",
-        "- Keep the basename close to the source filename",
-        "- STRONGLY prefer folders that already exist below — reuse them unless the new file is clearly unrelated to everything there. Do NOT invent a new folder that is a near-synonym of an existing one.",
-        "- Use at most 3 nested folders unless an existing folder already goes deeper",
-        "- If the basename is common (README.md, index.md, notes.md), include a project/source disambiguator from the source path",
-        opts.existingTree
-          ? `\nExisting filesystem (folders end with /, files are listed under the folders they live in so you can see what kinds of documents are already there):\n${opts.existingTree}`
-          : "\nExisting filesystem: (empty — you are placing the first file)",
-        opts.sourcePath ? `\nSource filesystem path: ${opts.sourcePath}` : "",
-      ]
-        .filter((s) => s.length > 0)
-        .join("\n")
-    : "";
-
-  const textPrompt = `Describe this file in one sentence. Be specific about what it contains, not generic.\n\nFilename: ${opts.filename}\nMIME type: ${opts.mimeType}${placementBlock ? `\n${placementBlock}` : ""}`;
-
-  // Text file — include content inline
+  const textPrompt = `Describe this file in one sentence. Be specific about what it contains, not generic.\n\nFilename: ${opts.filename}\nMIME type: ${opts.mimeType}`;
+
   if (opts.content) {
     const truncated =
       opts.content.length > MAX_CONTENT_CHARS
@@ -93,7 +46,6 @@ async function buildMessageContent(
     return [{ type: "text", text: `${textPrompt}\n\nContent:\n${truncated}` }];
   }
 
-  // Binary file — try to attach if we have a file path
   if (opts.filePath) {
     const file = Bun.file(opts.filePath);
     const size = file.size;
@@ -127,7 +79,6 @@ async function buildMessageContent(
     }
   }
 
-  // Fallback — describe from filename and MIME type only
   return [
     {
       type: "text",
@@ -141,20 +92,6 @@ interface DescriberOpts {
   mimeType: string;
   content: string | null;
   filePath?: string;
-  sourcePath?: string;
-  existingTree?: string;
-}
-
-/** Normalize and validate an LLM-suggested path. Returns null if invalid. */
-export function sanitizeSuggestedPath(raw: string): string | null {
-  const trimmed = raw.trim();
-  if (!trimmed) return null;
-  if (!trimmed.startsWith("/")) return null;
-  if (trimmed.includes("..")) return null;
-  // Collapse repeated slashes, strip trailing slash (unless root).
-  const collapsed = trimmed.replace(/\/+/g, "/");
-  if (collapsed === "/") return null; // needs a filename
-  return collapsed.endsWith("/") ? collapsed.slice(0, -1) : collapsed;
 }
 
 /**
@@ -173,7 +110,7 @@ export async function generateDescription(
   const client = new Anthropic({ apiKey: config.anthropic_api_key });
 
   try {
-    const content = await buildMessageContent(opts, false);
+    const content = await buildMessageContent(opts);
 
     const response = await Promise.race([
       client.messages.create({
@@ -201,55 +138,3 @@ export async function generateDescription(
     return "";
   }
 }
-
-/**
- * Generate description + suggested_path in a single LLM call.
- * Returns { description, suggested_path } on success, or null on failure.
- */
-export async function generateDescriptionAndPath(
-  config: Required<BotholomewConfig>,
-  opts: DescriberOpts,
-): Promise<{ description: string; suggested_path: string } | null> {
-  if (!config.anthropic_api_key) return null;
-
-  const client = new Anthropic({ apiKey: config.anthropic_api_key });
-
-  try {
-    const content = await buildMessageContent(opts, true);
-
-    const response = await Promise.race([
-      client.messages.create({
-        model: config.chunker_model,
-        max_tokens: 512,
-        tools: [DESCRIBE_AND_PLACE_TOOL],
-        tool_choice: { type: "tool", name: DESCRIBE_AND_PLACE_TOOL_NAME },
-        messages: [{ role: "user", content }],
-      }),
-      new Promise<never>((_, reject) =>
-        setTimeout(
-          () => reject(new Error("Description+path generation timeout")),
-          TIMEOUT_MS,
-        ),
-      ),
-    ]);
-
-    const toolBlock = response.content.find((b) => b.type === "tool_use");
-    if (!toolBlock || toolBlock.type !== "tool_use") return null;
-
-    const input = toolBlock.input as {
-      description?: string;
-      suggested_path?: string;
-    };
-    const suggested = input.suggested_path
-      ? sanitizeSuggestedPath(input.suggested_path)
-      : null;
-    if (!suggested) return null;
-    return {
-      description: input.description || "",
-      suggested_path: suggested,
-    };
-  } catch (err) {
-    logger.debug(`Description+path generation failed: ${err}`);
-    return null;
-  }
-}

package/src/context/drives.ts

@@ -0,0 +1,110 @@
+/**
+ * Drives name the origin of a context item. Every item lives at a
+ * `(drive, path)` pair; the `drive:/path` string form is a display and CLI
+ * convention (single column queries use the two columns directly).
+ *
+ * Built-in drives:
+ *   disk       — local filesystem; path is the absolute filesystem path
+ *   url        — generic HTTP(S) URL; path is the full URL
+ *   agent      — agent-authored scratch; path is whatever the agent chose
+ *   google-docs — Google Docs; path is `/<docId>`
+ *   github     — GitHub content; path is `/<owner>/<repo>/<rest>`
+ */
+
+export const BUILT_IN_DRIVES = [
+  "disk",
+  "url",
+  "agent",
+  "google-docs",
+  "github",
+] as const;
+
+export interface DriveTarget {
+  drive: string;
+  path: string;
+}
+
+/** Parse `drive:/path` → `{ drive, path }`. Returns null if not in drive form. */
+export function parseDriveRef(ref: string): DriveTarget | null {
+  const i = ref.indexOf(":");
+  if (i <= 0) return null;
+  const drive = ref.slice(0, i);
+  const path = ref.slice(i + 1);
+  if (!path.startsWith("/")) return null;
+  if (!/^[a-z][a-z0-9_-]*$/.test(drive)) return null;
+  return { drive, path };
+}
+
+/** Format a `(drive, path)` pair for display / CLI. */
+export function formatDriveRef(target: DriveTarget): string {
+  return `${target.drive}:${target.path}`;
+}
+
+/**
+ * Detect the right drive for a URL. If `mcpxServerName` is provided, prefer it
+ * as a hint (some MCP servers are named after the service they back).
+ */
+export function detectDriveFromUrl(
+  url: string,
+  mcpxServerName?: string | null,
+): DriveTarget {
+  const hint = mcpxServerName?.toLowerCase() ?? "";
+  let parsed: URL | null = null;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return { drive: "url", path: `/${url}` };
+  }
+
+  const host = parsed.hostname.toLowerCase();
+
+  if (
+    host === "docs.google.com" ||
+    (hint.includes("google") && hint.includes("doc"))
+  ) {
+    const docId = extractGoogleDocId(parsed);
+    if (docId) return { drive: "google-docs", path: `/${docId}` };
+  }
+
+  if (
+    host === "github.com" ||
+    host === "raw.githubusercontent.com" ||
+    hint.includes("github")
+  ) {
+    const ghPath = extractGithubPath(parsed);
+    if (ghPath) return { drive: "github", path: ghPath };
+  }
+
+  return { drive: "url", path: `/${url}` };
+}
+
+function extractGoogleDocId(u: URL): string | null {
+  // https://docs.google.com/document/d/<docId>/edit
+  // https://docs.google.com/spreadsheets/d/<docId>/edit
+  const m = u.pathname.match(/\/d\/([^/]+)/);
+  return m?.[1] ?? null;
+}
+
+function extractGithubPath(u: URL): string | null {
+  // https://github.com/<owner>/<repo>/blob/<ref>/<path...>
+  // https://github.com/<owner>/<repo>/tree/<ref>/<path...>
+  // https://github.com/<owner>/<repo>
+  // https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path...>
+  const segs = u.pathname.split("/").filter(Boolean);
+  if (segs.length < 2) return null;
+  const [owner, repo, kind, _ref, ...rest] = segs;
+  if (!owner || !repo) return null;
+  if (u.hostname === "raw.githubusercontent.com") {
+    // segs: owner, repo, ref, ...rest
+    const [_o, _r, _f, ...raw] = segs;
+    return raw.length > 0
+      ? `/${owner}/${repo}/${raw.join("/")}`
+      : `/${owner}/${repo}`;
+  }
+  if (kind === "blob" || kind === "tree") {
+    return rest.length > 0
+      ? `/${owner}/${repo}/${rest.join("/")}`
+      : `/${owner}/${repo}`;
+  }
+  return `/${owner}/${repo}`;
+}

package/src/context/fetcher.ts

@@ -15,6 +15,7 @@ import { mcpSearchTool } from "../tools/mcp/search.ts";
 import type { ToolContext } from "../tools/tool.ts";
 import { type AnyToolDefinition, toAnthropicTool } from "../tools/tool.ts";
 import { logger } from "../utils/logger.ts";
+import { detectDriveFromUrl } from "./drives.ts";
 import { stripHtmlTags } from "./url-utils.ts";
 
 const MAX_CONTENT_BYTES = 500_000;
@@ -28,6 +29,8 @@ export interface FetchedContent {
   content: string;
   mimeType: string;
   sourceUrl: string;
+  drive: string;
+  path: string;
 }
 
 export class FetchFailureError extends Error {
@@ -176,7 +179,8 @@ async function runFetcherLoop(
 
   // Cache of full mcp_exec results keyed by tool_use_id.
   // The LLM only sees a truncated preview; on accept_content it references
-  // the id and the harness saves the captured content.
+  // the id and the harness saves the captured content. `server` is retained so
+  // we can attribute the save to a specific MCP service when routing to a drive.
   const execResults = new Map<
     string,
     { server: string; tool: string; content: string; mimeType: string }
@@ -289,11 +293,14 @@ async function runFetcherLoop(
       logger.dim(
         `  turn ${turn + 1}: accept_content: "${input.title}" (${cached.content.length} chars, ${mimeType}, from ${cached.server}/${cached.tool})`,
       );
+      const { drive, path } = detectDriveFromUrl(url, cached.server);
       return {
         title: input.title,
         content: cached.content.slice(0, MAX_CONTENT_BYTES),
         mimeType,
         sourceUrl: url,
+        drive,
+        path,
       };
     }
 
@@ -428,10 +435,13 @@ export async function httpFallback(url: string): Promise<FetchedContent> {
     ? "text/markdown"
     : contentType.split(";")[0] || "text/plain";
 
+  const { drive, path } = detectDriveFromUrl(url);
   return {
     title,
     content: text,
     mimeType,
     sourceUrl: url,
+    drive,
+    path,
   };
 }

package/src/context/ingest.ts

@@ -1,9 +1,10 @@
 import type { BotholomewConfig } from "../config/schemas.ts";
 import type { DbConnection } from "../db/connection.ts";
-import { getContextItem, getContextItemByPath } from "../db/context.ts";
+import { getContextItem, getContextItemById } from "../db/context.ts";
 import { createEmbedding, deleteEmbeddingsForItem } from "../db/embeddings.ts";
 import { logger } from "../utils/logger.ts";
 import { chunk } from "./chunker.ts";
+import { type DriveTarget, formatDriveRef } from "./drives.ts";
 import { embed as defaultEmbed } from "./embedder.ts";
 
 type IngestEmbedFn = (texts: string[]) => Promise<number[][]>;
@@ -12,7 +13,8 @@ export interface PreparedIngestion {
   itemId: string;
   title: string;
   description: string;
-  sourcePath: string | null;
+  drive: string;
+  path: string;
   chunks: { index: number; content: string }[];
   vectors: number[][];
 }
@@ -27,7 +29,7 @@ export async function prepareIngestion(
   config: Required<BotholomewConfig>,
   embedFn?: IngestEmbedFn,
 ): Promise<PreparedIngestion | null> {
-  const item = await getContextItem(conn, itemId);
+  const item = await getContextItemById(conn, itemId);
   if (!item) {
     logger.warn(`ingest: context item ${itemId} not found`);
     return null;
@@ -52,11 +54,12 @@ export async function prepareIngestion(
   const chunks = await chunk(item.content, item.mime_type, config);
   if (chunks.length === 0) return null;
 
+  const ref = formatDriveRef(item);
   const textsForEmbedding = chunks.map((c) => {
     const parts: string[] = [];
     if (item.title) parts.push(`Title: ${item.title}`);
     if (item.description) parts.push(`Description: ${item.description}`);
-    if (item.source_path) parts.push(`Source: ${item.source_path}`);
+    parts.push(`Source: ${ref}`);
     parts.push(c.content);
     return parts.join("\n");
   });
@@ -66,7 +69,8 @@ export async function prepareIngestion(
     itemId,
     title: item.title,
     description: item.description,
-    sourcePath: item.source_path,
+    drive: item.drive,
+    path: item.path,
     chunks,
     vectors,
   };
@@ -102,7 +106,6 @@ export async function storeIngestion(
         chunkContent: c.content,
         title: prepared.title,
         description: prepared.description,
-        sourcePath: prepared.sourcePath,
         embedding: v,
       });
     }
@@ -144,17 +147,17 @@ export async function ingestContextItem(
 }
 
 /**
- * Ingest a context item by its virtual path.
+ * Ingest a context item by its (drive, path) pair.
  */
 export async function ingestByPath(
   conn: DbConnection,
-  contextPath: string,
+  target: DriveTarget,
   config: Required<BotholomewConfig>,
   embedFn?: IngestEmbedFn,
 ): Promise<number> {
-  const item = await getContextItemByPath(conn, contextPath);
+  const item = await getContextItem(conn, target);
   if (!item) {
-    logger.warn(`ingest: no item at path ${contextPath}`);
+    logger.warn(`ingest: no item at ${formatDriveRef(target)}`);
     return 0;
   }
   return ingestContextItem(conn, item.id, config, embedFn);