botholomew 0.8.10 → 0.9.5

package/src/db/embeddings.ts

@@ -13,7 +13,6 @@ export interface Embedding {
   chunk_content: string | null;
   title: string;
   description: string;
-  source_path: string | null;
   embedding: number[];
   created_at: Date;
 }
@@ -29,7 +28,6 @@ interface EmbeddingRow {
   chunk_content: string | null;
   title: string;
   description: string;
-  source_path: string | null;
   embedding: number[] | null;
   created_at: string;
 }
@@ -42,7 +40,6 @@ function rowToEmbedding(row: EmbeddingRow): Embedding {
     chunk_content: row.chunk_content,
     title: row.title,
     description: row.description,
-    source_path: row.source_path,
     embedding: row.embedding ?? [],
     created_at: new Date(row.created_at),
   };
@@ -56,21 +53,19 @@ export async function createEmbedding(
     chunkContent: string | null;
     title: string;
     description?: string;
-    sourcePath?: string | null;
     embedding: number[];
   },
 ): Promise<Embedding> {
   const id = uuidv7();
   await conn.queryRun(
-    `INSERT INTO embeddings (id, context_item_id, chunk_index, chunk_content, title, description, source_path, embedding)
-     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8::FLOAT[${EMBEDDING_DIMENSION}])`,
+    `INSERT INTO embeddings (id, context_item_id, chunk_index, chunk_content, title, description, embedding)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7::FLOAT[${EMBEDDING_DIMENSION}])`,
     id,
     params.contextItemId,
     params.chunkIndex,
     params.chunkContent,
     params.title,
     params.description ?? "",
-    params.sourcePath ?? null,
     params.embedding,
   );
 
@@ -81,7 +76,6 @@ export async function createEmbedding(
     chunk_content: params.chunkContent,
     title: params.title,
     description: params.description ?? "",
-    source_path: params.sourcePath ?? null,
     embedding: params.embedding,
     created_at: new Date(),
   };
@@ -139,15 +133,19 @@ export async function searchEmbeddings(
   }));
 }
 
+export interface HybridSearchResult extends EmbeddingSearchResult {
+  drive: string | null;
+  path: string | null;
+}
+
 export async function hybridSearch(
   conn: DbConnection,
   query: string,
   queryEmbedding: number[],
   limit = 10,
-): Promise<EmbeddingSearchResult[]> {
+): Promise<HybridSearchResult[]> {
   const k = 60; // RRF constant
 
-  // Keyword search: match on chunk_content and title
   const keywordRows = await conn.queryAll<EmbeddingRow>(
     `SELECT * FROM embeddings
      WHERE chunk_content ILIKE '%' || ?1 || '%'
@@ -158,10 +156,8 @@ export async function hybridSearch(
 
   const keywordRanked = keywordRows.map(rowToEmbedding);
 
-  // Vector search via DuckDB VSS
   const vectorResults = await searchEmbeddings(conn, queryEmbedding, 100);
 
-  // Reciprocal rank fusion
   const scores = new Map<string, { embedding: Embedding; score: number }>();
 
   for (const [i, emb] of keywordRanked.entries()) {
@@ -187,8 +183,31 @@ export async function hybridSearch(
   const merged = Array.from(scores.values());
   merged.sort((a, b) => b.score - a.score);
 
-  return merged.slice(0, limit).map((entry) => ({
-    ...entry.embedding,
-    score: entry.score,
-  }));
+  const top = merged.slice(0, limit);
+  if (top.length === 0) return [];
+
+  // Look up drive + path from context_items for each surviving embedding
+  const itemIds = Array.from(
+    new Set(top.map((t) => t.embedding.context_item_id)),
+  );
+  const placeholders = itemIds.map((_, i) => `?${i + 1}`).join(", ");
+  const itemRows = await conn.queryAll<{
+    id: string;
+    drive: string;
+    path: string;
+  }>(
+    `SELECT id, drive, path FROM context_items WHERE id IN (${placeholders})`,
+    ...itemIds,
+  );
+  const itemIndex = new Map(itemRows.map((r) => [r.id, r]));
+
+  return top.map((entry) => {
+    const item = itemIndex.get(entry.embedding.context_item_id);
+    return {
+      ...entry.embedding,
+      score: entry.score,
+      drive: item?.drive ?? null,
+      path: item?.path ?? null,
+    };
+  });
 }

package/src/db/sql/13-drive-paths.sql

@@ -0,0 +1,49 @@
+-- Milestone 10: collapse `source_path` + `context_path` + `source_type` into a
+-- single `(drive, path)` identity pair. Pre-1.0, no backwards-compat promise —
+-- we wipe context_items + embeddings and have the user re-add their content.
+--
+-- DuckDB's ALTER TABLE support is thin (no SET NOT NULL, flaky DROP COLUMN with
+-- existing indexes), so this is a table rebuild. Order matters: drop indexes
+-- first, then the old tables, then recreate with the new shape.
+
+DELETE FROM embeddings;
+DELETE FROM context_items;
+
+DROP INDEX IF EXISTS idx_embeddings_cosine;
+DROP INDEX IF EXISTS idx_context_items_context_path;
+
+DROP TABLE embeddings;
+DROP TABLE context_items;
+
+CREATE TABLE context_items (
+  id TEXT PRIMARY KEY,
+  title TEXT NOT NULL,
+  description TEXT NOT NULL DEFAULT '',
+  content TEXT,
+  content_blob BLOB,
+  mime_type TEXT NOT NULL DEFAULT 'text/plain',
+  is_textual BOOLEAN NOT NULL DEFAULT true,
+  drive TEXT NOT NULL,
+  path TEXT NOT NULL,
+  indexed_at TEXT,
+  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
+  updated_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR)
+);
+
+CREATE UNIQUE INDEX idx_context_items_drive_path ON context_items(drive, path);
+
+CREATE TABLE embeddings (
+  id TEXT PRIMARY KEY,
+  context_item_id TEXT NOT NULL,
+  chunk_index INTEGER NOT NULL,
+  chunk_content TEXT,
+  title TEXT NOT NULL,
+  description TEXT NOT NULL DEFAULT '',
+  embedding FLOAT[1536],
+  created_at TEXT NOT NULL DEFAULT (current_timestamp::VARCHAR),
+  UNIQUE(context_item_id, chunk_index)
+);
+
+CREATE INDEX idx_embeddings_cosine ON embeddings USING HNSW (embedding) WITH (metric = 'cosine');
+
+CHECKPOINT;

package/src/tools/context/list-drives.ts

@@ -0,0 +1,36 @@
+import { z } from "zod";
+import { listDriveSummaries } from "../../db/context.ts";
+import type { ToolDefinition } from "../tool.ts";
+
+const inputSchema = z.object({});
+
+const outputSchema = z.object({
+  drives: z.array(
+    z.object({
+      drive: z.string(),
+      count: z.number(),
+    }),
+  ),
+  is_error: z.boolean(),
+  hint: z.string().optional(),
+});
+
+export const contextListDrivesTool = {
+  name: "context_list_drives",
+  description:
+    "List every drive that currently has content, with its item count. Use this to discover which values to pass as `drive` on other context tools (disk / url / agent / google-docs / github / …).",
+  group: "context",
+  inputSchema,
+  outputSchema,
+  execute: async (_input, ctx) => {
+    const drives = await listDriveSummaries(ctx.conn);
+    if (drives.length === 0) {
+      return {
+        drives: [],
+        is_error: false,
+        hint: "No context has been ingested yet. The user can run `botholomew context add <path-or-url>` to add content.",
+      };
+    }
+    return { drives, is_error: false };
+  },
+} satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;

package/src/tools/context/refresh.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { parseDriveRef } from "../../context/drives.ts";
 import { refreshContextItems } from "../../context/refresh.ts";
 import {
   type ContextItem,
@@ -10,17 +11,17 @@ import { buildContextTree } from "../dir/tree.ts";
 import type { ToolDefinition } from "../tool.ts";
 
 const inputSchema = z.object({
-  path: z
+  ref: z
     .string()
     .optional()
     .describe(
-      "Context path or ID of a single item, or a path prefix to refresh a subtree. Mutually exclusive with `all`.",
+      "UUID or 'drive:/path' of a single item, or 'drive:/prefix' to refresh a subtree. Mutually exclusive with `all`.",
     ),
   all: z
     .boolean()
     .optional()
     .describe(
-      "Refresh every item that has a source_path (file or URL). Mutually exclusive with `path`.",
+      "Refresh every item that has an external origin (drive != 'agent'). Mutually exclusive with `ref`.",
     ),
 });
 
@@ -35,9 +36,9 @@ const outputSchema = z.object({
   items: z.array(
     z.object({
       id: z.string(),
-      context_path: z.string(),
-      source_path: z.string(),
-      source_type: z.enum(["file", "url"]),
+      drive: z.string(),
+      path: z.string(),
+      ref: z.string(),
       status: z.enum(["updated", "unchanged", "missing", "error"]),
       error: z.string().optional(),
     }),
@@ -67,22 +68,22 @@ const empty = {
 export const contextRefreshTool = {
   name: "context_refresh",
   description:
-    "[[ bash equivalent command: curl ]] Re-read source files from disk / re-fetch source URLs, update stored content if it changed, and re-embed only changed items. Use `path` for a single item or subtree, or `all: true` for every sourced item. Items without a source_path are skipped. URL fetches use the project's MCPX client when available and fall back to plain HTTP.",
+    "[[ bash equivalent command: curl ]] Re-import items from their origin (disk / URL / MCP) and re-embed changed items. Use `ref` for a single item or subtree, or `all: true` for every non-agent item. URL fetches use the project's MCPX client when available and fall back to plain HTTP.",
   group: "context",
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
-    if (!input.path && !input.all) {
+    if (!input.ref && !input.all) {
       return {
         ...empty,
-        message: "Provide a `path` or set `all: true`.",
+        message: "Provide a `ref` or set `all: true`.",
         is_error: true,
       };
     }
-    if (input.path && input.all) {
+    if (input.ref && input.all) {
       return {
         ...empty,
-        message: "`path` and `all` are mutually exclusive.",
+        message: "`ref` and `all` are mutually exclusive.",
         is_error: true,
       };
     }
@@ -91,35 +92,46 @@ export const contextRefreshTool = {
     if (input.all) {
       items = await listContextItems(ctx.conn);
     } else {
-      const exact = await resolveContextItem(ctx.conn, input.path as string);
-      items = exact
-        ? [exact]
-        : await listContextItemsByPrefix(ctx.conn, input.path as string, {
-            recursive: true,
-          });
+      const ref = input.ref as string;
+      const exact = await resolveContextItem(ctx.conn, ref);
+      if (exact) {
+        items = [exact];
+      } else {
+        const parsed = parseDriveRef(ref);
+        items = parsed
+          ? await listContextItemsByPrefix(
+              ctx.conn,
+              parsed.drive,
+              parsed.path,
+              {
+                recursive: true,
+              },
+            )
+          : [];
+      }
     }
 
     if (items.length === 0) {
       return {
         ...empty,
-        message: `No context items match \`${input.path ?? "all"}\`.`,
+        message: `No context items match \`${input.ref ?? "all"}\`.`,
         is_error: true,
       };
     }
 
-    const sourced = items.filter((i) => i.source_path);
-    if (sourced.length === 0) {
+    const refreshable = items.filter((i) => i.drive !== "agent");
+    if (refreshable.length === 0) {
       return {
         ...empty,
         message:
-          "No matching items have a source_path — nothing to refresh. (Items created via `context write` are not sourced.)",
+          "No refreshable items — everything matched lives on drive=agent (agent-authored content has no external origin).",
         is_error: false,
       };
     }
 
     const result = await refreshContextItems(
       ctx.conn,
-      sourced,
+      refreshable,
       ctx.config,
       ctx.mcpxClient,
     );
@@ -135,7 +147,13 @@ export const contextRefreshTool = {
       parts.push("embeddings skipped (no OpenAI API key configured)");
     }
 
-    const { tree } = await buildContextTree(ctx.conn);
+    // For a single-ref refresh, render that ref's drive; for `all: true`,
+    // render the top-level drive summary so the scope of the tree matches
+    // the scope of the operation.
+    const treeDrive = input.all
+      ? undefined
+      : (result.items[0]?.drive ?? undefined);
+    const { tree } = await buildContextTree(ctx.conn, { drive: treeDrive });
 
     return {
       ...result,

package/src/tools/context/search.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { formatDriveRef } from "../../context/drives.ts";
 import { searchContextByKeyword } from "../../db/context.ts";
 import type { ToolDefinition } from "../tool.ts";
 
@@ -14,7 +15,9 @@ const outputSchema = z.object({
     z.object({
       id: z.string(),
       title: z.string(),
-      context_path: z.string(),
+      drive: z.string(),
+      path: z.string(),
+      ref: z.string(),
       content_preview: z.string(),
     }),
   ),
@@ -25,7 +28,7 @@ const outputSchema = z.object({
 export const contextSearchTool = {
   name: "context_search",
   description:
-    "[[ bash equivalent command: grep -r ]] Search context by keyword.",
+    "[[ bash equivalent command: grep -r ]] Search context by keyword across all drives.",
   group: "context",
   inputSchema,
   outputSchema,
@@ -39,7 +42,9 @@ export const contextSearchTool = {
       results: items.map((item) => ({
         id: item.id,
         title: item.title,
-        context_path: item.context_path,
+        drive: item.drive,
+        path: item.path,
+        ref: formatDriveRef(item),
         content_preview: (item.content ?? "").slice(0, 500),
       })),
       count: items.length,

package/src/tools/dir/create.ts

@@ -1,41 +1,44 @@
 import { z } from "zod";
+import { formatDriveRef } from "../../context/drives.ts";
 import { contextPathExists, createContextItem } from "../../db/context.ts";
 import type { ToolDefinition } from "../tool.ts";
 
 const inputSchema = z.object({
-  path: z.string().describe("Directory path to create"),
-  parents: z
-    .boolean()
-    .optional()
-    .describe("Create parent directories as needed"),
+  drive: z
+    .string()
+    .default("agent")
+    .describe("Drive to create the directory in (defaults to 'agent')"),
+  path: z.string().describe("Directory path to create (starts with /)"),
 });
 
 const outputSchema = z.object({
   created: z.boolean(),
-  path: z.string(),
+  ref: z.string(),
   is_error: z.boolean(),
 });
 
 export const contextCreateDirTool = {
   name: "context_create_dir",
   description:
-    "[[ bash equivalent command: mkdir -p ]] Create a directory in context.",
+    "[[ bash equivalent command: mkdir -p ]] Create a directory placeholder in context.",
   group: "context",
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
-    const exists = await contextPathExists(ctx.conn, input.path);
+    const target = { drive: input.drive, path: input.path };
+    const exists = await contextPathExists(ctx.conn, target);
     if (exists) {
-      return { created: false, path: input.path, is_error: false };
+      return { created: false, ref: formatDriveRef(target), is_error: false };
     }
 
     await createContextItem(ctx.conn, {
       title: input.path.split("/").filter(Boolean).pop() ?? input.path,
-      contextPath: input.path,
+      drive: target.drive,
+      path: target.path,
       mimeType: "inode/directory",
       isTextual: false,
     });
 
-    return { created: true, path: input.path, is_error: false };
+    return { created: true, ref: formatDriveRef(target), is_error: false };
   },
 } satisfies ToolDefinition<typeof inputSchema, typeof outputSchema>;

package/src/tools/dir/size.ts

@@ -11,6 +11,7 @@ function formatBytes(bytes: number): string {
 }
 
 const inputSchema = z.object({
+  drive: z.string().describe("Drive name"),
   path: z.string().optional().describe("Directory path (defaults to /)"),
   recursive: z
     .boolean()
@@ -27,13 +28,13 @@ const outputSchema = z.object({
 export const contextDirSizeTool = {
   name: "context_dir_size",
   description:
-    "[[ bash equivalent command: du -s ]] Get the total size of context items in a directory.",
+    "[[ bash equivalent command: du -s ]] Get the total size of context items under a drive/directory.",
   group: "context",
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
     const path = input.path ?? "/";
-    const items = await listContextItemsByPrefix(ctx.conn, path, {
+    const items = await listContextItemsByPrefix(ctx.conn, input.drive, path, {
       recursive: input.recursive !== false,
     });
 

package/src/tools/dir/tree.ts

@@ -1,16 +1,19 @@
 import { z } from "zod";
+import { formatDriveRef } from "../../context/drives.ts";
 import type { DbConnection } from "../../db/connection.ts";
 import {
   countContextItemsByPrefix,
   listContextItemsByPrefix,
+  listDriveSummaries,
 } from "../../db/context.ts";
 import type { ToolDefinition } from "../tool.ts";
 
-const DEFAULT_MAX_DEPTH = 3;
+const DEFAULT_MAX_DEPTH = 10;
 const DEFAULT_ITEMS_PER_DIR = 15;
 const HARD_FETCH_CAP = 1000;
 
 export interface BuildContextTreeOptions {
+  drive?: string;
   path?: string;
   maxDepth?: number;
   itemsPerDir?: number;
@@ -38,36 +41,67 @@ interface FileNode {
 
 type TreeEntry = DirNode | FileNode;
 
+/**
+ * Build a markdown tree for a single drive, or — when no drive is given — a
+ * top-level summary listing every drive with its item count.
+ */
 export async function buildContextTree(
   conn: DbConnection,
   options: BuildContextTreeOptions = {},
 ): Promise<BuildContextTreeResult> {
+  if (!options.drive) {
+    const summaries = await listDriveSummaries(conn);
+    if (summaries.length === 0) {
+      return {
+        tree: "(no drives — context is empty)",
+        total_items: 0,
+        truncated_dirs: [],
+        hint: "No context has been ingested yet. Use `context add` from the CLI to ingest files or URLs.",
+      };
+    }
+    const lines = [
+      "Drives:",
+      ...summaries.map(
+        (s) =>
+          `  ${s.drive}:/  (${s.count} ${s.count === 1 ? "item" : "items"})`,
+      ),
+    ];
+    const total = summaries.reduce((sum, s) => sum + s.count, 0);
+    return {
+      tree: lines.join("\n"),
+      total_items: total,
+      truncated_dirs: [],
+      hint: `Call context_tree with drive="<name>" to drill into a drive.`,
+    };
+  }
+
+  const drive = options.drive;
   const path = options.path ?? "/";
   const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
   const itemsPerDir = options.itemsPerDir ?? DEFAULT_ITEMS_PER_DIR;
   const normalizedPath = path.endsWith("/") ? path : `${path}/`;
+  const rootLabel = formatDriveRef({ drive, path });
 
-  const totalItems = await countContextItemsByPrefix(conn, path, {
+  const totalItems = await countContextItemsByPrefix(conn, drive, path, {
     recursive: true,
   });
 
   if (totalItems === 0) {
     return {
-      tree: `${path}\n  (empty)`,
+      tree: `${rootLabel}\n  (empty)`,
       total_items: 0,
       truncated_dirs: [],
       hint: "Directory is empty.",
     };
   }
 
-  const items = await listContextItemsByPrefix(conn, path, {
+  const items = await listContextItemsByPrefix(conn, drive, path, {
     recursive: true,
     limit: HARD_FETCH_CAP,
   });
 
-  // Build tree structure: dirs map child name -> child node
   const root: DirNode = {
-    name: path,
+    name: rootLabel,
     fullPath: path,
     isDir: true,
     children: [],
@@ -76,12 +110,11 @@ export async function buildContextTree(
   dirIndex.set(stripTrailingSlash(path), root);
 
   for (const item of items) {
-    const relative = item.context_path.slice(normalizedPath.length);
-    if (relative.length === 0) continue; // root itself, skip
+    const relative = item.path.slice(normalizedPath.length);
+    if (relative.length === 0) continue;
     const parts = relative.split("/").filter((p) => p.length > 0);
     const isExplicitDir = item.mime_type === "inode/directory";
 
-    // Walk segments, creating intermediate directories as needed
     let parentDir = root;
     let currentRel = "";
     for (let i = 0; i < parts.length; i++) {
@@ -116,7 +149,6 @@ export async function buildContextTree(
     }
   }
 
-  // Sort each directory's children: dirs first, then alphabetical
   for (const dir of dirIndex.values()) {
     dir.children.sort((a, b) => {
       if (a.isDir !== b.isDir) return a.isDir ? -1 : 1;
@@ -131,7 +163,7 @@ export async function buildContextTree(
   }> = [];
   const depthLimitedDirs: string[] = [];
 
-  const lines: string[] = [path];
+  const lines: string[] = [rootLabel];
 
   const render = (dir: DirNode, indent: string, currentDepth: number): void => {
     const children = dir.children;
@@ -185,15 +217,21 @@ export async function buildContextTree(
     tree: lines.join("\n"),
     total_items: totalItems,
     truncated_dirs: truncatedDirs,
-    hint: buildHint({ truncatedDirs, depthLimitedDirs, totalItems }),
+    hint: buildHint({ truncatedDirs, depthLimitedDirs, totalItems, drive }),
   };
 }
 
 const inputSchema = z.object({
+  drive: z
+    .string()
+    .optional()
+    .describe(
+      "Drive to explore (e.g. 'disk', 'agent'). Omit to list every drive with its item count — useful as a first call.",
+    ),
   path: z
     .string()
     .optional()
-    .describe("Root path for the tree (defaults to /)"),
+    .describe("Root path for the tree within the drive (defaults to /)"),
   max_depth: z
     .number()
     .int()
@@ -231,12 +269,13 @@ const outputSchema = z.object({
 export const contextTreeTool = {
   name: "context_tree",
   description:
-    "[[ bash equivalent command: tree ]] Explore your context filesystem with a bird's-eye view — shows many paths across nested directories in one call. Reach for this first when you need to discover what content exists before reading a specific file (context_read) or running a keyword search (context_search). Returns a markdown-style tree; tune max_depth and items_per_dir to bound output, or pass a deeper path to drill into a subtree.",
+    "[[ bash equivalent command: tree ]] Explore your context with a bird's-eye view. Call with no `drive` to list every drive; call with a drive (and optional path) to render a tree of that drive. Returns a markdown-style tree; tune max_depth and items_per_dir to bound output.",
   group: "context",
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
     const result = await buildContextTree(ctx.conn, {
+      drive: input.drive,
       path: input.path,
       maxDepth: input.max_depth,
       itemsPerDir: input.items_per_dir,
@@ -262,15 +301,16 @@ function buildHint(args: {
   truncatedDirs: Array<{ path: string; shown: number; total: number }>;
   depthLimitedDirs: string[];
   totalItems: number;
+  drive: string;
 }): string {
-  const { truncatedDirs, depthLimitedDirs } = args;
+  const { truncatedDirs, depthLimitedDirs, drive } = args;
   const parts: string[] = [];
 
   if (truncatedDirs.length > 0) {
     const first = truncatedDirs[0];
     if (first) {
       parts.push(
-        `${truncatedDirs.length} ${truncatedDirs.length === 1 ? "directory was" : "directories were"} capped by items_per_dir; raise items_per_dir or call context_tree with path="${first.path}".`,
+        `${truncatedDirs.length} ${truncatedDirs.length === 1 ? "directory was" : "directories were"} capped by items_per_dir; raise items_per_dir or call context_tree with drive="${drive}", path="${first.path}".`,
       );
     }
   }
@@ -279,7 +319,7 @@ function buildHint(args: {
     const first = depthLimitedDirs[0];
     if (first) {
       parts.push(
-        `${depthLimitedDirs.length} ${depthLimitedDirs.length === 1 ? "directory was" : "directories were"} not expanded due to max_depth; raise max_depth or call context_tree with path="${first}".`,
+        `${depthLimitedDirs.length} ${depthLimitedDirs.length === 1 ? "directory was" : "directories were"} not expanded due to max_depth; raise max_depth or call context_tree with drive="${drive}", path="${first}".`,
       );
     }
   }