botholomew 0.8.10 → 0.9.4

package/src/context/refresh.ts

@@ -2,6 +2,7 @@ import type { McpxClient } from "@evantahler/mcpx";
 import type { BotholomewConfig } from "../config/schemas.ts";
 import type { DbConnection } from "../db/connection.ts";
 import { type ContextItem, updateContextItem } from "../db/context.ts";
+import { formatDriveRef } from "./drives.ts";
 import { fetchUrl } from "./fetcher.ts";
 import {
   type PreparedIngestion,
@@ -13,9 +14,9 @@ export type RefreshItemStatus = "updated" | "unchanged" | "missing" | "error";
 
 export interface RefreshItemResult {
   id: string;
-  context_path: string;
-  source_path: string;
-  source_type: "file" | "url";
+  drive: string;
+  path: string;
+  ref: string;
   status: RefreshItemStatus;
   error?: string;
 }
@@ -40,9 +41,16 @@ export interface RefreshOptions {
 type IngestEmbedFn = (texts: string[]) => Promise<number[][]>;
 
 /**
- * Refresh a batch of context items: re-read source (file or URL), diff, update
- * content, and re-embed only the items that changed. Side-effect free on the
- * outside world — the caller owns logging and spinners.
+ * Refresh a batch of context items: re-read from origin, diff, update
+ * content, and re-embed only the items that changed.
+ *
+ * Dispatches on `drive`:
+ *   disk  → read from filesystem
+ *   agent → skip (no external origin)
+ *   other → re-fetch as a URL (the path is either a full URL for `url` drive
+ *           or an origin-specific identifier that fetchUrl can re-derive via
+ *           the MCP agent; for now this only refreshes items stored under
+ *           `url:/<full-url>`)
  */
 export async function refreshContextItems(
   conn: DbConnection,
@@ -52,36 +60,44 @@ export async function refreshContextItems(
   opts: RefreshOptions = {},
   embedFn?: IngestEmbedFn,
 ): Promise<RefreshResult> {
-  const sourced = items.filter(
-    (i): i is ContextItem & { source_path: string } => !!i.source_path,
-  );
+  const refreshable = items.filter((i) => i.drive !== "agent");
 
   const results: RefreshItemResult[] = [];
   const toReembed: string[] = [];
 
-  // Phase 1: read each source, diff against stored content, update when changed.
-  for (const [idx, item] of sourced.entries()) {
-    opts.onItemProgress?.(idx, sourced.length);
+  for (const [idx, item] of refreshable.entries()) {
+    opts.onItemProgress?.(idx, refreshable.length);
     const base = {
       id: item.id,
-      context_path: item.context_path,
-      source_path: item.source_path,
-      source_type: item.source_type,
+      drive: item.drive,
+      path: item.path,
+      ref: formatDriveRef(item),
     };
 
     try {
       let content: string;
 
-      if (item.source_type === "url") {
-        const fetched = await fetchUrl(item.source_path, config, mcpxClient);
-        content = fetched.content;
-      } else {
-        const bunFile = Bun.file(item.source_path);
+      if (item.drive === "disk") {
+        const bunFile = Bun.file(item.path);
         if (!(await bunFile.exists())) {
           results.push({ ...base, status: "missing" });
           continue;
         }
         content = await bunFile.text();
+      } else if (item.drive === "url") {
+        const url = item.path.startsWith("/") ? item.path.slice(1) : item.path;
+        const fetched = await fetchUrl(url, config, mcpxClient);
+        content = fetched.content;
+      } else {
+        // Service-specific drives (google-docs, github, etc.) — only
+        // refreshable when the original URL can be reconstructed. For now,
+        // we punt: mark as error so the user knows to re-add from URL.
+        results.push({
+          ...base,
+          status: "error",
+          error: `Refresh not implemented for drive '${item.drive}' — re-add from the original URL.`,
+        });
+        continue;
       }
 
       if (content === item.content) {
@@ -100,17 +116,16 @@ export async function refreshContextItems(
       });
     }
   }
-  opts.onItemProgress?.(sourced.length, sourced.length);
+  opts.onItemProgress?.(refreshable.length, refreshable.length);
 
   const updated = results.filter((r) => r.status === "updated").length;
   const unchanged = results.filter((r) => r.status === "unchanged").length;
   const missing = results.filter((r) => r.status === "missing").length;
 
-  // Phase 2: re-embed changed items. Skip cleanly if no OpenAI key.
   const hasEmbedder = !!embedFn || !!config.openai_api_key;
   if (toReembed.length === 0 || !hasEmbedder) {
     return {
-      checked: sourced.length,
+      checked: refreshable.length,
       updated,
       unchanged,
       missing,
@@ -147,7 +162,7 @@ export async function refreshContextItems(
   }
 
   return {
-    checked: sourced.length,
+    checked: refreshable.length,
     updated,
     unchanged,
     missing,

package/src/context/url-utils.ts

@@ -10,29 +10,6 @@ export function isUrl(input: string): boolean {
   }
 }
 
-/**
- * Derives a virtual context path from a URL.
- * Example: `https://docs.google.com/document/d/abc123/edit` → `/{prefix}/docs.google.com/document-d-abc123.md`
- */
-export function urlToContextPath(url: string, prefix: string): string {
-  const parsed = new URL(url);
-  const hostname = parsed.hostname;
-  const pathname = parsed.pathname
-    .replace(/\/+$/, "") // strip trailing slashes
-    .replace(/^\/+/, "") // strip leading slashes
-    .replace(/[^a-zA-Z0-9\-_.]/g, "-") // slugify
-    .replace(/-{2,}/g, "-"); // collapse repeated dashes
-
-  const slug = pathname ? `${hostname}/${pathname}` : hostname;
-  const full = `${prefix.replace(/\/+$/, "")}/${slug}.md`;
-
-  if (full.length > 120) {
-    return `${full.slice(0, 117 - 3)}.md`;
-  }
-
-  return full;
-}
-
 /**
  * Strips HTML tags from a string, removing script/style blocks first,
  * then all remaining tags, and collapsing whitespace.