membot 0.5.1 → 0.6.0

package/src/refresh/runner.ts

@@ -1,14 +1,16 @@
-import type { McpxClient } from "@evantahler/mcpx";
+import { join } from "node:path";
+import { FILES } from "../constants.ts";
 import type { AppContext } from "../context.ts";
 import { upsertBlob } from "../db/blobs.ts";
 import { insertChunksForVersion, rebuildFts } from "../db/chunks.ts";
-import { getCurrent, insertVersion, millisIso, updateRefreshStatus } from "../db/files.ts";
+import { type FetcherKind, getCurrent, insertVersion, millisIso, updateRefreshStatus } from "../db/files.ts";
 import { HelpfulError } from "../errors.ts";
 import { chunkDeterministic } from "../ingest/chunker.ts";
 import { convert } from "../ingest/converter/index.ts";
 import { describe } from "../ingest/describer.ts";
+import { BrowserPool } from "../ingest/downloaders/browser.ts";
 import { embed } from "../ingest/embedder.ts";
-import { fetchRemote, isMcpToolError } from "../ingest/fetcher.ts";
+import { fetchRemoteByDownloader } from "../ingest/fetcher.ts";
 import { mimeFromPath, readLocalFile, sha256Hex } from "../ingest/local-reader.ts";
 import { buildSearchText } from "../ingest/search-text.ts";
 
@@ -20,19 +22,20 @@ export interface RefreshOutcome {
 }
 
 /**
- * Refresh one logical_path. Re-reads its source (local stat+sha or remote
- * via the persisted mcpx invocation), and creates a new version only if
- * the source bytes changed. Always updates `refreshed_at` and
- * `last_refresh_status` on the row. Returns a per-path outcome — never
- * throws unless the path doesn't exist. The optional `onEmbedProgress`
- * callback is forwarded to the embedder so interactive callers (e.g. the
- * `refresh` operation) can drive a spinner during the slow phase.
+ * Refresh one logical_path. Re-reads its source (local stat+sha or
+ * remote via the persisted downloader name + the original URL), and
+ * creates a new version only if the source bytes changed. Always
+ * updates `refreshed_at` and `last_refresh_status` on the row. Returns
+ * a per-path outcome — never throws unless the path doesn't exist. The
+ * optional `onPhase` callback is forwarded to the embedder so
+ * interactive callers (e.g. the `refresh` operation) can drive a
+ * spinner during the slow phase.
  */
 export async function refreshOne(
 	ctx: AppContext,
 	logicalPath: string,
 	force = false,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<RefreshOutcome> {
 	const cur = await getCurrent(ctx.db, logicalPath);
 	if (!cur) {
@@ -49,10 +52,10 @@ export async function refreshOne(
 
 	try {
 		if (cur.source_type === "local") {
-			return await refreshLocal(ctx, cur, force, onEmbedProgress);
+			return await refreshLocal(ctx, cur, force, onPhase);
 		}
 		if (cur.source_type === "remote") {
-			return await refreshRemote(ctx, cur, force, onEmbedProgress);
+			return await refreshRemote(ctx, cur, force, onPhase);
 		}
 	} catch (err) {
 		const message = err instanceof Error ? err.message : String(err);
@@ -74,9 +77,8 @@ interface CurrentRow {
 	source_sha256: string | null;
 	mime_type: string | null;
 	fetcher: string | null;
-	fetcher_server: string | null;
-	fetcher_tool: string | null;
-	fetcher_args: Record<string, unknown> | null;
+	downloader: string | null;
+	downloader_args: Record<string, unknown> | null;
 	refresh_frequency_sec: number | null;
 }
 
@@ -85,7 +87,7 @@ async function refreshLocal(
 	ctx: AppContext,
 	cur: CurrentRow,
 	force: boolean,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<RefreshOutcome> {
 	if (!cur.source_path) {
 		throw new HelpfulError({
@@ -116,22 +118,28 @@ async function refreshLocal(
 			sourceMtimeMs: local.mtimeMs,
 			sourceSha: local.sha256,
 			fetcher: "local",
-			fetcherServer: null,
-			fetcherTool: null,
-			fetcherArgs: null,
+			downloader: null,
+			downloaderArgs: null,
 			refreshSec: cur.refresh_frequency_sec,
 		},
-		onEmbedProgress,
+		onPhase,
 	);
 	return { logical_path: cur.logical_path, status: "ok", new_version_id: versionId };
 }
 
-/** Remote refresh: replay the persisted mcpx invocation, or plain HTTP. */
+/**
+ * Remote refresh: replay the persisted downloader against the original
+ * URL. Each downloader is deterministic (no LLM, no agent loop), so a
+ * row with `downloader='google-docs'` always re-runs the Google Docs
+ * downloader; rows from older membot versions whose `downloader` is
+ * NULL fall back to URL-based dispatch (the registry's `findDownloader`
+ * picks the right handler from the URL itself).
+ */
 async function refreshRemote(
 	ctx: AppContext,
 	cur: CurrentRow,
 	force: boolean,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<RefreshOutcome> {
 	if (!cur.source_path) {
 		throw new HelpfulError({
@@ -140,7 +148,14 @@ async function refreshRemote(
 			hint: "Inspect with `membot info` and consider re-ingesting.",
 		});
 	}
-	const fetched = await replayFetch(cur, ctx.mcpx);
+	const userDataDir = join(ctx.dataDir, FILES.BROWSER_PROFILE);
+	const pool = new BrowserPool({ userDataDir });
+	let fetched: Awaited<ReturnType<typeof fetchRemoteByDownloader>>;
+	try {
+		fetched = await fetchRemoteByDownloader(cur.downloader, cur.source_path, pool, ctx.config);
+	} finally {
+		await pool.dispose();
+	}
 
 	if (!force && cur.source_sha256 === fetched.sha256) {
 		await updateRefreshStatus(ctx.db, cur.logical_path, cur.version_id, {
@@ -161,93 +176,16 @@ async function refreshRemote(
 			sourcePath: cur.source_path,
 			sourceMtimeMs: null,
 			sourceSha: fetched.sha256,
-			fetcher: cur.fetcher === "mcpx" ? "mcpx" : "http",
-			fetcherServer: fetched.fetcherServer,
-			fetcherTool: fetched.fetcherTool,
-			fetcherArgs: fetched.fetcherArgs,
+			fetcher: "downloader",
+			downloader: fetched.downloader,
+			downloaderArgs: fetched.downloaderArgs,
 			refreshSec: cur.refresh_frequency_sec,
 		},
-		onEmbedProgress,
+		onPhase,
 	);
 	return { logical_path: cur.logical_path, status: "ok", new_version_id: versionId };
 }
 
-/**
- * Re-fetch a remote source. When the row recorded an mcpx invocation,
- * call it directly with the same args (no agent re-routing); otherwise
- * fall back to plain HTTP. The choice is deterministic — same row always
- * produces the same fetch path.
- */
-async function replayFetch(
-	cur: CurrentRow,
-	mcpx: McpxClient | null,
-): Promise<{
-	bytes: Uint8Array;
-	sha256: string;
-	mimeType: string;
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
-}> {
-	if (cur.fetcher === "mcpx" && cur.fetcher_server && cur.fetcher_tool && mcpx) {
-		const args = cur.fetcher_args ?? {};
-		const result = await mcpx.exec(cur.fetcher_server, cur.fetcher_tool, args);
-		if (isMcpToolError(result)) {
-			const detail = extractText(result).trim();
-			throw new HelpfulError({
-				kind: "network_error",
-				message: `mcpx tool ${cur.fetcher_server}/${cur.fetcher_tool} returned isError=true${detail ? `: ${detail}` : ""}`,
-				hint: `Re-add with a working fetcher: \`membot remove ${cur.logical_path}\` then \`membot add ${cur.source_path} --fetcher http\` (or another --fetcher hint).`,
-			});
-		}
-		const text = extractText(result);
-		const bytes = new TextEncoder().encode(text);
-		return {
-			bytes,
-			sha256: sha256Hex(bytes),
-			mimeType: "text/markdown",
-			fetcherServer: cur.fetcher_server,
-			fetcherTool: cur.fetcher_tool,
-			fetcherArgs: args,
-		};
-	}
-	const r = await fetchRemote(cur.source_path ?? "", { hint: "http" });
-	return {
-		bytes: r.bytes,
-		sha256: r.sha256,
-		mimeType: r.mimeType,
-		fetcherServer: null,
-		fetcherTool: null,
-		fetcherArgs: null,
-	};
-}
-
-/** Pull a string out of whatever shape an mcpx tool happens to return. */
-function extractText(result: unknown): string {
-	if (typeof result === "string") return result;
-	if (result && typeof result === "object") {
-		const r = result as Record<string, unknown>;
-		if (typeof r.text === "string") return r.text;
-		if (typeof r.content === "string") return r.content;
-		if (typeof r.markdown === "string") return r.markdown;
-		if (Array.isArray(r.content)) {
-			const out: string[] = [];
-			for (const c of r.content) {
-				if (c && typeof c === "object") {
-					const inner = c as Record<string, unknown>;
-					if (typeof inner.text === "string") out.push(inner.text);
-				}
-			}
-			if (out.length > 0) return out.join("\n\n");
-		}
-	}
-	try {
-		return JSON.stringify(result);
-	} catch {
-		return "";
-	}
-}
-
 interface PipelineParams {
 	logicalPath: string;
 	bytes: Uint8Array;
@@ -257,10 +195,9 @@ interface PipelineParams {
 	sourcePath: string | null;
 	sourceMtimeMs: number | null;
 	sourceSha: string;
-	fetcher: "local" | "http" | "mcpx";
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
+	fetcher: FetcherKind;
+	downloader: string | null;
+	downloaderArgs: Record<string, unknown> | null;
 	refreshSec: number | null;
 }
 
@@ -273,8 +210,9 @@ interface PipelineParams {
 async function runPipelineForRefresh(
 	ctx: AppContext,
 	p: PipelineParams,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<string> {
+	onPhase?.("storing blob");
 	await upsertBlob(ctx.db, {
 		sha256: p.sourceSha,
 		mime_type: p.mime,
@@ -282,12 +220,17 @@ async function runPipelineForRefresh(
 		bytes: p.bytes,
 	});
 
+	onPhase?.("converting");
 	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm);
 	const markdown = conversion.markdown;
+	onPhase?.("describing");
 	const description = await describe(p.logicalPath, p.mime, markdown, ctx.config.llm);
+	onPhase?.("chunking");
 	const chunks = chunkDeterministic(markdown, ctx.config.chunker);
 	const searchTexts = chunks.map((c) => buildSearchText(p.logicalPath, description, c.content));
-	const embeddings = await embed(searchTexts, ctx.config.embedding_model, { onProgress: onEmbedProgress });
+	const embeddings = await embed(searchTexts, ctx.config.embedding_model, {
+		onProgress: (done, total) => onPhase?.(`embedding ${done}/${total}`),
+	});
 
 	const versionId = millisIso(Date.now());
 	const contentSha = sha256Hex(new TextEncoder().encode(markdown));
@@ -305,15 +248,16 @@ async function runPipelineForRefresh(
 		mime_type: p.mime,
 		size_bytes: p.bytes.byteLength,
 		fetcher: p.fetcher,
-		fetcher_server: p.fetcherServer,
-		fetcher_tool: p.fetcherTool,
-		fetcher_args: p.fetcherArgs,
+		downloader: p.downloader,
+		downloader_args: p.downloaderArgs,
 		refresh_frequency_sec: p.refreshSec,
 		refreshed_at: new Date().toISOString(),
 		last_refresh_status: "ok",
 		change_note: "refresh: source updated",
 	});
 
+	onPhase?.("persisting");
+
 	await insertChunksForVersion(
 		ctx.db,
 		p.logicalPath,
@@ -326,6 +270,7 @@ async function runPipelineForRefresh(
 		})),
 	);
 
+	onPhase?.("indexing");
 	await rebuildFts(ctx.db);
 	return versionId;
 }

package/src/types/text-modules.d.ts

@@ -7,3 +7,8 @@ declare module "*.mdc" {
 	const content: string;
 	export default content;
 }
+
+declare module "*.mustache" {
+	const content: string;
+	export default content;
+}

package/patches/@evantahler%2Fmcpx@0.21.4.patch

@@ -1,51 +0,0 @@
-diff --git a/src/search/onnx-wasm-paths.ts b/src/search/onnx-wasm-paths.ts
---- a/src/search/onnx-wasm-paths.ts
-+++ b/src/search/onnx-wasm-paths.ts
-@@ -1,31 +1,16 @@
--// Embed the onnxruntime-web WASM runtime files into the compiled binary
--// (`bun build --compile`) so they survive in a single-binary distribution
--// where the user has no node_modules.
--//
--// This file is loaded **dynamically** by semantic.ts. The relative paths
--// only resolve in the local repo / compiled binary; for npm/bun-installed
--// mcpx the parent directory layout is different (deps are hoisted), the
--// dynamic import throws, and we fall back to letting transformers.js
--// load WASM via its default mechanism — which works fine because in
--// that environment node_modules exists and onnxruntime-web is reachable
--// through normal module resolution.
--
--// The relative `../../node_modules/...` paths only resolve from the local repo
--// layout (and inside `bun build --compile`). When this file is shipped via npm,
--// deps are hoisted, so consumer `tsc` runs hit TS2307. The `ts-ignore` directive
--// below silences that for consumers; we avoid the stricter `expect-error` form
--// because in the local repo the path resolves fine and there would be no error
--// to expect. At runtime the dynamic import in semantic.ts is wrapped in
--// try/catch and falls back to transformers.js's default WASM loader (issue #85).
--// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore per comment above
--// @ts-ignore - dynamic-only import
--import wasmMjsPath from "../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.mjs" with {
--	type: "file",
--};
--// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore per comment above
--// @ts-ignore - dynamic-only import
--import wasmBinPath from "../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.wasm" with {
--	type: "file",
--};
--
--export { wasmBinPath, wasmMjsPath };
-+// PATCHED (membot): point mcpx's onnx-wasm-paths at the onnxruntime-web installed
-+// at the top of membot's node_modules. Upstream's `../../node_modules/...` only
-+// resolves in mcpx's standalone repo layout; for consumers we walk up 4 levels:
-+// node_modules/@evantahler/mcpx/src/search → node_modules → onnxruntime-web.
-+// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore — relative path only resolves at runtime in consumer layout
-+// @ts-ignore - dynamic-only import
-+import wasmMjsPath from "../../../../onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.mjs" with {
-+	type: "file",
-+};
-+// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore — relative path only resolves at runtime in consumer layout
-+// @ts-ignore - dynamic-only import
-+import wasmBinPath from "../../../../onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.wasm" with {
-+	type: "file",
-+};
-+
-+export { wasmBinPath, wasmMjsPath };

package/src/commands/mcpx.ts

@@ -1,112 +0,0 @@
-import { createRequire } from "node:module";
-import { fileURLToPath } from "node:url";
-import type { Command } from "commander";
-import { logger } from "../output/logger.ts";
-
-const require = createRequire(import.meta.url);
-
-/**
- * Resolve the path to the bundled mcpx CLI entrypoint. We spawn it as a
- * child process rather than calling its functions directly so that the
- * upstream's argv parsing, output formatting, and config conventions stay
- * authoritative — `membot mcpx <subcmd>` behaves identically to the user
- * running `mcpx <subcmd>` themselves, just with our project's `--config`
- * resolution layered on top when applicable.
- */
-const MCPX_CLI = fileURLToPath(import.meta.resolve("@evantahler/mcpx/cli"));
-
-/**
- * Forward an argv slice to the bundled mcpx CLI. Inherits stdio so prompts
- * and pretty output flow through as if mcpx were called directly. Exits
- * the parent process with the child's status code on failure.
- */
-export async function runMcpx(args: string[]): Promise<void> {
-	const proc = Bun.spawn(["bun", MCPX_CLI, ...args], {
-		stdout: "inherit",
-		stderr: "inherit",
-		stdin: "inherit",
-	});
-	const code = await proc.exited;
-	if (code !== 0) process.exit(code);
-}
-
-/**
- * Pull the verbatim argv tokens that follow `mcpx` in `process.argv`. We
- * forward them unmodified so flags (`--help`, `-c`, etc.) reach the
- * upstream CLI exactly as the user typed them.
- */
-function getRawMcpxArgs(): string[] {
-	const idx = process.argv.indexOf("mcpx");
-	return idx === -1 ? [] : process.argv.slice(idx + 1);
-}
-
-const PASSTHROUGH_SUBCOMMANDS: ReadonlyArray<[name: string, desc: string]> = [
-	["servers", "List configured MCP server names"],
-	["info", "Show server overview or schema for a specific tool"],
-	["search", "Search tools by keyword and/or semantic similarity"],
-	["exec", "Execute a tool call"],
-	["add", "Add an MCP server"],
-	["remove", "Remove an MCP server"],
-	["ping", "Check connectivity to MCP servers"],
-	["auth", "Authenticate with an HTTP MCP server"],
-	["deauth", "Remove stored authentication for a server"],
-	["resource", "List resources for a server, or read a specific resource"],
-	["prompt", "List prompts for a server, or get a specific prompt"],
-	["task", "Manage async tool tasks (list, get, result, cancel)"],
-	["index", "Build the search index from all configured servers"],
-];
-
-/**
- * Register `membot mcpx <subcommand>` for every passthrough subcommand on
- * the upstream CLI. `--help` and unknown options are forwarded so users
- * always get authoritative mcpx help text.
- */
-export function registerMcpxCommand(program: Command): void {
-	const mcpx = program.command("mcpx").description("Forward to the bundled mcpx CLI for managing MCP servers");
-
-	const verifyVersion = (() => {
-		try {
-			const ourPkg = require("../../package.json") as { dependencies: Record<string, string> };
-			const mcpxPkg = require("@evantahler/mcpx/package.json") as { version: string };
-			const declared = ourPkg.dependencies["@evantahler/mcpx"];
-			if (!declared) return true;
-			return (
-				mcpxPkg.version === declared ||
-				declared.startsWith(mcpxPkg.version) ||
-				mcpxPkg.version.startsWith(declared.replace(/^[\^~]/, ""))
-			);
-		} catch {
-			return true;
-		}
-	})();
-	if (!verifyVersion) {
-		logger.warn("@evantahler/mcpx version mismatch — `membot mcpx` may behave unexpectedly.");
-	}
-
-	for (const [name, description] of PASSTHROUGH_SUBCOMMANDS) {
-		mcpx
-			.command(name)
-			.description(description)
-			.allowUnknownOption(true)
-			.helpOption(false)
-			.argument("[args...]", "arguments forwarded to mcpx")
-			.action(async () => {
-				await runMcpx(getRawMcpxArgs());
-			});
-	}
-
-	// Upstream mcpx's "list" is the default action when invoked with no
-	// subcommand — not a registered subcommand — so strip the "list" token
-	// before forwarding.
-	mcpx
-		.command("list")
-		.description("List all tools, resources, and prompts across all configured servers")
-		.allowUnknownOption(true)
-		.helpOption(false)
-		.argument("[args...]", "arguments forwarded to mcpx")
-		.action(async () => {
-			const raw = getRawMcpxArgs();
-			const args = raw[0] === "list" ? raw.slice(1) : raw;
-			await runMcpx(args);
-		});
-}