membot 0.4.2 → 0.5.1

package/src/ingest/fetcher.ts

@@ -1,6 +1,9 @@
+import type { LlmConfig } from "../config/schemas.ts";
 import { DEFAULTS } from "../constants.ts";
 import { asHelpful, HelpfulError } from "../errors.ts";
 import { logger } from "../output/logger.ts";
+import type { AgentMcpxAdapter } from "./agent-fetcher.ts";
+import { agentFetch } from "./agent-fetcher.ts";
 import { sha256Hex } from "./local-reader.ts";
 
 export interface FetchedRemote {
@@ -14,38 +17,40 @@ export interface FetchedRemote {
 	sourceUrl: string;
 }
 
-export interface McpxToolDescriptor {
-	server: string;
-	tool: { name: string; description?: string };
-}
-
-export interface McpxSearchHit {
-	server: string;
-	tool: { name: string; description?: string };
-	score?: number;
-}
-
 export interface FetchOptions {
 	/**
 	 * User-provided hint. Free-form keyword (e.g. "firecrawl", "github",
 	 * "google-docs", "http"). Special-cased: "http" forces plain fetch.
-	 * Otherwise the hint is used as a search query against the live
-	 * mcpx tool catalog — we never hardcode server names.
+	 * Otherwise the hint is passed verbatim to the agent loop as extra
+	 * guidance about which provider to prefer.
 	 */
 	hint?: string;
-	/** Live mcpx adapter. Use listTools/search/exec to find a fetcher on the fly. */
-	mcpx?: {
-		exec(server: string, tool: string, args: Record<string, unknown>): Promise<unknown>;
-		listTools(): Promise<McpxToolDescriptor[]>;
-		search?(query: string): Promise<McpxSearchHit[]>;
-	} | null;
+	/** Live mcpx adapter the agent loop drives via search/list/info/exec. */
+	mcpx?: AgentMcpxAdapter | null;
+	/**
+	 * LLM config. The agent loop needs an Anthropic key; without one the
+	 * mcpx path is skipped and we fall back to plain HTTP.
+	 */
+	llm?: LlmConfig;
 }
 
 /**
- * Fetch a remote URL, preferring an mcpx-managed server (Firecrawl, Google
- * Docs, GitHub, …) for known providers and falling back to a plain `fetch`
- * otherwise. The chosen invocation (server/tool/args) is returned alongside
- * the bytes so the caller can persist it on the row for replay-on-refresh.
+ * Fetch a remote URL.
+ *
+ * - `--fetcher http` (or no mcpx, or no LLM key) → plain HTTP.
+ * - Otherwise → multi-turn agent loop: Claude is given mcpx tools
+ *   (search/list/info/exec) and decides how to retrieve the URL,
+ *   including multi-step flows (start a job → poll → download).
+ *   The agent's selected mcp_exec invocation is recorded on the
+ *   returned row so refresh can replay it deterministically without
+ *   another agent round-trip.
+ *
+ * If the agent decides plain HTTP is the right call (`request_http_fallback`,
+ * no tool calls, max turns) we transparently fall through to `httpFetch`.
+ * If the agent reports an actionable failure, we surface that as a
+ * `HelpfulError`. If mcpx is configured but the LLM key is missing AND
+ * the HTTP fallback also fails, we surface an `auth_error` naming the env
+ * var so users see the real cause instead of a misleading 401.
  */
 export async function fetchRemote(url: string, options: FetchOptions = {}): Promise<FetchedRemote> {
 	const mcpx = options.mcpx;
@@ -54,8 +59,46 @@ export async function fetchRemote(url: string, options: FetchOptions = {}): Prom
 	if (hint === "http") return httpFetch(url);
 	if (!mcpx) return httpFetch(url);
 
-	const tried = await tryMcpx(url, mcpx, hint);
-	if (tried) return tried;
+	const apiKey = options.llm?.anthropic_api_key?.trim();
+	if (!apiKey) {
+		// No way to drive the agent. Try HTTP; if that fails, the user
+		// almost certainly wanted mcpx — surface a clear key-missing error.
+		try {
+			return await httpFetch(url);
+		} catch (err) {
+			if (err instanceof HelpfulError && err.kind === "network_error") {
+				throw new HelpfulError({
+					kind: "auth_error",
+					message: `${url} couldn't be fetched directly (${err.message}). Membot has mcpx configured, but routing through it requires Claude to translate the URL into the right tool arguments — and ANTHROPIC_API_KEY isn't set.`,
+					hint: `Set ANTHROPIC_API_KEY in your environment (or under llm.anthropic_api_key in ~/.membot/config.json), then retry. To force the HTTP path explicitly, run \`membot add ${url} --fetcher http\`.`,
+				});
+			}
+			throw err;
+		}
+	}
+
+	let outcome: Awaited<ReturnType<typeof agentFetch>>;
+	try {
+		outcome = await agentFetch({ url, mcpx, llm: options.llm!, hint });
+	} catch (err) {
+		if (err instanceof HelpfulError) throw err;
+		logger.warn(`agent-fetch failed (${err instanceof Error ? err.message : String(err)}) — falling back to HTTP`);
+		return httpFetch(url);
+	}
+
+	if (outcome.kind === "accepted") {
+		return {
+			bytes: outcome.result.bytes,
+			sha256: outcome.result.sha256,
+			mimeType: outcome.result.mimeType,
+			fetcher: "mcpx",
+			fetcherServer: outcome.result.fetcherServer,
+			fetcherTool: outcome.result.fetcherTool,
+			fetcherArgs: outcome.result.fetcherArgs,
+			sourceUrl: url,
+		};
+	}
+	logger.debug(`agent-fetch fell back to HTTP: ${outcome.reason}`);
 	return httpFetch(url);
 }
 
@@ -71,7 +114,7 @@ async function httpFetch(url: string): Promise<FetchedRemote> {
 		throw asHelpful(
 			err,
 			`while fetching ${url}`,
-			`Check your network and that ${url} is reachable. For mcpx-managed sources (gdocs/github/firecrawl), set --fetcher firecrawl etc.`,
+			`Check your network and that ${url} is reachable. For mcpx-managed sources (gdocs/github/firecrawl), set ANTHROPIC_API_KEY so membot can drive an mcpx tool.`,
 			"network_error",
 		);
 	}
@@ -79,7 +122,7 @@ async function httpFetch(url: string): Promise<FetchedRemote> {
 		throw new HelpfulError({
 			kind: "network_error",
 			message: `HTTP ${resp.status} ${resp.statusText}: ${url}`,
-			hint: "Verify the URL is reachable and not gated behind auth. For private docs use mcpx via --fetcher.",
+			hint: "Verify the URL is reachable and not gated behind auth. For private docs use mcpx (set ANTHROPIC_API_KEY).",
 		});
 	}
 	const bytes = new Uint8Array(await resp.arrayBuffer());
@@ -98,183 +141,13 @@ async function httpFetch(url: string): Promise<FetchedRemote> {
 }
 
 /**
- * Attempt to fetch via mcpx by discovering a suitable tool at runtime.
- *
- * Strategy:
- *   1. If the user passed a hint, search for it via mcpx.search() (semantic
- *      tool search over the live catalog). The hint is the user's free-text
- *      label for which provider they want — we never assume server names.
- *   2. Otherwise, fall back to a host-based search query (e.g. URL host
- *      "github.com" → search for "github fetch markdown").
- *   3. From the returned candidates, prefer tools whose name or description
- *      signals markdown output. Failing that, the first tool that takes a
- *      URL-shaped argument.
- *   4. Execute the tool with `{ url, format: "markdown" }`-shaped args.
- *      If exec fails, return null so the caller falls back to plain HTTP.
- */
-async function tryMcpx(
-	url: string,
-	mcpx: NonNullable<FetchOptions["mcpx"]>,
-	hint: string | undefined,
-): Promise<FetchedRemote | null> {
-	const candidates = await discoverCandidates(url, mcpx, hint);
-	if (candidates.length === 0) return null;
-
-	const chosen = pickTool(candidates);
-	if (!chosen) return null;
-
-	const args = buildArgs(chosen.tool.name, url);
-	let result: unknown;
-	try {
-		result = await mcpx.exec(chosen.server, chosen.tool.name, args);
-	} catch (err) {
-		logger.warn(
-			`mcpx: ${chosen.server}/${chosen.tool.name} failed (${err instanceof Error ? err.message : String(err)})`,
-		);
-		return null;
-	}
-
-	const text = extractText(result);
-	if (!text || text.trim().length === 0) return null;
-	const bytes = new TextEncoder().encode(text);
-	return {
-		bytes,
-		sha256: sha256Hex(bytes),
-		mimeType: "text/markdown",
-		fetcher: "mcpx",
-		fetcherServer: chosen.server,
-		fetcherTool: chosen.tool.name,
-		fetcherArgs: args,
-		sourceUrl: url,
-	};
-}
-
-/**
- * Build a list of candidate fetcher tools by querying mcpx's live catalog.
- * Tries semantic search first (using the hint or the URL's host as the
- * query) then falls back to listing all tools and filtering by name. Never
- * hardcodes a server name — the catalog is the source of truth.
- */
-async function discoverCandidates(
-	url: string,
-	mcpx: NonNullable<FetchOptions["mcpx"]>,
-	hint: string | undefined,
-): Promise<McpxToolDescriptor[]> {
-	const host = safeHost(url);
-	const queries = buildQueries(hint, host);
-
-	if (mcpx.search) {
-		for (const q of queries) {
-			try {
-				const hits = await mcpx.search(q);
-				if (hits.length > 0) {
-					return hits.slice(0, 5).map((h) => ({ server: h.server, tool: h.tool }));
-				}
-			} catch (err) {
-				logger.debug(`mcpx: search(${q}) failed (${err instanceof Error ? err.message : String(err)})`);
-			}
-		}
-	}
-
-	let tools: McpxToolDescriptor[];
-	try {
-		tools = await mcpx.listTools();
-	} catch (err) {
-		logger.debug(`mcpx: listTools failed (${err instanceof Error ? err.message : String(err)})`);
-		return [];
-	}
-
-	const lowercaseHaystack = (t: McpxToolDescriptor) =>
-		`${t.server} ${t.tool.name} ${t.tool.description ?? ""}`.toLowerCase();
-
-	if (hint) {
-		const needle = hint.toLowerCase();
-		const matched = tools.filter((t) => lowercaseHaystack(t).includes(needle));
-		if (matched.length > 0) return matched;
-	}
-
-	if (host) {
-		const tokens = host.split(".");
-		const matched = tools.filter((t) => tokens.some((tok) => tok.length > 2 && lowercaseHaystack(t).includes(tok)));
-		if (matched.length > 0) return matched;
-	}
-
-	// Fall back to any tool that looks like a URL fetcher.
-	return tools.filter((t) => /fetch|scrape|http|url/i.test(`${t.tool.name} ${t.tool.description ?? ""}`));
-}
-
-/** Compose semantic-search queries to feed mcpx.search. */
-function buildQueries(hint: string | undefined, host: string | null): string[] {
-	const out: string[] = [];
-	if (hint) out.push(`${hint} fetch markdown`);
-	if (host) out.push(`fetch ${host} as markdown`, `scrape ${host}`);
-	out.push("fetch URL as markdown", "scrape webpage to markdown");
-	return out;
-}
-
-/** URL → hostname or null. */
-function safeHost(url: string): string | null {
-	try {
-		return new URL(url).hostname.toLowerCase();
-	} catch {
-		return null;
-	}
-}
-
-/**
- * Among the candidate tools, prefer one whose name or description signals
- * markdown output (contains "markdown", "md", "Docmd", etc.). Falls back
- * to anything that looks like a generic fetch/scrape verb, and finally
- * to the first candidate so we always try something.
- */
-function pickTool(tools: McpxToolDescriptor[]): McpxToolDescriptor | null {
-	const score = (t: McpxToolDescriptor) => {
-		const hay = `${t.tool.name} ${t.tool.description ?? ""}`.toLowerCase();
-		let s = 0;
-		if (/markdown|docmd|asmd|\bmd\b/.test(hay)) s += 5;
-		if (/scrape|extract|fetch|get|read/.test(hay)) s += 2;
-		if (/url|web|html|page/.test(hay)) s += 1;
-		return s;
-	};
-	const sorted = [...tools].sort((a, b) => score(b) - score(a));
-	return sorted[0] ?? null;
-}
-
-/**
- * Build the argument object the mcpx fetcher tool likely accepts. We can't
- * know the schema without calling info(), so we build a permissive bag with
- * the common shapes (`{url, format: "markdown", formats: ["markdown"]}`)
- * and trust the underlying tool to ignore unknown fields.
+ * Detect MCP `CallToolResult` envelopes that signal tool failure. MCP
+ * tool errors don't throw — they return `{ isError: true, content: [...] }`
+ * — so callers must check this explicitly before treating the content
+ * as a successful payload. Used by the refresh runner; the agent loop
+ * has its own preview-aware check.
  */
-function buildArgs(toolName: string, url: string): Record<string, unknown> {
-	const args: Record<string, unknown> = { url };
-	if (/markdown|md/i.test(toolName)) args.format = "markdown";
-	args.formats = ["markdown"];
-	return args;
-}
-
-/** Pull a string out of the heterogeneous shapes mcpx tools return. */
-function extractText(result: unknown): string {
-	if (typeof result === "string") return result;
-	if (result && typeof result === "object") {
-		const maybe = result as Record<string, unknown>;
-		if (typeof maybe.text === "string") return maybe.text;
-		if (typeof maybe.content === "string") return maybe.content;
-		if (typeof maybe.markdown === "string") return maybe.markdown;
-		if (Array.isArray(maybe.content)) {
-			const out: string[] = [];
-			for (const c of maybe.content) {
-				if (c && typeof c === "object") {
-					const inner = c as Record<string, unknown>;
-					if (typeof inner.text === "string") out.push(inner.text);
-				}
-			}
-			if (out.length > 0) return out.join("\n\n");
-		}
-	}
-	try {
-		return JSON.stringify(result);
-	} catch {
-		return "";
-	}
+export function isMcpToolError(result: unknown): boolean {
+	if (!result || typeof result !== "object") return false;
+	return (result as { isError?: unknown }).isError === true;
 }

package/src/ingest/ingest.ts

@@ -3,6 +3,7 @@ import { upsertBlob } from "../db/blobs.ts";
 import { insertChunksForVersion, rebuildFts } from "../db/chunks.ts";
 import { type FetcherKind, getCurrent, insertVersion, millisIso, type SourceType } from "../db/files.ts";
 import { asHelpful, HelpfulError } from "../errors.ts";
+import { logger } from "../output/logger.ts";
 import { chunkDeterministic } from "./chunker.ts";
 import { convert } from "./converter/index.ts";
 import { describe } from "./describer.ts";
@@ -188,12 +189,32 @@ async function ingestUrl(
 ): Promise<IngestResult> {
 	const mcpxAdapter = ctx.mcpx
 		? {
-				async listTools() {
-					const tools = await ctx.mcpx!.listTools();
-					return tools;
+				async search(query: string, options?: { keywordOnly?: boolean; semanticOnly?: boolean }) {
+					try {
+						const results = await ctx.mcpx!.search(query, options);
+						return results.map((r) => ({
+							server: r.server,
+							tool: r.tool,
+							description: r.description ?? undefined,
+							score: r.score,
+							matchType: r.matchType ?? undefined,
+						}));
+					} catch (err) {
+						logger.debug(`mcpx.search(${query}) failed: ${err instanceof Error ? err.message : String(err)}`);
+						return [];
+					}
 				},
-				async exec(server: string, tool: string, args: Record<string, unknown>) {
-					return ctx.mcpx!.exec(server, tool, args);
+				async listTools(server?: string) {
+					const tools = await ctx.mcpx!.listTools(server);
+					return tools.map((t) => ({ server: t.server, tool: { name: t.tool.name, description: t.tool.description } }));
+				},
+				async info(server: string, tool: string) {
+					const t = await ctx.mcpx!.info(server, tool);
+					if (!t) return undefined;
+					return { name: t.name, description: t.description, inputSchema: t.inputSchema };
+				},
+				async exec(server: string, tool: string, args?: Record<string, unknown>) {
+					return ctx.mcpx!.exec(server, tool, args ?? {});
 				},
 			}
 		: null;
@@ -212,7 +233,11 @@ async function ingestUrl(
 	};
 
 	try {
-		const fetched = await fetchRemote(url, { hint: input.fetcher_hint, mcpx: mcpxAdapter });
+		const fetched = await fetchRemote(url, {
+			hint: input.fetcher_hint,
+			mcpx: mcpxAdapter,
+			llm: ctx.config.llm,
+		});
 		result.mime_type = fetched.mimeType;
 		result.size_bytes = fetched.bytes.byteLength;
 		result.fetcher = fetched.fetcher;
@@ -583,6 +608,7 @@ function summarize(entries: IngestEntryResult[]): IngestResult {
 }
 
 function errorMessage(err: unknown): string {
+	if (err instanceof HelpfulError) return `${err.message} — ${err.hint}`;
 	if (err instanceof Error) return err.message;
 	return String(err);
 }

package/src/refresh/runner.ts

@@ -8,7 +8,7 @@ import { chunkDeterministic } from "../ingest/chunker.ts";
 import { convert } from "../ingest/converter/index.ts";
 import { describe } from "../ingest/describer.ts";
 import { embed } from "../ingest/embedder.ts";
-import { fetchRemote } from "../ingest/fetcher.ts";
+import { fetchRemote, isMcpToolError } from "../ingest/fetcher.ts";
 import { mimeFromPath, readLocalFile, sha256Hex } from "../ingest/local-reader.ts";
 import { buildSearchText } from "../ingest/search-text.ts";
 
@@ -192,6 +192,14 @@ async function replayFetch(
 	if (cur.fetcher === "mcpx" && cur.fetcher_server && cur.fetcher_tool && mcpx) {
 		const args = cur.fetcher_args ?? {};
 		const result = await mcpx.exec(cur.fetcher_server, cur.fetcher_tool, args);
+		if (isMcpToolError(result)) {
+			const detail = extractText(result).trim();
+			throw new HelpfulError({
+				kind: "network_error",
+				message: `mcpx tool ${cur.fetcher_server}/${cur.fetcher_tool} returned isError=true${detail ? `: ${detail}` : ""}`,
+				hint: `Re-add with a working fetcher: \`membot remove ${cur.logical_path}\` then \`membot add ${cur.source_path} --fetcher http\` (or another --fetcher hint).`,
+			});
+		}
 		const text = extractText(result);
 		const bytes = new TextEncoder().encode(text);
 		return {