membot 0.5.2 → 0.7.0

package/src/ingest/fetcher.ts

@@ -1,158 +1,133 @@
-import type { LlmConfig } from "../config/schemas.ts";
-import { DEFAULTS } from "../constants.ts";
-import { asHelpful, HelpfulError } from "../errors.ts";
+import { join } from "node:path";
+import type { MembotConfig } from "../config/schemas.ts";
+import { FILES } from "../constants.ts";
+import { HelpfulError } from "../errors.ts";
 import { logger } from "../output/logger.ts";
-import type { AgentMcpxAdapter } from "./agent-fetcher.ts";
-import { agentFetch } from "./agent-fetcher.ts";
-import { sha256Hex } from "./local-reader.ts";
+import { BrowserPool } from "./downloaders/browser.ts";
+import {
+	type DownloadedRemote,
+	type Downloader,
+	type DownloaderCtx,
+	findDownloader,
+	findDownloaderByName,
+	listDownloaders,
+} from "./downloaders/index.ts";
 
-export interface FetchedRemote {
-	bytes: Uint8Array;
-	sha256: string;
-	mimeType: string;
-	fetcher: "http" | "mcpx";
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
-	sourceUrl: string;
-}
+export type FetchedRemote = DownloadedRemote;
 
 export interface FetchOptions {
 	/**
-	 * User-provided hint. Free-form keyword (e.g. "firecrawl", "github",
-	 * "google-docs", "http"). Special-cased: "http" forces plain fetch.
-	 * Otherwise the hint is passed verbatim to the agent loop as extra
-	 * guidance about which provider to prefer.
+	 * Optional explicit downloader override. Free-form; matched
+	 * case-insensitively against `Downloader.name`. When given, skips the
+	 * URL-based matching and forces that downloader (useful for the
+	 * "use the generic-web fallback even though google-docs claimed
+	 * this URL" escape hatch).
 	 */
-	hint?: string;
-	/** Live mcpx adapter the agent loop drives via search/list/info/exec. */
-	mcpx?: AgentMcpxAdapter | null;
+	downloaderName?: string;
 	/**
-	 * LLM config. The agent loop needs an Anthropic key; without one the
-	 * mcpx path is skipped and we fall back to plain HTTP.
+	 * Override the on-disk path of the persistent chromium profile.
+	 * Defaults to `<ctx.dataDir>/auth/browser-profile`.
 	 */
-	llm?: LlmConfig;
+	userDataDir?: string;
+	/** Pre-built BrowserPool to share across many fetches (set by ingest's outer loop). */
+	pool?: BrowserPool;
 	/**
-	 * Forwarded to the agent loop so callers (e.g. the ingest progress
-	 * reporter) can drive a spinner sublabel from per-turn agent activity.
+	 * Sublabel hook forwarded to the downloader's `DownloaderCtx`.
+	 * Drives the per-entry spinner text during multi-step fetches.
 	 */
 	onProgress?: (sublabel: string) => void;
 }
 
 /**
- * Fetch a remote URL.
- *
- * - `--fetcher http` (or no mcpx, or no LLM key) → plain HTTP.
- * - Otherwise → multi-turn agent loop: Claude is given mcpx tools
- *   (search/list/info/exec) and decides how to retrieve the URL,
- *   including multi-step flows (start a job → poll → download).
- *   The agent's selected mcp_exec invocation is recorded on the
- *   returned row so refresh can replay it deterministically without
- *   another agent round-trip.
- *
- * If the agent decides plain HTTP is the right call (`request_http_fallback`,
- * no tool calls, max turns) we transparently fall through to `httpFetch`.
- * If the agent reports an actionable failure, we surface that as a
- * `HelpfulError`. If mcpx is configured but the LLM key is missing AND
- * the HTTP fallback also fails, we surface an `auth_error` naming the env
- * var so users see the real cause instead of a misleading 401.
+ * Fetch a remote URL via the per-service downloader registry. Specific
+ * downloaders (Google, GitHub, Linear) match first; the generic-web
+ * downloader is the always-matching catch-all. Every fetch authenticates
+ * via the cookies the user persisted with `membot login`. The returned
+ * shape includes the chosen downloader name and its args so refresh can
+ * replay it deterministically without involving the LLM.
  */
-export async function fetchRemote(url: string, options: FetchOptions = {}): Promise<FetchedRemote> {
-	const mcpx = options.mcpx;
-	const hint = options.hint?.trim();
-
-	if (hint === "http") return httpFetch(url);
-	if (!mcpx) return httpFetch(url);
-
-	const apiKey = options.llm?.anthropic_api_key?.trim();
-	if (!apiKey) {
-		// No way to drive the agent. Try HTTP; if that fails, the user
-		// almost certainly wanted mcpx — surface a clear key-missing error.
-		try {
-			return await httpFetch(url);
-		} catch (err) {
-			if (err instanceof HelpfulError && err.kind === "network_error") {
-				throw new HelpfulError({
-					kind: "auth_error",
-					message: `${url} couldn't be fetched directly (${err.message}). Membot has mcpx configured, but routing through it requires Claude to translate the URL into the right tool arguments — and ANTHROPIC_API_KEY isn't set.`,
-					hint: `Set ANTHROPIC_API_KEY in your environment (or under llm.anthropic_api_key in ~/.membot/config.json), then retry. To force the HTTP path explicitly, run \`membot add ${url} --fetcher http\`.`,
-				});
-			}
-			throw err;
-		}
-	}
+export async function fetchRemote(
+	url: string,
+	config: MembotConfig,
+	options: FetchOptions = {},
+	dataDir?: string,
+): Promise<FetchedRemote> {
+	const downloader = pickDownloader(url, options.downloaderName);
+	const userDataDir = options.userDataDir ?? defaultProfileDir(dataDir);
+	const ownsPool = !options.pool;
+	const headless = !downloader.requireHeaded;
+	const pool = options.pool ?? new BrowserPool({ userDataDir, headless });
+	const dctx: DownloaderCtx = { pool, logger, config, onProgress: options.onProgress };
 
-	let outcome: Awaited<ReturnType<typeof agentFetch>>;
 	try {
-		outcome = await agentFetch({ url, mcpx, llm: options.llm!, hint, onProgress: options.onProgress });
-	} catch (err) {
-		if (err instanceof HelpfulError) throw err;
-		logger.warn(`agent-fetch failed (${err instanceof Error ? err.message : String(err)}) — falling back to HTTP`);
-		return httpFetch(url);
+		// Fetches are strictly non-interactive: there's no auto-launch
+		// of a browser when auth fails. Batch ingest (`membot add` of
+		// many URLs) and the refresh daemon both run without a human
+		// available to drive a window, so any auth_error must
+		// propagate as-is. The HelpfulError's hint tells the user to
+		// `membot login` (cookie-based services) or `membot config set
+		// downloaders.<svc>.api_key` (API-key services); they fix it
+		// once and re-run.
+		return await downloader.download(new URL(url), dctx);
+	} finally {
+		if (ownsPool) await pool.dispose();
 	}
+}
 
-	if (outcome.kind === "accepted") {
-		return {
-			bytes: outcome.result.bytes,
-			sha256: outcome.result.sha256,
-			mimeType: outcome.result.mimeType,
-			fetcher: "mcpx",
-			fetcherServer: outcome.result.fetcherServer,
-			fetcherTool: outcome.result.fetcherTool,
-			fetcherArgs: outcome.result.fetcherArgs,
-			sourceUrl: url,
-		};
+/**
+ * Replay a fetch by downloader name (used by refresh). Looks up the
+ * persisted downloader by name and calls it against the original URL —
+ * deterministic, no agent loop. When the persisted downloader is no
+ * longer registered (e.g. from a prior membot version), falls back to
+ * URL-based dispatch so refresh degrades gracefully instead of erroring.
+ */
+export async function fetchRemoteByDownloader(
+	downloaderName: string | null,
+	url: string,
+	pool: BrowserPool,
+	config: MembotConfig,
+): Promise<FetchedRemote> {
+	const named = downloaderName ? findDownloaderByName(downloaderName) : null;
+	const downloader = named ?? findDownloader(url);
+	if (!downloader) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `no downloader matches ${url}`,
+			hint: "Re-add the URL with `membot add <url>` to pick a fresh downloader.",
+		});
 	}
-	logger.info(`[fetcher] falling back to HTTP: ${outcome.reason}`);
-	return httpFetch(url);
+	const dctx: DownloaderCtx = { pool, logger, config };
+	return downloader.download(new URL(url), dctx);
 }
 
-/** Plain `fetch` fallback. Used when mcpx isn't configured or the hint says so. */
-async function httpFetch(url: string): Promise<FetchedRemote> {
-	let resp: Response;
-	try {
-		resp = await fetch(url, {
-			headers: { "User-Agent": "membot/0.1" },
-			signal: AbortSignal.timeout(DEFAULTS.HTTP_TIMEOUT_MS),
-		});
-	} catch (err) {
-		throw asHelpful(
-			err,
-			`while fetching ${url}`,
-			`Check your network and that ${url} is reachable. For mcpx-managed sources (gdocs/github/firecrawl), set ANTHROPIC_API_KEY so membot can drive an mcpx tool.`,
-			"network_error",
-		);
+function pickDownloader(url: string, override?: string): Downloader {
+	if (override) {
+		const named = findDownloaderByName(override.toLowerCase());
+		if (!named) {
+			const available = listDownloaders()
+				.map((d) => d.name)
+				.join(", ");
+			throw new HelpfulError({
+				kind: "input_error",
+				message: `unknown downloader '${override}'`,
+				hint: `Pick one of: ${available}.`,
+			});
+		}
+		return named;
 	}
-	if (!resp.ok) {
+	const matched = findDownloader(url);
+	if (!matched) {
 		throw new HelpfulError({
-			kind: "network_error",
-			message: `HTTP ${resp.status} ${resp.statusText}: ${url}`,
-			hint: "Verify the URL is reachable and not gated behind auth. For private docs use mcpx (set ANTHROPIC_API_KEY).",
+			kind: "input_error",
+			message: `not a fetchable URL: ${url}`,
+			hint: "Pass an http(s):// URL.",
 		});
 	}
-	const bytes = new Uint8Array(await resp.arrayBuffer());
-	const ct = resp.headers.get("content-type") ?? "";
-	const mime = ct.split(";")[0]?.trim() || "application/octet-stream";
-	return {
-		bytes,
-		sha256: sha256Hex(bytes),
-		mimeType: mime,
-		fetcher: "http",
-		fetcherServer: null,
-		fetcherTool: null,
-		fetcherArgs: null,
-		sourceUrl: url,
-	};
+	return matched;
 }
 
-/**
- * Detect MCP `CallToolResult` envelopes that signal tool failure. MCP
- * tool errors don't throw — they return `{ isError: true, content: [...] }`
- * — so callers must check this explicitly before treating the content
- * as a successful payload. Used by the refresh runner; the agent loop
- * has its own preview-aware check.
- */
-export function isMcpToolError(result: unknown): boolean {
-	if (!result || typeof result !== "object") return false;
-	return (result as { isError?: unknown }).isError === true;
+function defaultProfileDir(dataDir?: string): string {
+	if (dataDir) return join(dataDir, FILES.BROWSER_PROFILE);
+	const home = process.env.MEMBOT_HOME ?? `${process.env.HOME ?? "."}/.membot`;
+	return join(home, FILES.BROWSER_PROFILE);
 }

package/src/ingest/ingest.ts

@@ -20,7 +20,7 @@ export interface IngestInput {
 	exclude?: string;
 	follow_symlinks?: boolean;
 	refresh_frequency?: string;
-	fetcher_hint?: string;
+	downloader?: string;
 	change_note?: string;
 	force?: boolean;
 }
@@ -161,13 +161,12 @@ async function ingestInline(
 				bytes: null,
 				markdown: text,
 				fetcher: "inline",
-				fetcherServer: null,
-				fetcherTool: null,
-				fetcherArgs: null,
+				downloader: null,
+				downloaderArgs: null,
 				refreshSec,
 				changeNote: input.change_note ?? null,
 			},
-			(done, total) => callbacks?.onEntryProgress?.(logicalPath, `embedding ${done}/${total}`),
+			(sublabel) => callbacks?.onEntryProgress?.(logicalPath, sublabel),
 		);
 		result.version_id = versionId;
 	} catch (err) {
@@ -187,38 +186,6 @@ async function ingestUrl(
 	force: boolean,
 	callbacks?: IngestCallbacks,
 ): Promise<IngestResult> {
-	const mcpxAdapter = ctx.mcpx
-		? {
-				async search(query: string, options?: { keywordOnly?: boolean; semanticOnly?: boolean }) {
-					try {
-						const results = await ctx.mcpx!.search(query, options);
-						return results.map((r) => ({
-							server: r.server,
-							tool: r.tool,
-							description: r.description ?? undefined,
-							score: r.score,
-							matchType: r.matchType ?? undefined,
-						}));
-					} catch (err) {
-						logger.debug(`mcpx.search(${query}) failed: ${err instanceof Error ? err.message : String(err)}`);
-						return [];
-					}
-				},
-				async listTools(server?: string) {
-					const tools = await ctx.mcpx!.listTools(server);
-					return tools.map((t) => ({ server: t.server, tool: { name: t.tool.name, description: t.tool.description } }));
-				},
-				async info(server: string, tool: string) {
-					const t = await ctx.mcpx!.info(server, tool);
-					if (!t) return undefined;
-					return { name: t.name, description: t.description, inputSchema: t.inputSchema };
-				},
-				async exec(server: string, tool: string, args?: Record<string, unknown>) {
-					return ctx.mcpx!.exec(server, tool, args ?? {});
-				},
-			}
-		: null;
-
 	const logicalPath = input.logical_path ?? defaultLogicalForUrl(url);
 	callbacks?.onEntryStart?.(url);
 	const result: IngestEntryResult = {
@@ -228,20 +195,24 @@ async function ingestUrl(
 		status: "ok",
 		mime_type: null,
 		size_bytes: 0,
-		fetcher: "http",
+		fetcher: "downloader",
 		source_sha256: "",
 	};
 
 	try {
-		const fetched = await fetchRemote(url, {
-			hint: input.fetcher_hint,
-			mcpx: mcpxAdapter,
-			llm: ctx.config.llm,
-			onProgress: (sublabel) => callbacks?.onEntryProgress?.(url, sublabel),
-		});
+		callbacks?.onEntryProgress?.(url, "fetching");
+		const fetched = await fetchRemote(
+			url,
+			ctx.config,
+			{
+				downloaderName: input.downloader,
+				onProgress: (sublabel) => callbacks?.onEntryProgress?.(url, sublabel),
+			},
+			ctx.dataDir,
+		);
 		result.mime_type = fetched.mimeType;
 		result.size_bytes = fetched.bytes.byteLength;
-		result.fetcher = fetched.fetcher;
+		result.fetcher = "downloader";
 		result.source_sha256 = fetched.sha256;
 
 		if (!force) {
@@ -265,14 +236,13 @@ async function ingestUrl(
 				sourcePath: url,
 				sourceMtimeMs: null,
 				sourceSha: fetched.sha256,
-				fetcher: fetched.fetcher,
-				fetcherServer: fetched.fetcherServer,
-				fetcherTool: fetched.fetcherTool,
-				fetcherArgs: fetched.fetcherArgs,
+				fetcher: "downloader",
+				downloader: fetched.downloader,
+				downloaderArgs: fetched.downloaderArgs,
 				refreshSec,
 				changeNote: input.change_note ?? null,
 			},
-			(done, total) => callbacks?.onEntryProgress?.(url, `embedding ${done}/${total}`),
+			(sublabel) => callbacks?.onEntryProgress?.(url, sublabel),
 		);
 		result.version_id = versionId;
 	} catch (err) {
@@ -352,13 +322,12 @@ async function ingestLocalFiles(
 					sourceMtimeMs: local.mtimeMs,
 					sourceSha: local.sha256,
 					fetcher: "local",
-					fetcherServer: null,
-					fetcherTool: null,
-					fetcherArgs: null,
+					downloader: null,
+					downloaderArgs: null,
 					refreshSec,
 					changeNote: input.change_note ?? null,
 				},
-				(done, total) => callbacks?.onEntryProgress?.(entry.relPathFromBase, `embedding ${done}/${total}`),
+				(sublabel) => callbacks?.onEntryProgress?.(entry.relPathFromBase, sublabel),
 			);
 			result.version_id = versionId;
 		} catch (err) {
@@ -387,9 +356,8 @@ interface PipelineParams {
 	sourceMtimeMs: number | null;
 	sourceSha: string;
 	fetcher: FetcherKind;
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
+	downloader: string | null;
+	downloaderArgs: Record<string, unknown> | null;
 	refreshSec: number | null;
 	changeNote: string | null;
 }
@@ -404,8 +372,9 @@ interface PipelineParams {
 async function pipelineForBytes(
 	ctx: AppContext,
 	p: PipelineParams,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<string> {
+	onPhase?.("storing blob");
 	await upsertBlob(ctx.db, {
 		sha256: p.sourceSha,
 		mime_type: p.mime,
@@ -413,7 +382,8 @@ async function pipelineForBytes(
 		bytes: p.bytes,
 	});
 
-	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm);
+	onPhase?.("converting");
+	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm, ctx.config.converters);
 	const markdown = conversion.markdown;
 	const contentSha = sha256Hex(new TextEncoder().encode(markdown));
 
@@ -431,13 +401,12 @@ async function pipelineForBytes(
 			markdown,
 			contentSha,
 			fetcher: p.fetcher,
-			fetcherServer: p.fetcherServer,
-			fetcherTool: p.fetcherTool,
-			fetcherArgs: p.fetcherArgs,
+			downloader: p.downloader,
+			downloaderArgs: p.downloaderArgs,
 			refreshSec: p.refreshSec,
 			changeNote: p.changeNote,
 		},
-		onEmbedProgress,
+		onPhase,
 	);
 }
 
@@ -453,9 +422,8 @@ interface PersistParams {
 	markdown: string;
 	contentSha?: string;
 	fetcher: FetcherKind;
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
+	downloader: string | null;
+	downloaderArgs: Record<string, unknown> | null;
 	refreshSec: number | null;
 	changeNote: string | null;
 }
@@ -469,14 +437,18 @@ interface PersistParams {
 async function persistVersion(
 	ctx: AppContext,
 	p: PersistParams,
-	onEmbedProgress?: (done: number, total: number) => void,
+	onPhase?: (sublabel: string) => void,
 ): Promise<string> {
+	onPhase?.("describing");
 	const description = await describe(p.logicalPath, p.mime, p.markdown, ctx.config.llm);
+	onPhase?.("chunking");
 	const chunks = chunkDeterministic(p.markdown, ctx.config.chunker);
 	const searchTexts = chunks.map((c) => buildSearchText(p.logicalPath, description, c.content));
 	let embeddings: number[][];
 	try {
-		embeddings = await embed(searchTexts, ctx.config.embedding_model, { onProgress: onEmbedProgress });
+		embeddings = await embed(searchTexts, ctx.config.embedding_model, {
+			onProgress: (done, total) => onPhase?.(`embedding ${done}/${total}`),
+		});
 	} catch (err) {
 		throw asHelpful(
 			err,
@@ -485,6 +457,7 @@ async function persistVersion(
 		);
 	}
 
+	onPhase?.("persisting");
 	const versionId = millisIso(Date.now());
 	const contentSha = p.contentSha ?? sha256Hex(new TextEncoder().encode(p.markdown));
 	await insertVersion(ctx.db, {
@@ -501,9 +474,8 @@ async function persistVersion(
 		mime_type: p.mime,
 		size_bytes: p.bytes?.byteLength ?? new TextEncoder().encode(p.markdown).byteLength,
 		fetcher: p.fetcher,
-		fetcher_server: p.fetcherServer,
-		fetcher_tool: p.fetcherTool,
-		fetcher_args: p.fetcherArgs,
+		downloader: p.downloader,
+		downloader_args: p.downloaderArgs,
 		refresh_frequency_sec: p.refreshSec,
 		refreshed_at: new Date().toISOString(),
 		last_refresh_status: "ok",
@@ -521,6 +493,7 @@ async function persistVersion(
 			embedding: embeddings[i] ?? new Array(embeddings[0]?.length ?? 0).fill(0),
 		})),
 	);
+	onPhase?.("indexing");
 	await rebuildFts(ctx.db);
 	return versionId;
 }

package/src/mcp/instructions.ts

@@ -11,8 +11,10 @@ indexed with BM25 — so prefer membot_search to membot_read+grep for discovery.
 Workflow:
   1. membot_tree or membot_search to find what already exists before adding new content.
   2. membot_add to ingest a local file, a URL, or a remote document. URLs are
-     fetched via mcpx (the chosen invocation is stored so refresh is fast and
-     deterministic).
+     fetched via per-service downloaders (Google Docs, Sheets, Slides, GitHub,
+     Linear, with a generic browser print-to-PDF fallback). Authentication
+     comes from the user's logged-in browser cookies (saved via \`membot login\`).
+     Each row stores which downloader was used so refresh is deterministic.
   3. membot_read or membot_search hits to consume content.
   4. membot_write to record agent-authored notes (source_type='inline').
 

package/src/operations/add.ts

@@ -10,7 +10,7 @@ import { type ResolvedSource, resolveSource } from "../ingest/source-resolver.ts
 import { colors } from "../output/formatter.ts";
 import { defineOperation } from "./types.ts";
 
-const FetcherKindEnum = z.enum(["http", "mcpx", "local", "inline"]);
+const FetcherKindEnum = z.enum(["downloader", "local", "inline"]);
 
 export const addOperation = defineOperation({
 	name: "membot_add",
@@ -19,7 +19,7 @@ export const addOperation = defineOperation({
   - a local file path
   - a local directory (recursive walk, symlinks followed)
   - a glob pattern (e.g. "docs/**/*.md")
-  - a URL (fetched via mcpx if configured, otherwise plain HTTP)
+  - a URL (fetched via the per-service downloader registry — Google Docs/Sheets/Slides via export endpoints, GitHub + Linear as rendered HTML, anything else through a generic browser print-to-PDF fallback. All fetches authenticate via the user's logged-in browser session — run \`membot login\` once to sign in.)
   - "inline:<text>" literal
 Pass any number of args; each is resolved independently and the matched entries are concatenated into one response. PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. By default, re-ingesting an unchanged source (same source_sha256 as the current version) is a no-op and reports \`status: "unchanged"\`; pass \`force=true\` to always create a new version. Each newly-ingested file becomes a new version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.
 
@@ -54,10 +54,12 @@ Pass \`logical_path\` to override. For a multi-source / directory / glob walk it
 			.default(true)
 			.describe("Follow symlinks during directory walks (cycles broken via realpath)"),
 		refresh_frequency: z.string().optional().describe("Auto-refresh cadence: 5m | 1h | 24h | 7d. Omit to disable."),
-		fetcher_hint: z
+		downloader: z
 			.string()
 			.optional()
-			.describe("Free-form hint passed to mcpx tool search (e.g. 'firecrawl', 'github', 'google docs', 'http')"),
+			.describe(
+				"Force a specific downloader by name (e.g. 'google-docs', 'github', 'generic-web'). Skips URL-based matching.",
+			),
 		change_note: z.string().optional().describe("Free-text note attached to the new version"),
 		force: z
 			.boolean()

package/src/operations/info.ts

@@ -25,9 +25,8 @@ export const infoOperation = defineOperation({
 		size_bytes: z.number().nullable(),
 		description: z.string().nullable(),
 		fetcher: z.string().nullable(),
-		fetcher_server: z.string().nullable(),
-		fetcher_tool: z.string().nullable(),
-		fetcher_args: z.record(z.string(), z.unknown()).nullable(),
+		downloader: z.string().nullable(),
+		downloader_args: z.record(z.string(), z.unknown()).nullable(),
 		refresh_frequency_sec: z.number().nullable(),
 		refreshed_at: z.string().nullable(),
 		last_refresh_status: z.string().nullable(),
@@ -53,9 +52,8 @@ export const infoOperation = defineOperation({
 		lines.push(fmt("blob_sha256", orDash(result.blob_sha256)));
 		lines.push(fmt("source_sha256", orDash(result.source_sha256)));
 		if (result.fetcher) lines.push(fmt("fetcher", result.fetcher));
-		if (result.fetcher_server) lines.push(fmt("fetcher_server", result.fetcher_server));
-		if (result.fetcher_tool) lines.push(fmt("fetcher_tool", result.fetcher_tool));
-		if (result.fetcher_args) lines.push(fmt("fetcher_args", JSON.stringify(result.fetcher_args)));
+		if (result.downloader) lines.push(fmt("downloader", result.downloader));
+		if (result.downloader_args) lines.push(fmt("downloader_args", JSON.stringify(result.downloader_args)));
 		lines.push(
 			fmt(
 				"refresh_frequency",

package/src/operations/move.ts

@@ -54,9 +54,8 @@ export const moveOperation = defineOperation({
 			mime_type: cur.mime_type,
 			size_bytes: cur.size_bytes,
 			fetcher: cur.fetcher,
-			fetcher_server: cur.fetcher_server,
-			fetcher_tool: cur.fetcher_tool,
-			fetcher_args: cur.fetcher_args,
+			downloader: cur.downloader,
+			downloader_args: cur.downloader_args,
 			refresh_frequency_sec: cur.refresh_frequency_sec,
 			refreshed_at: cur.refreshed_at,
 			last_refresh_status: cur.last_refresh_status,

package/src/operations/refresh.ts

@@ -7,7 +7,7 @@ import { defineOperation } from "./types.ts";
 export const refreshOperation = defineOperation({
 	name: "membot_refresh",
 	cliName: "refresh",
-	description: `Re-read a file's source and create a new version only if the source bytes changed. Pass \`logical_path\` to refresh one file, or omit it to refresh every file whose refresh_frequency_sec has elapsed. Local files are detected via mtime+sha; remote files are re-fetched via the same mcpx invocation that was originally used. On auth or network failure the prior version stays current — check \`last_refresh_status\`.`,
+	description: `Re-read a file's source and create a new version only if the source bytes changed. Pass \`logical_path\` to refresh one file, or omit it to refresh every file whose refresh_frequency_sec has elapsed. Local files are detected via mtime+sha; remote files are re-fetched via the same downloader (Google Docs, GitHub, etc.) that was originally chosen. On auth or network failure the prior version stays current — check \`last_refresh_status\`. If the failure mentions a login redirect, re-run \`membot login\` and try again.`,
 	inputSchema: z.object({
 		logical_path: z.string().optional().describe("Single path to refresh; omit for all-due"),
 		force: z.boolean().default(false).describe("Re-embed even if source sha is unchanged"),
@@ -60,9 +60,7 @@ export const refreshOperation = defineOperation({
 		for (const path of targets) {
 			ctx.progress.tick(path);
 			try {
-				const r = await refreshOne(ctx, path, input.force, (done, total) =>
-					ctx.progress.update(`embedding ${done}/${total}`),
-				);
+				const r = await refreshOne(ctx, path, input.force, (sublabel) => ctx.progress.update(sublabel));
 				out.push(r);
 			} catch (err) {
 				out.push({ logical_path: path, status: "failed", error: err instanceof Error ? err.message : String(err) });