membot 0.0.1 → 0.1.1

package/src/ingest/converter/pdf.ts

@@ -0,0 +1,38 @@
+import { extractText, getDocumentProxy } from "unpdf";
+import { logger } from "../../output/logger.ts";
+
+export interface PdfConversion {
+	markdown: string;
+	textRatio: number;
+	usedOcrFallback: boolean;
+}
+
+const LOW_TEXT_RATIO = 0.005; // < ~5 chars per kB → very likely scanned
+
+/**
+ * Extract the text layer from a PDF using unpdf. Returns the extracted
+ * markdown and a `textRatio` (chars / file-bytes) so the dispatcher can
+ * decide whether to fall through to OCR. The OCR step itself happens in
+ * converter/index.ts so this module stays free of WASM dependencies.
+ */
+export async function convertPdf(bytes: Uint8Array): Promise<PdfConversion> {
+	try {
+		const pdf = await getDocumentProxy(bytes);
+		const { text } = await extractText(pdf, { mergePages: false });
+		const pages: string[] = Array.isArray(text) ? text : [String(text)];
+		const md = pages
+			.map((p, i) => `## Page ${i + 1}\n\n${p.trim()}`)
+			.filter((p) => p.length > 0)
+			.join("\n\n");
+		const ratio = bytes.byteLength === 0 ? 0 : md.length / bytes.byteLength;
+		return { markdown: md, textRatio: ratio, usedOcrFallback: false };
+	} catch (err) {
+		logger.warn(`pdf: text extraction failed (${err instanceof Error ? err.message : String(err)})`);
+		return { markdown: "", textRatio: 0, usedOcrFallback: false };
+	}
+}
+
+/** Decide whether unpdf's output is "low text ratio" enough to warrant OCR fallback. */
+export function shouldOcrPdf(conversion: PdfConversion): boolean {
+	return conversion.markdown.trim().length === 0 || conversion.textRatio < LOW_TEXT_RATIO;
+}

package/src/ingest/converter/text.ts

@@ -0,0 +1,8 @@
+/**
+ * Plain-text / markdown passthrough converter. Decodes bytes as UTF-8 and
+ * returns them unchanged — the chunker downstream handles paragraph
+ * boundaries the same way as it would for an LLM-converted file.
+ */
+export function convertText(bytes: Uint8Array): string {
+	return new TextDecoder("utf-8").decode(bytes);
+}

package/src/ingest/describer.ts

@@ -0,0 +1,72 @@
+import Anthropic from "@anthropic-ai/sdk";
+import type { LlmConfig } from "../config/schemas.ts";
+import { logger } from "../output/logger.ts";
+
+const DESCRIBER_PROMPT = `You write a one-paragraph description of a file for use in a search index.
+
+Rules:
+- One paragraph, 1-3 sentences.
+- Plain prose, no headings, no markdown formatting.
+- Cover what the file IS and what it's ABOUT — both subject and shape.
+- For images, focus on the visual subject. For documents, focus on the topic and intended reader.
+- Output the description ONLY — no preamble, no quoting, no labels.`;
+
+/**
+ * Generate a one-paragraph description for the file's surrogate, used
+ * as the `<description>` line in chunks.search_text. Falls back to a
+ * deterministic heuristic when no API key is configured so the pipeline
+ * still produces a non-empty description offline.
+ */
+export async function describe(
+	logicalPath: string,
+	mimeType: string,
+	surrogate: string,
+	llm: LlmConfig,
+): Promise<string> {
+	if (!llm.anthropic_api_key || llm.anthropic_api_key.trim() === "") {
+		return deterministicDescription(logicalPath, mimeType, surrogate);
+	}
+	const client = new Anthropic({ apiKey: llm.anthropic_api_key });
+	const sample = surrogate.slice(0, 4_000);
+	try {
+		const resp = await client.messages.create({
+			model: llm.describer_model,
+			max_tokens: 300,
+			system: DESCRIBER_PROMPT,
+			messages: [
+				{
+					role: "user",
+					content: `Logical path: ${logicalPath}\nMIME type: ${mimeType}\n\nFile body:\n${sample}`,
+				},
+			],
+		});
+		const text = resp.content
+			.flatMap((b) => (b.type === "text" ? [b.text] : []))
+			.join("")
+			.trim();
+		if (!text) return deterministicDescription(logicalPath, mimeType, surrogate);
+		return text;
+	} catch (err) {
+		logger.warn(`describer: failed (${err instanceof Error ? err.message : String(err)}) — falling back`);
+		return deterministicDescription(logicalPath, mimeType, surrogate);
+	}
+}
+
+/**
+ * Cheap, deterministic description used when the LLM isn't available.
+ * For markdown/text it's the first heading + a 200-char prefix; for
+ * binaries it's `<mime> · <size> bytes`.
+ */
+export function deterministicDescription(logicalPath: string, mimeType: string, surrogate: string): string {
+	if (mimeType.startsWith("text/") || mimeType === "application/json" || mimeType === "application/yaml") {
+		const trimmed = surrogate.trim();
+		const headingMatch = trimmed.match(/^#+\s+(.+)$/m);
+		const heading = headingMatch?.[1]?.trim();
+		const prefix = trimmed.slice(0, 200).replace(/\s+/g, " ").trim();
+		if (heading && prefix) return `${heading} — ${prefix}`;
+		if (heading) return heading;
+		if (prefix) return prefix;
+		return `${logicalPath} (${mimeType})`;
+	}
+	return `${mimeType} · ${surrogate.length} chars`;
+}

package/src/ingest/embedder.ts

@@ -0,0 +1,98 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { env, type FeatureExtractionPipeline, pipeline } from "@huggingface/transformers";
+import { EMBEDDING_DIMENSION, EMBEDDING_MODEL } from "../constants.ts";
+import { HelpfulError } from "../errors.ts";
+import { logger } from "../output/logger.ts";
+
+// We patch @huggingface/transformers to use onnxruntime-web (WASM). Pin the
+// loader to the on-disk copy so we stay offline-capable.
+const ortWasm = env.backends.onnx?.wasm;
+if (ortWasm) {
+	ortWasm.wasmPaths = {
+		mjs: import.meta.resolve("onnxruntime-web/ort-wasm-simd-threaded.asyncify.mjs"),
+		wasm: import.meta.resolve("onnxruntime-web/ort-wasm-simd-threaded.asyncify.wasm"),
+	};
+}
+
+const pipelinePromises = new Map<string, Promise<FeatureExtractionPipeline>>();
+
+/** Configure where transformers caches downloaded model weights. */
+export function setEmbeddingCacheDir(dir: string): void {
+	env.cacheDir = dir.endsWith("/") ? dir : `${dir}/`;
+}
+
+function isModelCached(model: string): boolean {
+	if (!env.cacheDir) return false;
+	return existsSync(join(env.cacheDir, model));
+}
+
+/**
+ * Lazily load (and cache) the feature-extraction pipeline for a model. Loading
+ * is expensive (downloads weights on first run, ~100s of ms to instantiate
+ * ONNX), so we hold one promise per model name for the life of the process.
+ *
+ * Try `wasm` first, fall back to `cpu` on "Unsupported device". The transformers
+ * patch (applied for `bun build --compile` and via `bun run prebuild` for local
+ * dev) registers `wasm` as a supported device backed by onnxruntime-web — that's
+ * mandatory for the single-binary build because native bindings can't be
+ * bundled. When the package is unpatched (npm-installed membot, or `bun dev`
+ * before `prebuild`), `wasm` is rejected and we fall back to the default `cpu`
+ * device, which uses the onnxruntime-node native bindings that ship with the
+ * unpatched package.
+ */
+async function getPipeline(model: string): Promise<FeatureExtractionPipeline> {
+	let p = pipelinePromises.get(model);
+	if (!p) {
+		if (isModelCached(model)) {
+			logger.debug(`embedder: loading cached model ${model}`);
+		} else {
+			logger.info(`embedder: loading model ${model} (first run, downloading weights)`);
+		}
+		p = (async () => {
+			try {
+				return (await pipeline("feature-extraction", model, { device: "wasm" })) as FeatureExtractionPipeline;
+			} catch (err) {
+				if (!String((err as Error)?.message ?? "").includes("Unsupported device")) throw err;
+				logger.debug("embedder: wasm backend unavailable, falling back to cpu (onnxruntime-node)");
+				return (await pipeline("feature-extraction", model, { device: "cpu" })) as FeatureExtractionPipeline;
+			}
+		})();
+		pipelinePromises.set(model, p);
+	}
+	return p;
+}
+
+/**
+ * Embed an array of texts to L2-normalized vectors with the configured
+ * model. Throws a HelpfulError when the model's dimension doesn't match
+ * EMBEDDING_DIMENSION (the value baked into the DB schema).
+ */
+export async function embed(texts: string[], model: string = EMBEDDING_MODEL): Promise<number[][]> {
+	if (texts.length === 0) return [];
+	const extractor = await getPipeline(model);
+	const output = await extractor(texts, { pooling: "mean", normalize: true });
+	const data = output.tolist() as number[][];
+	if (data[0] && data[0].length !== EMBEDDING_DIMENSION) {
+		throw new HelpfulError({
+			kind: "internal_error",
+			message: `embedding model ${model} returned ${data[0].length}-dim vectors, expected ${EMBEDDING_DIMENSION}`,
+			hint: `Set config.embedding_model to a ${EMBEDDING_DIMENSION}-dim model (default: ${EMBEDDING_MODEL}).`,
+		});
+	}
+	return data;
+}
+
+/** Embed a single text — convenience wrapper for query-time embedding. */
+export async function embedSingle(text: string, model: string = EMBEDDING_MODEL): Promise<number[]> {
+	const all = await embed([text], model);
+	const vec = all[0];
+	if (!vec) {
+		throw new HelpfulError({
+			kind: "internal_error",
+			message: "embed() returned no vectors",
+			hint: "This is likely a transformers WASM patch issue. Run `bun run prebuild` and retry.",
+		});
+	}
+	return vec;
+}

package/src/ingest/fetcher.ts

@@ -0,0 +1,280 @@
+import { DEFAULTS } from "../constants.ts";
+import { asHelpful, HelpfulError } from "../errors.ts";
+import { logger } from "../output/logger.ts";
+import { sha256Hex } from "./local-reader.ts";
+
+export interface FetchedRemote {
+	bytes: Uint8Array;
+	sha256: string;
+	mimeType: string;
+	fetcher: "http" | "mcpx";
+	fetcherServer: string | null;
+	fetcherTool: string | null;
+	fetcherArgs: Record<string, unknown> | null;
+	sourceUrl: string;
+}
+
+export interface McpxToolDescriptor {
+	server: string;
+	tool: { name: string; description?: string };
+}
+
+export interface McpxSearchHit {
+	server: string;
+	tool: { name: string; description?: string };
+	score?: number;
+}
+
+export interface FetchOptions {
+	/**
+	 * User-provided hint. Free-form keyword (e.g. "firecrawl", "github",
+	 * "google-docs", "http"). Special-cased: "http" forces plain fetch.
+	 * Otherwise the hint is used as a search query against the live
+	 * mcpx tool catalog — we never hardcode server names.
+	 */
+	hint?: string;
+	/** Live mcpx adapter. Use listTools/search/exec to find a fetcher on the fly. */
+	mcpx?: {
+		exec(server: string, tool: string, args: Record<string, unknown>): Promise<unknown>;
+		listTools(): Promise<McpxToolDescriptor[]>;
+		search?(query: string): Promise<McpxSearchHit[]>;
+	} | null;
+}
+
+/**
+ * Fetch a remote URL, preferring an mcpx-managed server (Firecrawl, Google
+ * Docs, GitHub, …) for known providers and falling back to a plain `fetch`
+ * otherwise. The chosen invocation (server/tool/args) is returned alongside
+ * the bytes so the caller can persist it on the row for replay-on-refresh.
+ */
+export async function fetchRemote(url: string, options: FetchOptions = {}): Promise<FetchedRemote> {
+	const mcpx = options.mcpx;
+	const hint = options.hint?.trim();
+
+	if (hint === "http") return httpFetch(url);
+	if (!mcpx) return httpFetch(url);
+
+	const tried = await tryMcpx(url, mcpx, hint);
+	if (tried) return tried;
+	return httpFetch(url);
+}
+
+/** Plain `fetch` fallback. Used when mcpx isn't configured or the hint says so. */
+async function httpFetch(url: string): Promise<FetchedRemote> {
+	let resp: Response;
+	try {
+		resp = await fetch(url, {
+			headers: { "User-Agent": "membot/0.1" },
+			signal: AbortSignal.timeout(DEFAULTS.HTTP_TIMEOUT_MS),
+		});
+	} catch (err) {
+		throw asHelpful(
+			err,
+			`while fetching ${url}`,
+			`Check your network and that ${url} is reachable. For mcpx-managed sources (gdocs/github/firecrawl), set --fetcher firecrawl etc.`,
+			"network_error",
+		);
+	}
+	if (!resp.ok) {
+		throw new HelpfulError({
+			kind: "network_error",
+			message: `HTTP ${resp.status} ${resp.statusText}: ${url}`,
+			hint: "Verify the URL is reachable and not gated behind auth. For private docs use mcpx via --fetcher.",
+		});
+	}
+	const bytes = new Uint8Array(await resp.arrayBuffer());
+	const ct = resp.headers.get("content-type") ?? "";
+	const mime = ct.split(";")[0]?.trim() || "application/octet-stream";
+	return {
+		bytes,
+		sha256: sha256Hex(bytes),
+		mimeType: mime,
+		fetcher: "http",
+		fetcherServer: null,
+		fetcherTool: null,
+		fetcherArgs: null,
+		sourceUrl: url,
+	};
+}
+
+/**
+ * Attempt to fetch via mcpx by discovering a suitable tool at runtime.
+ *
+ * Strategy:
+ *   1. If the user passed a hint, search for it via mcpx.search() (semantic
+ *      tool search over the live catalog). The hint is the user's free-text
+ *      label for which provider they want — we never assume server names.
+ *   2. Otherwise, fall back to a host-based search query (e.g. URL host
+ *      "github.com" → search for "github fetch markdown").
+ *   3. From the returned candidates, prefer tools whose name or description
+ *      signals markdown output. Failing that, the first tool that takes a
+ *      URL-shaped argument.
+ *   4. Execute the tool with `{ url, format: "markdown" }`-shaped args.
+ *      If exec fails, return null so the caller falls back to plain HTTP.
+ */
+async function tryMcpx(
+	url: string,
+	mcpx: NonNullable<FetchOptions["mcpx"]>,
+	hint: string | undefined,
+): Promise<FetchedRemote | null> {
+	const candidates = await discoverCandidates(url, mcpx, hint);
+	if (candidates.length === 0) return null;
+
+	const chosen = pickTool(candidates);
+	if (!chosen) return null;
+
+	const args = buildArgs(chosen.tool.name, url);
+	let result: unknown;
+	try {
+		result = await mcpx.exec(chosen.server, chosen.tool.name, args);
+	} catch (err) {
+		logger.warn(
+			`mcpx: ${chosen.server}/${chosen.tool.name} failed (${err instanceof Error ? err.message : String(err)})`,
+		);
+		return null;
+	}
+
+	const text = extractText(result);
+	if (!text || text.trim().length === 0) return null;
+	const bytes = new TextEncoder().encode(text);
+	return {
+		bytes,
+		sha256: sha256Hex(bytes),
+		mimeType: "text/markdown",
+		fetcher: "mcpx",
+		fetcherServer: chosen.server,
+		fetcherTool: chosen.tool.name,
+		fetcherArgs: args,
+		sourceUrl: url,
+	};
+}
+
+/**
+ * Build a list of candidate fetcher tools by querying mcpx's live catalog.
+ * Tries semantic search first (using the hint or the URL's host as the
+ * query) then falls back to listing all tools and filtering by name. Never
+ * hardcodes a server name — the catalog is the source of truth.
+ */
+async function discoverCandidates(
+	url: string,
+	mcpx: NonNullable<FetchOptions["mcpx"]>,
+	hint: string | undefined,
+): Promise<McpxToolDescriptor[]> {
+	const host = safeHost(url);
+	const queries = buildQueries(hint, host);
+
+	if (mcpx.search) {
+		for (const q of queries) {
+			try {
+				const hits = await mcpx.search(q);
+				if (hits.length > 0) {
+					return hits.slice(0, 5).map((h) => ({ server: h.server, tool: h.tool }));
+				}
+			} catch (err) {
+				logger.debug(`mcpx: search(${q}) failed (${err instanceof Error ? err.message : String(err)})`);
+			}
+		}
+	}
+
+	let tools: McpxToolDescriptor[];
+	try {
+		tools = await mcpx.listTools();
+	} catch (err) {
+		logger.debug(`mcpx: listTools failed (${err instanceof Error ? err.message : String(err)})`);
+		return [];
+	}
+
+	const lowercaseHaystack = (t: McpxToolDescriptor) =>
+		`${t.server} ${t.tool.name} ${t.tool.description ?? ""}`.toLowerCase();
+
+	if (hint) {
+		const needle = hint.toLowerCase();
+		const matched = tools.filter((t) => lowercaseHaystack(t).includes(needle));
+		if (matched.length > 0) return matched;
+	}
+
+	if (host) {
+		const tokens = host.split(".");
+		const matched = tools.filter((t) => tokens.some((tok) => tok.length > 2 && lowercaseHaystack(t).includes(tok)));
+		if (matched.length > 0) return matched;
+	}
+
+	// Fall back to any tool that looks like a URL fetcher.
+	return tools.filter((t) => /fetch|scrape|http|url/i.test(`${t.tool.name} ${t.tool.description ?? ""}`));
+}
+
+/** Compose semantic-search queries to feed mcpx.search. */
+function buildQueries(hint: string | undefined, host: string | null): string[] {
+	const out: string[] = [];
+	if (hint) out.push(`${hint} fetch markdown`);
+	if (host) out.push(`fetch ${host} as markdown`, `scrape ${host}`);
+	out.push("fetch URL as markdown", "scrape webpage to markdown");
+	return out;
+}
+
+/** URL → hostname or null. */
+function safeHost(url: string): string | null {
+	try {
+		return new URL(url).hostname.toLowerCase();
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Among the candidate tools, prefer one whose name or description signals
+ * markdown output (contains "markdown", "md", "Docmd", etc.). Falls back
+ * to anything that looks like a generic fetch/scrape verb, and finally
+ * to the first candidate so we always try something.
+ */
+function pickTool(tools: McpxToolDescriptor[]): McpxToolDescriptor | null {
+	const score = (t: McpxToolDescriptor) => {
+		const hay = `${t.tool.name} ${t.tool.description ?? ""}`.toLowerCase();
+		let s = 0;
+		if (/markdown|docmd|asmd|\bmd\b/.test(hay)) s += 5;
+		if (/scrape|extract|fetch|get|read/.test(hay)) s += 2;
+		if (/url|web|html|page/.test(hay)) s += 1;
+		return s;
+	};
+	const sorted = [...tools].sort((a, b) => score(b) - score(a));
+	return sorted[0] ?? null;
+}
+
+/**
+ * Build the argument object the mcpx fetcher tool likely accepts. We can't
+ * know the schema without calling info(), so we build a permissive bag with
+ * the common shapes (`{url, format: "markdown", formats: ["markdown"]}`)
+ * and trust the underlying tool to ignore unknown fields.
+ */
+function buildArgs(toolName: string, url: string): Record<string, unknown> {
+	const args: Record<string, unknown> = { url };
+	if (/markdown|md/i.test(toolName)) args.format = "markdown";
+	args.formats = ["markdown"];
+	return args;
+}
+
+/** Pull a string out of the heterogeneous shapes mcpx tools return. */
+function extractText(result: unknown): string {
+	if (typeof result === "string") return result;
+	if (result && typeof result === "object") {
+		const maybe = result as Record<string, unknown>;
+		if (typeof maybe.text === "string") return maybe.text;
+		if (typeof maybe.content === "string") return maybe.content;
+		if (typeof maybe.markdown === "string") return maybe.markdown;
+		if (Array.isArray(maybe.content)) {
+			const out: string[] = [];
+			for (const c of maybe.content) {
+				if (c && typeof c === "object") {
+					const inner = c as Record<string, unknown>;
+					if (typeof inner.text === "string") out.push(inner.text);
+				}
+			}
+			if (out.length > 0) return out.join("\n\n");
+		}
+	}
+	try {
+		return JSON.stringify(result);
+	} catch {
+		return "";
+	}
+}