membot 0.5.2 → 0.7.0

package/src/context.ts

@@ -1,8 +1,7 @@
 import { join } from "node:path";
-import { McpxClient } from "@evantahler/mcpx";
 import { loadConfig } from "./config/loader.ts";
 import type { MembotConfig } from "./config/schemas.ts";
-import { ENV, FILES } from "./constants.ts";
+import { FILES } from "./constants.ts";
 import { type DbConnection, openDb } from "./db/connection.ts";
 import { logger } from "./output/logger.ts";
 import type { Progress } from "./output/progress.ts";
@@ -16,7 +15,6 @@ export interface AppContext {
 	db: DbConnection;
 	logger: typeof logger;
 	progress: Progress;
-	mcpx: McpxClient | null;
 }
 
 export interface BuildContextOptions {
@@ -32,7 +30,6 @@ export interface BuildContextOptions {
  *  - output mode (TTY/JSON/color detection — frozen for the rest of the run)
  *  - config (~/.membot/config.json with env overrides)
  *  - DuckDB connection (~/.membot/index.duckdb), running migrations on first open
- *  - mcpx client (lazy — opened on first remote fetch; null when no servers)
  */
 export async function buildContext(options: BuildContextOptions = {}): Promise<AppContext> {
 	setMode(detectMode({ json: options.json, verbose: options.verbose, noColor: options.noColor }));
@@ -45,8 +42,6 @@ export async function buildContext(options: BuildContextOptions = {}): Promise<A
 		maxDelayMs: config.db_lock_retry.max_delay_ms,
 	});
 
-	const mcpx = await maybeMcpx(config);
-
 	return {
 		config,
 		dataDir,
@@ -54,31 +49,13 @@ export async function buildContext(options: BuildContextOptions = {}): Promise<A
 		db,
 		logger,
 		progress: createProgress(),
-		mcpx,
 	};
 }
 
-async function maybeMcpx(config: MembotConfig): Promise<McpxClient | null> {
-	const configDir = config.mcpx.config_path || process.env[ENV.MCPX_CONFIG_PATH];
-	try {
-		const client = new McpxClient(configDir ? { configDir } : {});
-		return client;
-	} catch {
-		return null;
-	}
-}
-
 export async function closeContext(ctx: AppContext): Promise<void> {
 	try {
 		await ctx.db.close();
 	} catch {
 		// best effort
 	}
-	if (ctx.mcpx) {
-		try {
-			await ctx.mcpx.close();
-		} catch {
-			// best effort
-		}
-	}
 }

package/src/db/files.ts

@@ -1,7 +1,7 @@
 import type { DbConnection, SqlParam } from "./connection.ts";
 
 export type SourceType = "local" | "remote" | "inline";
-export type FetcherKind = "http" | "mcpx" | "local" | "inline";
+export type FetcherKind = "downloader" | "local" | "inline";
 
 export interface FileRow {
 	logical_path: string;
@@ -18,9 +18,8 @@ export interface FileRow {
 	mime_type: string | null;
 	size_bytes: number | null;
 	fetcher: FetcherKind | null;
-	fetcher_server: string | null;
-	fetcher_tool: string | null;
-	fetcher_args: Record<string, unknown> | null;
+	downloader: string | null;
+	downloader_args: Record<string, unknown> | null;
 	refresh_frequency_sec: number | null;
 	refreshed_at: string | null;
 	last_refresh_status: string | null;
@@ -43,9 +42,8 @@ export interface NewFileVersion {
 	mime_type?: string | null;
 	size_bytes?: number | null;
 	fetcher?: FetcherKind | null;
-	fetcher_server?: string | null;
-	fetcher_tool?: string | null;
-	fetcher_args?: Record<string, unknown> | null;
+	downloader?: string | null;
+	downloader_args?: Record<string, unknown> | null;
 	refresh_frequency_sec?: number | null;
 	refreshed_at?: string | null;
 	last_refresh_status?: string | null;
@@ -67,9 +65,8 @@ const ROW_COLUMNS = [
 	"mime_type",
 	"size_bytes",
 	"fetcher",
-	"fetcher_server",
-	"fetcher_tool",
-	"fetcher_args",
+	"downloader",
+	"downloader_args",
 	"refresh_frequency_sec",
 	"refreshed_at",
 	"last_refresh_status",
@@ -86,21 +83,21 @@ const COLUMN_LIST = ROW_COLUMNS.join(", ");
  */
 export async function insertVersion(db: DbConnection, file: NewFileVersion): Promise<string> {
 	const versionId = file.version_id ?? millisIso(Date.now());
-	const fetcherArgsJson = file.fetcher_args ? JSON.stringify(file.fetcher_args) : null;
+	const downloaderArgsJson = file.downloader_args ? JSON.stringify(file.downloader_args) : null;
 
 	await db.queryRun(
 		`INSERT INTO files (
 			logical_path, version_id, tombstone, source_type,
 			source_path, source_mtime_ms, source_sha256, blob_sha256,
 			content_sha256, content, description, mime_type, size_bytes,
-			fetcher, fetcher_server, fetcher_tool, fetcher_args,
+			fetcher, downloader, downloader_args,
 			refresh_frequency_sec, refreshed_at, last_refresh_status, change_note
 		) VALUES (
 			?1, CAST(?2 AS TIMESTAMP), ?3, ?4,
 			?5, ?6, ?7, ?8,
 			?9, ?10, ?11, ?12, ?13,
-			?14, ?15, ?16, ?17,
-			?18, ?19, ?20, ?21
+			?14, ?15, ?16,
+			?17, ?18, ?19, ?20
 		)`,
 		file.logical_path,
 		versionId,
@@ -116,9 +113,8 @@ export async function insertVersion(db: DbConnection, file: NewFileVersion): Pro
 		file.mime_type ?? null,
 		file.size_bytes ?? null,
 		file.fetcher ?? null,
-		file.fetcher_server ?? null,
-		file.fetcher_tool ?? null,
-		fetcherArgsJson,
+		file.downloader ?? null,
+		downloaderArgsJson,
 		file.refresh_frequency_sec ?? null,
 		file.refreshed_at ?? null,
 		file.last_refresh_status ?? null,
@@ -132,33 +128,33 @@ export function millisIso(ms: number): string {
 	return new Date(ms).toISOString();
 }
 
-interface RawFileRow extends Omit<FileRow, "fetcher_args" | "tombstone"> {
-	fetcher_args: string | null | Record<string, unknown>;
+interface RawFileRow extends Omit<FileRow, "downloader_args" | "tombstone"> {
+	downloader_args: string | null | Record<string, unknown>;
 	tombstone: boolean | number;
 	[key: string]: unknown;
 }
 
 /**
  * Coerce a raw DuckDB row into a typed `FileRow`. JSON-parses the
- * `fetcher_args` column (DuckDB returns it as text or a parsed object
+ * `downloader_args` column (DuckDB returns it as text or a parsed object
  * depending on driver version) and normalizes `tombstone` to a boolean
  * (some drivers return 0/1).
  */
 function toFileRow(row: RawFileRow | null): FileRow | null {
 	if (!row) return null;
 	let parsed: Record<string, unknown> | null = null;
-	if (row.fetcher_args && typeof row.fetcher_args === "string") {
+	if (row.downloader_args && typeof row.downloader_args === "string") {
 		try {
-			parsed = JSON.parse(row.fetcher_args);
+			parsed = JSON.parse(row.downloader_args);
 		} catch {
 			parsed = null;
 		}
-	} else if (row.fetcher_args && typeof row.fetcher_args === "object") {
-		parsed = row.fetcher_args;
+	} else if (row.downloader_args && typeof row.downloader_args === "object") {
+		parsed = row.downloader_args;
 	}
 	return {
 		...row,
-		fetcher_args: parsed,
+		downloader_args: parsed,
 		tombstone: !!row.tombstone,
 	};
 }

package/src/db/migrations/003-downloader-columns.ts

@@ -0,0 +1,58 @@
+import type { Migration } from "../migrations.ts";
+
+/**
+ * Replace the old mcpx-era fetcher metadata triple
+ * (`fetcher_server` / `fetcher_tool` / `fetcher_args`) with a flat
+ * `(downloader, downloader_args)` shape. The mcpx-driven agent fetcher
+ * is gone; per-service downloaders match a URL → run a known fetch
+ * (Playwright export endpoints for Google, rendered HTML for GitHub /
+ * Linear, headless print-to-PDF for everything else) → return bytes
+ * for the existing native converter pipeline.
+ *
+ * Existing rows whose `fetcher` was `'http'` or `'mcpx'` are migrated
+ * to `'downloader'` with `downloader=NULL`. The mcpx-driven ones
+ * become refresh-broken (the `fetcher_*` arguments that drove them no
+ * longer exist) but their stored `content` is still readable; the
+ * plain-HTTP ones will be re-routed through the generic-web downloader
+ * the next time refresh runs. The `fetcher` enum loses both `'http'`
+ * and `'mcpx'` — every remote row is `'downloader'` now, since even
+ * the plain-HTTP fallback is wrapped by the generic-web downloader.
+ *
+ * The `current_files` view is `SELECT f.* FROM files f`, so it pins the
+ * old column shape; we drop and recreate it (and the dependent
+ * `current_chunks` view) around the schema change.
+ */
+export const MIGRATION_003: Migration = {
+	id: 3,
+	name: "downloader-columns",
+	statements: [
+		// DuckDB refuses DROP COLUMN when an index covers any column that
+		// comes AFTER the dropped one in the schema, so the indexes have
+		// to come down first. The view drops are needed for the same
+		// reason — `current_files` is `SELECT f.*`, which pins every
+		// column at view-creation time.
+		`DROP VIEW IF EXISTS current_chunks`,
+		`DROP VIEW IF EXISTS current_files`,
+		`DROP INDEX IF EXISTS files_refresh_due_idx`,
+		`DROP INDEX IF EXISTS files_blob_sha256_idx`,
+		`DROP INDEX IF EXISTS files_logical_path_idx`,
+		`UPDATE files SET fetcher = 'downloader' WHERE fetcher IN ('http', 'mcpx')`,
+		`ALTER TABLE files DROP COLUMN fetcher_server`,
+		`ALTER TABLE files DROP COLUMN fetcher_tool`,
+		`ALTER TABLE files DROP COLUMN fetcher_args`,
+		`ALTER TABLE files ADD COLUMN downloader TEXT`,
+		`ALTER TABLE files ADD COLUMN downloader_args JSON`,
+		`CREATE INDEX files_logical_path_idx ON files (logical_path)`,
+		`CREATE INDEX files_blob_sha256_idx ON files (blob_sha256)`,
+		`CREATE INDEX files_refresh_due_idx ON files (refresh_frequency_sec, refreshed_at)`,
+		`CREATE VIEW current_files AS
+			SELECT f.* FROM files f
+			WHERE (f.logical_path, f.version_id) IN (
+				SELECT logical_path, MAX(version_id) FROM files GROUP BY logical_path
+			)
+				AND f.tombstone = FALSE`,
+		`CREATE VIEW current_chunks AS
+			SELECT c.* FROM chunks c
+			JOIN current_files cf USING (logical_path, version_id)`,
+	],
+};

package/src/db/migrations.ts

@@ -2,6 +2,7 @@ import { logger } from "../output/logger.ts";
 import type { DbConnection } from "./connection.ts";
 import { MIGRATION_001 } from "./migrations/001-init.ts";
 import { MIGRATION_002 } from "./migrations/002-fts.ts";
+import { MIGRATION_003 } from "./migrations/003-downloader-columns.ts";
 
 /**
  * One DDL/DML migration step. The id is monotonically increasing; the name
@@ -14,7 +15,7 @@ export interface Migration {
 	statements: string[];
 }
 
-const MIGRATIONS: Migration[] = [MIGRATION_001, MIGRATION_002];
+const MIGRATIONS: Migration[] = [MIGRATION_001, MIGRATION_002, MIGRATION_003];
 
 /**
  * Process-level cache of paths whose migrations have been applied (or

package/src/ingest/converter/docx.ts

@@ -1,15 +1,57 @@
 import mammoth from "mammoth";
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { type CapturedImage, inlineImageCaptions, MEMBOT_IMG_PREFIX } from "./images-inline.ts";
 
 const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
 
+/**
+ * Mammoth's image element wears an `altText` field that isn't reflected in
+ * the published `.d.ts`. We declare the bits we actually touch so the rest
+ * of the module can stay strict-typed.
+ */
+interface MammothImage {
+	contentType: string;
+	altText?: string;
+	readAsBuffer: () => Promise<Buffer>;
+}
+
 /**
  * Convert a DOCX file to markdown. Mammoth gives us HTML; we then run that
- * through turndown to get clean markdown. Any conversion warnings are
- * silently dropped — they're typically about styles we don't preserve.
+ * through turndown to get clean markdown. Embedded images (which mammoth
+ * would otherwise inline as 5MB base64 `data:` URIs) are intercepted and
+ * replaced with `membot-img://<id>` placeholders, then expanded into Claude
+ * vision captions by `inlineImageCaptions`. Conversion warnings from
+ * mammoth are silently dropped — they're typically about styles we don't
+ * preserve.
  */
-export async function convertDocx(bytes: Uint8Array): Promise<string> {
+export async function convertDocx(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const buf = Buffer.from(bytes);
-	const result = await mammoth.convertToHtml({ buffer: buf });
-	return turndown.turndown(result.value).trim();
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+
+	const result = await mammoth.convertToHtml(
+		{ buffer: buf },
+		{
+			convertImage: mammoth.images.imgElement(async (image) => {
+				const img = image as unknown as MammothImage;
+				const id = `img-${counter++}`;
+				try {
+					const buffer = await img.readAsBuffer();
+					images.set(id, {
+						bytes: new Uint8Array(buffer),
+						mimeType: img.contentType,
+						altText: img.altText,
+					});
+				} catch {
+					// If we can't read the image bytes, still emit the placeholder so
+					// turndown doesn't fall back to a giant inline data URI.
+				}
+				return { src: `${MEMBOT_IMG_PREFIX}${id}` };
+			}),
+		},
+	);
+
+	const md = turndown.turndown(result.value).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/html.ts

@@ -1,4 +1,6 @@
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { extractDataUriImages, inlineImageCaptions } from "./images-inline.ts";
 
 const turndown = new TurndownService({
 	headingStyle: "atx",
@@ -8,13 +10,18 @@ const turndown = new TurndownService({
 
 /**
  * Convert HTML bytes to markdown using turndown. Strips script/style blocks
- * before conversion so they don't leak into the chunker.
+ * before conversion so they don't leak into the chunker. Inline data-URI
+ * images are extracted into their bytes and replaced with vision captions
+ * via `inlineImageCaptions`; external `<img src="https://…">` references
+ * are left for turndown to render normally.
  */
-export function convertHtml(bytes: Uint8Array): string {
+export async function convertHtml(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const html = new TextDecoder("utf-8").decode(bytes);
 	const cleaned = html
 		.replace(/<script[\s\S]*?<\/script>/gi, "")
 		.replace(/<style[\s\S]*?<\/style>/gi, "")
 		.replace(/<noscript[\s\S]*?<\/noscript>/gi, "");
-	return turndown.turndown(cleaned).trim();
+	const { html: rewritten, images } = extractDataUriImages(cleaned);
+	const md = turndown.turndown(rewritten).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/image.ts

@@ -12,6 +12,13 @@ Output the caption only, no preamble.`;
 
 const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
 
+/** Anthropic vision rejects images > 5MB; stay under that with margin. */
+const VISION_MAX_BYTES = 4 * 1024 * 1024;
+/** Tesseract is roughly linear in pixel count; bail past this byte size to avoid pathological hangs. */
+const OCR_MAX_BYTES = 8 * 1024 * 1024;
+/** Hard wall-clock for either subtask so a stuck network call never freezes ingest. */
+const SUBTASK_TIMEOUT_MS = 60_000;
+
 /**
  * Build the markdown surrogate for an image: an LLM-generated caption
  * (when an API key is available) folded together with any text recovered
@@ -19,17 +26,47 @@ const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/web
  * when no API key is set.
  */
 export async function convertImage(bytes: Uint8Array, mimeType: string, llm: LlmConfig): Promise<string> {
-	const captionPromise = describeImage(bytes, mimeType, llm);
-	const ocrPromise = ocrImage(bytes);
+	const captionPromise =
+		bytes.byteLength <= VISION_MAX_BYTES
+			? withTimeout(describeImage(bytes, mimeType, llm), SUBTASK_TIMEOUT_MS, "vision")
+			: Promise.resolve("");
+	const ocrPromise =
+		bytes.byteLength <= OCR_MAX_BYTES ? withTimeout(ocrImage(bytes), SUBTASK_TIMEOUT_MS, "ocr") : Promise.resolve("");
 	const [caption, ocrText] = await Promise.all([captionPromise, ocrPromise]);
 
 	const sections: string[] = [];
 	if (caption) sections.push(caption);
 	if (ocrText) sections.push(`## Text detected via OCR\n\n${ocrText}`);
-	if (sections.length === 0) sections.push(`(image, ${mimeType}, no caption available)`);
+	if (sections.length === 0) {
+		const note =
+			bytes.byteLength > VISION_MAX_BYTES
+				? `(image, ${mimeType}, ${bytes.byteLength} bytes — exceeds vision size limit, no caption available)`
+				: `(image, ${mimeType}, no caption available)`;
+		sections.push(note);
+	}
 	return sections.join("\n\n");
 }
 
+/**
+ * Race a promise against a timer so a stuck network call (vision) or a
+ * pathological CPU-bound job (OCR on a multi-megapixel image) never freezes
+ * the whole conversion pipeline. Logs a warning when the timer wins.
+ */
+async function withTimeout<T extends string>(p: Promise<T>, ms: number, label: string): Promise<T | ""> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const timeout = new Promise<"">((resolve) => {
+		timer = setTimeout(() => {
+			logger.warn(`image: ${label} timed out after ${ms}ms`);
+			resolve("");
+		}, ms);
+	});
+	try {
+		return await Promise.race([p, timeout]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
 /**
  * Single-shot vision call asking Claude to caption an image. Returns the
  * caption text or an empty string when the API key is missing or the

package/src/ingest/converter/images-inline.ts

@@ -0,0 +1,132 @@
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { logger } from "../../output/logger.ts";
+import { convertImage } from "./image.ts";
+
+/**
+ * Bytes captured from an embedded image during DOCX/HTML conversion. The
+ * image-inlining helpers run `convertImage` over each one to produce a
+ * markdown caption that gets spliced back into the document body in place
+ * of the original `<img>` reference.
+ */
+export interface CapturedImage {
+	bytes: Uint8Array;
+	mimeType: string;
+	altText?: string;
+}
+
+/** URI scheme used to mark images that the inliner should expand. */
+export const MEMBOT_IMG_PREFIX = "membot-img://";
+
+/**
+ * Match `![alt](membot-img://<id>)` markdown image references. The id may
+ * contain any non-whitespace, non-`)` character so we don't accidentally
+ * stop at characters mammoth/turndown might emit inside an id.
+ */
+const TOKEN_RE = /!\[([^\]]*)\]\(membot-img:\/\/([^)\s]+)\)/g;
+
+/**
+ * Extract data-URI images from raw HTML and rewrite each `<img src="data:…">`
+ * to `<img src="membot-img://<id>">`. The captured bytes flow through the
+ * shared `inlineImageCaptions` step so HTML and DOCX share one captioning
+ * code path. Non-data `<img>` references are left untouched.
+ */
+export function extractDataUriImages(html: string): { html: string; images: Map<string, CapturedImage> } {
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+	const rewritten = html.replace(
+		/<img\b([^>]*?)\bsrc\s*=\s*(?:"data:([^";]+);base64,([^"]*)"|'data:([^';]+);base64,([^']*)')([^>]*)>/gi,
+		(
+			_match,
+			beforeSrc: string,
+			mimeDouble: string | undefined,
+			b64Double: string | undefined,
+			mimeSingle: string | undefined,
+			b64Single: string | undefined,
+			afterSrc: string,
+		) => {
+			const mimeType = (mimeDouble ?? mimeSingle ?? "image/png").trim();
+			const b64 = (b64Double ?? b64Single ?? "").replace(/\s+/g, "");
+			const id = `img-${counter++}`;
+			try {
+				const bytes = new Uint8Array(Buffer.from(b64, "base64"));
+				images.set(id, { bytes, mimeType });
+			} catch (err) {
+				logger.warn(
+					`images-inline: failed to decode embedded image (${err instanceof Error ? err.message : String(err)})`,
+				);
+				return `<img${beforeSrc} src=""${afterSrc}>`;
+			}
+			return `<img${beforeSrc} src="${MEMBOT_IMG_PREFIX}${id}"${afterSrc}>`;
+		},
+	);
+	return { html: rewritten, images };
+}
+
+/**
+ * Replace each `![alt](membot-img://<id>)` token in `markdown` with the
+ * caption produced by `convertImage`. Captures are processed in document
+ * order; once `max_inline_image_captions` (from `ConvertersConfig`) has been
+ * reached, the remaining tokens get a tiny deterministic placeholder rather
+ * than an LLM call so a doc full of embedded images doesn't fan out into
+ * hundreds of vision requests.
+ *
+ * No-ops on a markdown string with no `membot-img://` references; safe to
+ * call unconditionally from the converters.
+ */
+export async function inlineImageCaptions(
+	markdown: string,
+	images: Map<string, CapturedImage>,
+	llm: LlmConfig,
+	converters: ConvertersConfig,
+): Promise<string> {
+	if (images.size === 0) return markdown;
+
+	const captions = new Map<string, string>();
+	const overflow = new Set<string>();
+	let captioned = 0;
+
+	for (const match of markdown.matchAll(TOKEN_RE)) {
+		const alt = match[1] ?? "";
+		const id = match[2];
+		if (!id || captions.has(id) || overflow.has(id)) continue;
+		const img = images.get(id);
+		if (!img) continue;
+
+		if (captioned >= converters.max_inline_image_captions) {
+			overflow.add(id);
+			continue;
+		}
+		captioned++;
+		try {
+			const caption = await convertImage(img.bytes, img.mimeType, llm);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", caption));
+		} catch (err) {
+			logger.warn(`images-inline: caption failed for ${id} (${err instanceof Error ? err.message : String(err)})`);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", `(image, ${img.mimeType}, no caption available)`));
+		}
+	}
+
+	return markdown.replace(TOKEN_RE, (_match, alt: string, id: string) => {
+		const cached = captions.get(id);
+		if (cached) return cached;
+		const img = images.get(id);
+		if (!img) return formatCaptionBlock(alt, "(image, no caption available)");
+		return formatCaptionBlock(
+			alt || img.altText || "",
+			`(image, ${img.mimeType}, ${img.bytes.byteLength} bytes — caption skipped, exceeded max_inline_image_captions)`,
+		);
+	});
+}
+
+/**
+ * Render a captioned image as its own markdown paragraph block. Wrapping the
+ * caption in blank lines guarantees the deterministic chunker sees it as a
+ * paragraph boundary; an HTML comment with the alt text keeps the original
+ * positional cue without polluting search snippets.
+ */
+function formatCaptionBlock(alt: string, caption: string): string {
+	const trimmed = caption.trim();
+	const header = alt.trim() ? `<!-- image: ${alt.trim()} -->` : `<!-- image -->`;
+	const body = trimmed.length > 0 ? trimmed : "(image, no caption available)";
+	return `\n\n${header}\n\n${body}\n\n`;
+}

package/src/ingest/converter/index.ts

@@ -1,4 +1,4 @@
-import type { LlmConfig } from "../../config/schemas.ts";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
 import { convertDocx } from "./docx.ts";
 import { convertHtml } from "./html.ts";
 import { convertImage } from "./image.ts";
@@ -6,6 +6,7 @@ import { convertWithLlm } from "./llm.ts";
 import { ocrImage } from "./ocr.ts";
 import { convertPdf, shouldOcrPdf } from "./pdf.ts";
 import { convertText } from "./text.ts";
+import { convertXlsx } from "./xlsx.ts";
 
 export interface ConvertResult {
 	markdown: string;
@@ -25,6 +26,10 @@ const STRUCTURED_TEXT_MIMES = new Set([
 	"application/typescript",
 ]);
 const DOCX_MIMES = new Set(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]);
+const XLSX_MIMES = new Set([
+	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+	"application/vnd.ms-excel",
+]);
 const PDF_MIMES = new Set(["application/pdf"]);
 
 /**
@@ -39,6 +44,7 @@ export async function convert(
 	mimeType: string,
 	source: string,
 	llm: LlmConfig,
+	converters: ConvertersConfig,
 ): Promise<ConvertResult> {
 	const mt = mimeType.toLowerCase();
 
@@ -47,11 +53,15 @@ export async function convert(
 	}
 
 	if (HTML_MIMES.has(mt)) {
-		return { markdown: convertHtml(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertHtml(bytes, llm, converters), contentMimeType: "text/markdown" };
 	}
 
 	if (DOCX_MIMES.has(mt)) {
-		return { markdown: await convertDocx(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertDocx(bytes, llm, converters), contentMimeType: "text/markdown" };
+	}
+
+	if (XLSX_MIMES.has(mt)) {
+		return { markdown: await convertXlsx(bytes), contentMimeType: "text/markdown" };
 	}
 
 	if (PDF_MIMES.has(mt)) {