membot 0.5.1 → 0.6.0

package/src/db/files.ts

@@ -1,7 +1,7 @@
 import type { DbConnection, SqlParam } from "./connection.ts";
 
 export type SourceType = "local" | "remote" | "inline";
-export type FetcherKind = "http" | "mcpx" | "local" | "inline";
+export type FetcherKind = "downloader" | "local" | "inline";
 
 export interface FileRow {
 	logical_path: string;
@@ -18,9 +18,8 @@ export interface FileRow {
 	mime_type: string | null;
 	size_bytes: number | null;
 	fetcher: FetcherKind | null;
-	fetcher_server: string | null;
-	fetcher_tool: string | null;
-	fetcher_args: Record<string, unknown> | null;
+	downloader: string | null;
+	downloader_args: Record<string, unknown> | null;
 	refresh_frequency_sec: number | null;
 	refreshed_at: string | null;
 	last_refresh_status: string | null;
@@ -43,9 +42,8 @@ export interface NewFileVersion {
 	mime_type?: string | null;
 	size_bytes?: number | null;
 	fetcher?: FetcherKind | null;
-	fetcher_server?: string | null;
-	fetcher_tool?: string | null;
-	fetcher_args?: Record<string, unknown> | null;
+	downloader?: string | null;
+	downloader_args?: Record<string, unknown> | null;
 	refresh_frequency_sec?: number | null;
 	refreshed_at?: string | null;
 	last_refresh_status?: string | null;
@@ -67,9 +65,8 @@ const ROW_COLUMNS = [
 	"mime_type",
 	"size_bytes",
 	"fetcher",
-	"fetcher_server",
-	"fetcher_tool",
-	"fetcher_args",
+	"downloader",
+	"downloader_args",
 	"refresh_frequency_sec",
 	"refreshed_at",
 	"last_refresh_status",
@@ -86,21 +83,21 @@ const COLUMN_LIST = ROW_COLUMNS.join(", ");
  */
 export async function insertVersion(db: DbConnection, file: NewFileVersion): Promise<string> {
 	const versionId = file.version_id ?? millisIso(Date.now());
-	const fetcherArgsJson = file.fetcher_args ? JSON.stringify(file.fetcher_args) : null;
+	const downloaderArgsJson = file.downloader_args ? JSON.stringify(file.downloader_args) : null;
 
 	await db.queryRun(
 		`INSERT INTO files (
 			logical_path, version_id, tombstone, source_type,
 			source_path, source_mtime_ms, source_sha256, blob_sha256,
 			content_sha256, content, description, mime_type, size_bytes,
-			fetcher, fetcher_server, fetcher_tool, fetcher_args,
+			fetcher, downloader, downloader_args,
 			refresh_frequency_sec, refreshed_at, last_refresh_status, change_note
 		) VALUES (
 			?1, CAST(?2 AS TIMESTAMP), ?3, ?4,
 			?5, ?6, ?7, ?8,
 			?9, ?10, ?11, ?12, ?13,
-			?14, ?15, ?16, ?17,
-			?18, ?19, ?20, ?21
+			?14, ?15, ?16,
+			?17, ?18, ?19, ?20
 		)`,
 		file.logical_path,
 		versionId,
@@ -116,9 +113,8 @@ export async function insertVersion(db: DbConnection, file: NewFileVersion): Pro
 		file.mime_type ?? null,
 		file.size_bytes ?? null,
 		file.fetcher ?? null,
-		file.fetcher_server ?? null,
-		file.fetcher_tool ?? null,
-		fetcherArgsJson,
+		file.downloader ?? null,
+		downloaderArgsJson,
 		file.refresh_frequency_sec ?? null,
 		file.refreshed_at ?? null,
 		file.last_refresh_status ?? null,
@@ -132,33 +128,33 @@ export function millisIso(ms: number): string {
 	return new Date(ms).toISOString();
 }
 
-interface RawFileRow extends Omit<FileRow, "fetcher_args" | "tombstone"> {
-	fetcher_args: string | null | Record<string, unknown>;
+interface RawFileRow extends Omit<FileRow, "downloader_args" | "tombstone"> {
+	downloader_args: string | null | Record<string, unknown>;
 	tombstone: boolean | number;
 	[key: string]: unknown;
 }
 
 /**
  * Coerce a raw DuckDB row into a typed `FileRow`. JSON-parses the
- * `fetcher_args` column (DuckDB returns it as text or a parsed object
+ * `downloader_args` column (DuckDB returns it as text or a parsed object
  * depending on driver version) and normalizes `tombstone` to a boolean
  * (some drivers return 0/1).
  */
 function toFileRow(row: RawFileRow | null): FileRow | null {
 	if (!row) return null;
 	let parsed: Record<string, unknown> | null = null;
-	if (row.fetcher_args && typeof row.fetcher_args === "string") {
+	if (row.downloader_args && typeof row.downloader_args === "string") {
 		try {
-			parsed = JSON.parse(row.fetcher_args);
+			parsed = JSON.parse(row.downloader_args);
 		} catch {
 			parsed = null;
 		}
-	} else if (row.fetcher_args && typeof row.fetcher_args === "object") {
-		parsed = row.fetcher_args;
+	} else if (row.downloader_args && typeof row.downloader_args === "object") {
+		parsed = row.downloader_args;
 	}
 	return {
 		...row,
-		fetcher_args: parsed,
+		downloader_args: parsed,
 		tombstone: !!row.tombstone,
 	};
 }

package/src/db/migrations/003-downloader-columns.ts

@@ -0,0 +1,58 @@
+import type { Migration } from "../migrations.ts";
+
+/**
+ * Replace the old mcpx-era fetcher metadata triple
+ * (`fetcher_server` / `fetcher_tool` / `fetcher_args`) with a flat
+ * `(downloader, downloader_args)` shape. The mcpx-driven agent fetcher
+ * is gone; per-service downloaders match a URL → run a known fetch
+ * (Playwright export endpoints for Google, rendered HTML for GitHub /
+ * Linear, headless print-to-PDF for everything else) → return bytes
+ * for the existing native converter pipeline.
+ *
+ * Existing rows whose `fetcher` was `'http'` or `'mcpx'` are migrated
+ * to `'downloader'` with `downloader=NULL`. The mcpx-driven ones
+ * become refresh-broken (the `fetcher_*` arguments that drove them no
+ * longer exist) but their stored `content` is still readable; the
+ * plain-HTTP ones will be re-routed through the generic-web downloader
+ * the next time refresh runs. The `fetcher` enum loses both `'http'`
+ * and `'mcpx'` — every remote row is `'downloader'` now, since even
+ * the plain-HTTP fallback is wrapped by the generic-web downloader.
+ *
+ * The `current_files` view is `SELECT f.* FROM files f`, so it pins the
+ * old column shape; we drop and recreate it (and the dependent
+ * `current_chunks` view) around the schema change.
+ */
+export const MIGRATION_003: Migration = {
+	id: 3,
+	name: "downloader-columns",
+	statements: [
+		// DuckDB refuses DROP COLUMN when an index covers any column that
+		// comes AFTER the dropped one in the schema, so the indexes have
+		// to come down first. The view drops are needed for the same
+		// reason — `current_files` is `SELECT f.*`, which pins every
+		// column at view-creation time.
+		`DROP VIEW IF EXISTS current_chunks`,
+		`DROP VIEW IF EXISTS current_files`,
+		`DROP INDEX IF EXISTS files_refresh_due_idx`,
+		`DROP INDEX IF EXISTS files_blob_sha256_idx`,
+		`DROP INDEX IF EXISTS files_logical_path_idx`,
+		`UPDATE files SET fetcher = 'downloader' WHERE fetcher IN ('http', 'mcpx')`,
+		`ALTER TABLE files DROP COLUMN fetcher_server`,
+		`ALTER TABLE files DROP COLUMN fetcher_tool`,
+		`ALTER TABLE files DROP COLUMN fetcher_args`,
+		`ALTER TABLE files ADD COLUMN downloader TEXT`,
+		`ALTER TABLE files ADD COLUMN downloader_args JSON`,
+		`CREATE INDEX files_logical_path_idx ON files (logical_path)`,
+		`CREATE INDEX files_blob_sha256_idx ON files (blob_sha256)`,
+		`CREATE INDEX files_refresh_due_idx ON files (refresh_frequency_sec, refreshed_at)`,
+		`CREATE VIEW current_files AS
+			SELECT f.* FROM files f
+			WHERE (f.logical_path, f.version_id) IN (
+				SELECT logical_path, MAX(version_id) FROM files GROUP BY logical_path
+			)
+				AND f.tombstone = FALSE`,
+		`CREATE VIEW current_chunks AS
+			SELECT c.* FROM chunks c
+			JOIN current_files cf USING (logical_path, version_id)`,
+	],
+};

package/src/db/migrations.ts

@@ -2,6 +2,7 @@ import { logger } from "../output/logger.ts";
 import type { DbConnection } from "./connection.ts";
 import { MIGRATION_001 } from "./migrations/001-init.ts";
 import { MIGRATION_002 } from "./migrations/002-fts.ts";
+import { MIGRATION_003 } from "./migrations/003-downloader-columns.ts";
 
 /**
  * One DDL/DML migration step. The id is monotonically increasing; the name
@@ -14,7 +15,7 @@ export interface Migration {
 	statements: string[];
 }
 
-const MIGRATIONS: Migration[] = [MIGRATION_001, MIGRATION_002];
+const MIGRATIONS: Migration[] = [MIGRATION_001, MIGRATION_002, MIGRATION_003];
 
 /**
  * Process-level cache of paths whose migrations have been applied (or

package/src/ingest/converter/index.ts

@@ -6,6 +6,7 @@ import { convertWithLlm } from "./llm.ts";
 import { ocrImage } from "./ocr.ts";
 import { convertPdf, shouldOcrPdf } from "./pdf.ts";
 import { convertText } from "./text.ts";
+import { convertXlsx } from "./xlsx.ts";
 
 export interface ConvertResult {
 	markdown: string;
@@ -25,6 +26,10 @@ const STRUCTURED_TEXT_MIMES = new Set([
 	"application/typescript",
 ]);
 const DOCX_MIMES = new Set(["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]);
+const XLSX_MIMES = new Set([
+	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+	"application/vnd.ms-excel",
+]);
 const PDF_MIMES = new Set(["application/pdf"]);
 
 /**
@@ -54,6 +59,10 @@ export async function convert(
 		return { markdown: await convertDocx(bytes), contentMimeType: "text/markdown" };
 	}
 
+	if (XLSX_MIMES.has(mt)) {
+		return { markdown: await convertXlsx(bytes), contentMimeType: "text/markdown" };
+	}
+
 	if (PDF_MIMES.has(mt)) {
 		const conversion = await convertPdf(bytes);
 		if (!shouldOcrPdf(conversion)) {

package/src/ingest/converter/xlsx.ts

@@ -0,0 +1,111 @@
+import * as XLSX from "xlsx";
+
+export interface ConvertXlsxOptions {
+	/** Optional sublabel callback driven per-sheet (`parsing 3/8 tabs`). */
+	onProgress?: (sublabel: string) => void;
+}
+
+/**
+ * Convert an XLSX workbook into markdown — one `## <SheetName>`
+ * section per tab, each tab rendered as a GitHub-flavored pipe table
+ * with the first non-empty row treated as the header. Empty sheets
+ * are skipped. Cell values are stringified (numbers, dates, formulas
+ * use their displayed value via `XLSX.utils.format_cell`).
+ *
+ * Pure-JS via SheetJS — no native deps, bundles cleanly with
+ * `bun build --compile`. Yields a macrotask between sheets so
+ * nanospinner's setInterval keeps animating during big workbooks
+ * (otherwise the spinner visibly freezes).
+ */
+export async function convertXlsx(bytes: Uint8Array, opts: ConvertXlsxOptions = {}): Promise<string> {
+	const workbook = XLSX.read(bytes, { type: "array", cellDates: true });
+	const sections: string[] = [];
+	const sheetNames = workbook.SheetNames;
+
+	for (let i = 0; i < sheetNames.length; i++) {
+		const sheetName = sheetNames[i] as string;
+		opts.onProgress?.(`parsing ${i + 1}/${sheetNames.length} tabs`);
+		const sheet = workbook.Sheets[sheetName];
+		if (sheet) {
+			const rows = sheetToMatrix(sheet);
+			const trimmed = trimEmptyEdges(rows);
+			if (trimmed.length > 0) sections.push(`## ${sheetName}\n\n${renderTable(trimmed)}`);
+		}
+		// Yield so the spinner can repaint between sheets — large
+		// workbooks would otherwise freeze the UI for the duration of
+		// the parse.
+		await new Promise<void>((resolve) => setTimeout(resolve, 0));
+	}
+
+	if (sections.length === 0) return "(empty workbook)";
+	return sections.join("\n\n");
+}
+
+/**
+ * Walk every cell in the sheet's used range and produce a 2-D array
+ * of display strings. Uses the cell's formatted text (e.g. dates as
+ * "2026-05-09", percentages as "12.5%") rather than raw values, so
+ * the markdown matches what a human sees in the spreadsheet.
+ */
+function sheetToMatrix(sheet: XLSX.WorkSheet): string[][] {
+	if (!sheet["!ref"]) return [];
+	const range = XLSX.utils.decode_range(sheet["!ref"]);
+	const out: string[][] = [];
+	for (let r = range.s.r; r <= range.e.r; r++) {
+		const row: string[] = [];
+		for (let c = range.s.c; c <= range.e.c; c++) {
+			const addr = XLSX.utils.encode_cell({ r, c });
+			const cell = sheet[addr];
+			row.push(cell ? XLSX.utils.format_cell(cell) : "");
+		}
+		out.push(row);
+	}
+	return out;
+}
+
+/**
+ * Drop fully-empty leading/trailing rows and columns. Spreadsheets
+ * commonly have the used range padded out beyond the actual data.
+ */
+function trimEmptyEdges(rows: string[][]): string[][] {
+	if (rows.length === 0) return rows;
+	let firstRow = 0;
+	let lastRow = rows.length - 1;
+	while (firstRow <= lastRow && rows[firstRow]?.every((v) => v === "")) firstRow++;
+	while (lastRow >= firstRow && rows[lastRow]?.every((v) => v === "")) lastRow--;
+	if (firstRow > lastRow) return [];
+	const sliced = rows.slice(firstRow, lastRow + 1);
+	const cols = sliced[0]?.length ?? 0;
+	let firstCol = 0;
+	let lastCol = cols - 1;
+	while (firstCol <= lastCol && sliced.every((r) => (r[firstCol] ?? "") === "")) firstCol++;
+	while (lastCol >= firstCol && sliced.every((r) => (r[lastCol] ?? "") === "")) lastCol--;
+	if (firstCol > lastCol) return [];
+	return sliced.map((r) => r.slice(firstCol, lastCol + 1));
+}
+
+/**
+ * Render a 2-D matrix as a GitHub pipe table. The first row becomes
+ * the header. Pipe and newline characters in cells are escaped so
+ * they don't break the table layout.
+ */
+function renderTable(rows: string[][]): string {
+	const colCount = Math.max(...rows.map((r) => r.length));
+	const norm = rows.map((r) => {
+		const padded = [...r];
+		while (padded.length < colCount) padded.push("");
+		return padded.map(escapeCell);
+	});
+	const lines: string[] = [];
+	const header = norm[0] ?? Array(colCount).fill("");
+	lines.push(`| ${header.join(" | ")} |`);
+	lines.push(`| ${Array(colCount).fill("---").join(" | ")} |`);
+	for (let i = 1; i < norm.length; i++) {
+		lines.push(`| ${(norm[i] as string[]).join(" | ")} |`);
+	}
+	return lines.join("\n");
+}
+
+function escapeCell(value: string): string {
+	return value.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}

package/src/ingest/downloaders/browser.ts

@@ -0,0 +1,180 @@
+import { mkdir } from "node:fs/promises";
+import type { APIRequestContext, BrowserContext, Page } from "playwright";
+import { HelpfulError } from "../../errors.ts";
+
+let chromiumModule: typeof import("playwright").chromium | null = null;
+
+/**
+ * Lazy-import `playwright.chromium`. Keeping the import deferred so the
+ * heavy module isn't loaded on cold paths (e.g. `membot list`); ALSO
+ * lets us produce a `HelpfulError` if Playwright isn't installed yet
+ * instead of a stack trace at module-load time.
+ */
+async function loadChromium(): Promise<typeof import("playwright").chromium> {
+	if (chromiumModule) return chromiumModule;
+	try {
+		const playwright = await import("playwright");
+		chromiumModule = playwright.chromium;
+		return chromiumModule;
+	} catch (err) {
+		throw new HelpfulError({
+			kind: "internal_error",
+			message: `failed to load playwright: ${err instanceof Error ? err.message : String(err)}`,
+			hint: "Run `bun add -g membot` to reinstall, then `bunx playwright install chromium` to fetch the browser binary.",
+		});
+	}
+}
+
+export interface BrowserPoolOptions {
+	userDataDir: string;
+	headless?: boolean;
+}
+
+/**
+ * Process-scoped lazy-launched chromium context backed by a *persistent
+ * profile directory* (`launchPersistentContext`). Persistent profiles
+ * survive cookies, localStorage, sessionStorage, IndexedDB, and service
+ * worker state across runs — necessary for SPA-heavy services like
+ * Linear that stash critical session/sync state in IndexedDB (which
+ * the lighter `storageState` JSON snapshot doesn't capture).
+ *
+ * Trade-offs:
+ *  - The profile is a directory, not a single JSON file (a few MBs).
+ *  - Chromium's single-instance lock means only one BrowserPool can
+ *    have the profile open at a time. Sequential `membot add` calls
+ *    are fine; concurrent CLI processes against the same profile will
+ *    fail to launch.
+ */
+export class BrowserPool {
+	private readonly userDataDir: string;
+	private readonly headless: boolean;
+	private context: BrowserContext | null = null;
+
+	constructor(options: BrowserPoolOptions) {
+		this.userDataDir = options.userDataDir;
+		this.headless = options.headless ?? true;
+	}
+
+	/**
+	 * Lazy-init the persistent context. The first call launches
+	 * chromium against `userDataDir` (creating it if needed); subsequent
+	 * calls reuse the same context so cookies, IDB, and inflight
+	 * navigation state stay shared across downloaders within one run.
+	 */
+	private async ensureContext(): Promise<BrowserContext> {
+		if (this.context) return this.context;
+		const chromium = await loadChromium();
+		await mkdir(this.userDataDir, { recursive: true });
+		try {
+			this.context = await chromium.launchPersistentContext(this.userDataDir, {
+				headless: this.headless,
+			});
+		} catch (err) {
+			throw new HelpfulError({
+				kind: "internal_error",
+				message: `chromium failed to launch: ${err instanceof Error ? err.message : String(err)}`,
+				hint: this.headless
+					? "Run `bunx playwright install chromium` to download the browser binary."
+					: "Close any other membot process holding the browser profile, then retry.",
+			});
+		}
+		return this.context;
+	}
+
+	/** Return the request context for downloaders that just need authenticated HTTP. */
+	async request(): Promise<APIRequestContext> {
+		const ctx = await this.ensureContext();
+		return ctx.request;
+	}
+
+	/** Open a fresh page (caller is responsible for `page.close()`). */
+	async newPage(): Promise<Page> {
+		const ctx = await this.ensureContext();
+		return ctx.newPage();
+	}
+
+	/**
+	 * How many cookies are in the live context. Used by the auth-prompt
+	 * flow to detect "user closed the window without logging in" — must
+	 * be called BEFORE `dispose()` since the context closes its own
+	 * stores when shutting down.
+	 */
+	async cookieCount(): Promise<number> {
+		if (!this.context) return 0;
+		try {
+			const cookies = await this.context.cookies();
+			return cookies.length;
+		} catch {
+			return 0;
+		}
+	}
+
+	/**
+	 * Return the cookies stored in the persistent profile for a given
+	 * URL/origin (or all cookies when omitted). Used by downloaders that
+	 * call services with their own HTTP client (e.g. Node's built-in
+	 * `fetch`) — they read the cookies once here and pass them via a
+	 * `Cookie` header. Bypasses Playwright's APIRequestContext, which
+	 * has a known cookie-parser bug on Google's same-origin redirects.
+	 */
+	async cookieHeader(url: string): Promise<string> {
+		const ctx = await this.ensureContext();
+		const cookies = await ctx.cookies(url);
+		return cookies.map((c) => `${c.name}=${c.value}`).join("; ");
+	}
+
+	/**
+	 * Resolve when the user is "done" with the headed browser session,
+	 * detected as: the supplied page closes, OR its context closes, OR
+	 * the underlying browser disconnects — whichever fires first. We
+	 * can't rely on the browser-disconnect event alone: on macOS,
+	 * closing the last window does NOT quit chromium (the app stays
+	 * alive in the background), so the disconnect event never fires
+	 * and the caller hangs forever. The page-close event is the only
+	 * signal that's consistent across macOS, Linux, and Windows.
+	 */
+	async waitForUserDone(page: Page): Promise<void> {
+		const ctx = page.context();
+		const browser = ctx.browser();
+		await new Promise<void>((resolve) => {
+			let done = false;
+			const finish = () => {
+				if (done) return;
+				done = true;
+				resolve();
+			};
+			page.on("close", finish);
+			ctx.on("close", finish);
+			browser?.on("disconnected", finish);
+			if (page.isClosed() || (browser && !browser.isConnected())) finish();
+		});
+	}
+
+	/** Close the context (which releases the userDataDir lock). Idempotent. */
+	async dispose(): Promise<void> {
+		try {
+			await this.context?.close();
+		} catch {
+			// best-effort
+		}
+		this.context = null;
+	}
+}
+
+/**
+ * Resolve `maybeRelative` against `base` and return a `URL`, or `null`
+ * if neither parses. Playwright's `APIResponse.url()` sometimes hands
+ * back a path-only string (`"/"`) instead of an absolute URL after a
+ * same-origin redirect — every downloader that wants to inspect the
+ * final URL goes through this helper so the relative-URL handling
+ * lives in one place. Login-redirect detection itself is each
+ * downloader's responsibility — it's the only code that knows which
+ * host its export endpoint redirects to when the session is missing.
+ */
+export function safeResolveUrl(maybeRelative: string, base: string): URL | null {
+	try {
+		return new URL(maybeRelative, base);
+	} catch {
+		return null;
+	}
+}

package/src/ingest/downloaders/generic-web.ts

@@ -0,0 +1,81 @@
+import { HelpfulError } from "../../errors.ts";
+import { sha256Hex } from "../local-reader.ts";
+import type { DownloadedRemote, Downloader } from "./index.ts";
+
+/**
+ * Catch-all downloader. Always matches HTTP/HTTPS URLs that no
+ * specific downloader claimed. Strategy:
+ *  - Issue an authenticated GET via Playwright's request context
+ *    (cookies from `membot login` flow through automatically).
+ *  - If the server returned `text/html`, the page is probably a SPA
+ *    or auth-gated render — open a real `page`, wait for
+ *    `networkidle`, and `page.pdf()` the visible result. The rendered
+ *    PDF goes through `convertPdf` so SPAs and login-walled docs
+ *    work uniformly.
+ *  - Otherwise the response IS the file (markdown, JSON, PDF, image,
+ *    docx, …) — return its bytes verbatim and let the mime
+ *    dispatcher pick the right native converter.
+ *
+ * This is what gives "no specific downloader needed" coverage to any
+ * URL the user throws at `membot add`.
+ */
+export const genericWebDownloader: Downloader = {
+	name: "generic-web",
+	description:
+		"Catch-all for any URL no other downloader handled — HEAD/GET, then either page.pdf() the rendered HTML or stream the raw bytes through the mime converter.",
+	matches(url) {
+		return url.protocol === "http:" || url.protocol === "https:";
+	},
+	async download(url, ctx): Promise<DownloadedRemote> {
+		ctx.onProgress?.("fetching");
+		const request = await ctx.pool.request();
+		const response = await request.get(url.toString(), { timeout: 30_000 });
+		// As the catch-all we don't know which login page each unknown
+		// service redirects to. If the user lands on a rendered login
+		// page, it goes through the print-to-PDF path and they'll see
+		// an obviously-wrong "Sign in" PDF — the cue to run `membot login`.
+		// Specific downloaders own auth-redirect detection for the services
+		// they understand.
+		if (!response.ok() && response.status() !== 304) {
+			throw new HelpfulError({
+				kind: "network_error",
+				message: `HTTP ${response.status()} ${response.statusText()}: ${url.toString()}`,
+				hint: "Open the URL in your browser to verify it exists. For auth-gated content, run `membot login` first.",
+			});
+		}
+		const headers = response.headers();
+		const contentType =
+			(headers["content-type"] ?? "application/octet-stream").split(";")[0]?.trim() ?? "application/octet-stream";
+
+		if (contentType === "text/html" || contentType === "application/xhtml+xml") {
+			const page = await ctx.pool.newPage();
+			try {
+				ctx.onProgress?.("rendering page");
+				await page.goto(url.toString(), { waitUntil: "networkidle", timeout: 45_000 });
+				ctx.onProgress?.("printing to pdf");
+				const pdfBuf = await page.pdf({ format: "A4", printBackground: true, preferCSSPageSize: false });
+				const bytes = new Uint8Array(pdfBuf);
+				return {
+					bytes,
+					sha256: sha256Hex(pdfBuf),
+					mimeType: "application/pdf",
+					downloader: "generic-web",
+					downloaderArgs: { rendered: true, source_content_type: contentType },
+					sourceUrl: url.toString(),
+				};
+			} finally {
+				await page.close().catch(() => {});
+			}
+		}
+
+		const body = Buffer.from(await response.body());
+		return {
+			bytes: new Uint8Array(body),
+			sha256: sha256Hex(body),
+			mimeType: contentType,
+			downloader: "generic-web",
+			downloaderArgs: { rendered: false, source_content_type: contentType },
+			sourceUrl: url.toString(),
+		};
+	},
+};