membot 0.6.0 → 0.8.0

package/.claude/skills/membot.md

@@ -64,6 +64,7 @@ membot read <logical_path>                       # current markdown surrogate
 membot read <logical_path> --bytes               # original bytes (base64) — PDF/DOCX/image as ingested
 membot read <logical_path> --version <ts>        # historical snapshot
 membot info <logical_path>                       # metadata only (no content)
+membot stats [prefix]                            # whole-index summary; optional prefix scopes the aggregates
 membot versions <logical_path>                   # every version, newest first
 membot diff <logical_path> --a <ts> [--b <ts>]   # unified diff between versions
 ```
@@ -129,6 +130,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |
 | `membot info <path>`                  | Inspect metadata (source, downloader, refresh schedule, digests) without content |
+| `membot stats [prefix]`               | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes |
 | `membot versions <path>`              | List every version newest-first with version_id and change notes               |
 | `membot diff <path> --a <ts>`         | Unified diff between two versions                                              |
 | `membot mv <old> <new>`               | Rename a logical_path (history preserved)                                      |
@@ -161,4 +163,5 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Data lives in `~/.membot/index.duckdb` (override via `MEMBOT_HOME`).
 - Optional `ANTHROPIC_API_KEY` enables LLM fallback for messy/binary input. Without it, conversion degrades to deterministic native output.
+- `embedding.workers` (config key) caps the per-command embed-worker subprocess pool spawned at the top of `add` / `refresh` / `write`. Default `null` resolves to `cpus()-1`; set `1` to disable the pool.
 - Config file: `~/.membot/config.json` (see `membot --help` for the global flags).

package/.cursor/rules/membot.mdc

@@ -64,6 +64,7 @@ membot read <logical_path>                       # current markdown surrogate
 membot read <logical_path> --bytes               # original bytes (base64) — PDF/DOCX/image as ingested
 membot read <logical_path> --version <ts>        # historical snapshot
 membot info <logical_path>                       # metadata only (no content)
+membot stats [prefix]                            # whole-index summary; optional prefix scopes the aggregates
 membot versions <logical_path>                   # every version, newest first
 membot diff <logical_path> --a <ts> [--b <ts>]   # unified diff between versions
 ```
@@ -129,6 +130,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |
 | `membot info <path>`                  | Inspect metadata (source, downloader, refresh schedule, digests) without content |
+| `membot stats [prefix]`               | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes |
 | `membot versions <path>`              | List every version newest-first with version_id and change notes               |
 | `membot diff <path> --a <ts>`         | Unified diff between two versions                                              |
 | `membot mv <old> <new>`               | Rename a logical_path (history preserved)                                      |
@@ -161,4 +163,5 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Data lives in `~/.membot/index.duckdb` (override via `MEMBOT_HOME`).
 - Optional `ANTHROPIC_API_KEY` enables LLM fallback for messy/binary input. Without it, conversion degrades to deterministic native output.
+- `embedding.workers` (config key) caps the per-command embed-worker subprocess pool spawned at the top of `add` / `refresh` / `write`. Default `null` resolves to `cpus()-1`; set `1` to disable the pool.
 - Config file: `~/.membot/config.json` (see `membot --help` for the global flags).

package/README.md

@@ -83,6 +83,7 @@ The skill files describe the discover → ingest → search → read → write w
 | `membot read <path>`            | Read the markdown surrogate (or `--bytes` for original bytes, base64)             |
 | `membot search <query>`         | Hybrid search (semantic + BM25); `--include-history` searches older versions      |
 | `membot info <path>`            | Inspect metadata (source, fetcher, schedule, digests) without content             |
+| `membot stats [prefix]`         | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes the aggregates |
 | `membot versions <path>`        | List every version newest-first                                                   |
 | `membot diff <path> <a> [b]`    | Unified diff between two versions                                                 |
 | `membot write <path>`           | Write inline agent-authored markdown as a new version                             |
@@ -136,11 +137,15 @@ Add `--watch` (and optional `--tick <sec>`) to also run the refresh daemon, whic
   membot config list                                            # show every value (secrets masked)
   membot config set llm.anthropic_api_key sk-ant-...            # enable LLM-fallback paths
   membot config set chunker.target_chars 800                    # tweak any nested value
+  membot config set embedding.workers 4                         # cap parallel embed workers
+  membot config set converters.max_inline_image_captions 50     # raise per-doc cap on vision captions for embedded images
   membot config get llm.anthropic_api_key --show-secrets        # reveal the masked key
   membot config unset chunker.target_chars                      # back to schema default
   membot config path                                            # print the absolute config path
   ```
 
+  **Parallel embedding:** `embedding.workers` (default `null` → `cpus()-1`) controls how many subprocess workers fan out the WASM embedding work. The pool is **per-command** — spawned at the start of `add` / `refresh` / `write` and killed before the command returns, so membot doesn't keep idle workers around between invocations. Each worker loads its own ~50MB copy of the model, so on RAM-constrained machines drop it to a small fixed number (e.g. `4`); set `1` to disable the pool entirely and embed inline.
+
   Values are written with file mode `0600`. `ANTHROPIC_API_KEY` set in the environment still wins on read, so existing env-var setups keep working.
 - **Browser session:** `~/.membot/auth/browser-profile/` (Playwright persistent profile — cookies, localStorage, IndexedDB). Captured by `membot login`; cookie-based downloaders (Google) reuse it on every fetch. Delete the directory to force a fresh login.
 - **API keys:** stored under `downloaders.<service>.api_key` in `~/.membot/config.json`. Read by API-based downloaders (GitHub, Linear).

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.6.0",
+	"version": "0.8.0",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {

package/scripts/build-test-docx.ts

@@ -0,0 +1,84 @@
+#!/usr/bin/env bun
+/**
+ * One-shot generator for `test/fixtures/sample-with-image.docx`. Run this
+ * (`bun scripts/build-test-docx.ts`) when the fixture is missing or when
+ * the embedded test image needs to change. The DOCX itself is committed
+ * to the repo so test runs don't depend on jszip-as-transitive-dep.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+// jszip ships transitively via mammoth; this script is run by hand, not in tests.
+import JSZip from "../node_modules/jszip/lib/index.js";
+
+const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+
+const documentXml = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+            xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
+            xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"
+            xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"
+            xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture">
+  <w:body>
+    <w:p><w:r><w:t>Lead paragraph before the diagram.</w:t></w:r></w:p>
+    <w:p><w:r><w:drawing>
+      <wp:inline>
+        <wp:extent cx="635" cy="635"/>
+        <wp:docPr id="1" name="Picture 1" descr="architecture diagram"/>
+        <a:graphic>
+          <a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/picture">
+            <pic:pic>
+              <pic:nvPicPr>
+                <pic:cNvPr id="1" name="img.png" descr="architecture diagram"/>
+                <pic:cNvPicPr/>
+              </pic:nvPicPr>
+              <pic:blipFill>
+                <a:blip r:embed="rId1"/>
+                <a:stretch><a:fillRect/></a:stretch>
+              </pic:blipFill>
+              <pic:spPr>
+                <a:xfrm><a:off x="0" y="0"/><a:ext cx="635" cy="635"/></a:xfrm>
+                <a:prstGeom prst="rect"><a:avLst/></a:prstGeom>
+              </pic:spPr>
+            </pic:pic>
+          </a:graphicData>
+        </a:graphic>
+      </wp:inline>
+    </w:drawing></w:r></w:p>
+    <w:p><w:r><w:t>Trailing paragraph after the diagram.</w:t></w:r></w:p>
+  </w:body>
+</w:document>`;
+
+const documentRels = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/image1.png"/>
+</Relationships>`;
+
+const rootRels = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+</Relationships>`;
+
+const contentTypes = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+  <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+  <Default Extension="xml" ContentType="application/xml"/>
+  <Default Extension="png" ContentType="image/png"/>
+  <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+</Types>`;
+
+async function main(): Promise<void> {
+	const zip = new JSZip();
+	zip.file("[Content_Types].xml", contentTypes);
+	zip.file("_rels/.rels", rootRels);
+	zip.file("word/document.xml", documentXml);
+	zip.file("word/_rels/document.xml.rels", documentRels);
+	zip.file("word/media/image1.png", Buffer.from(TINY_PNG_BASE64, "base64"));
+
+	const buffer = await zip.generateAsync({ type: "nodebuffer" });
+	const out = "test/fixtures/sample-with-image.docx";
+	mkdirSync(dirname(out), { recursive: true });
+	writeFileSync(out, buffer);
+	console.log(`wrote ${out} (${buffer.byteLength} bytes)`);
+}
+
+await main();

package/src/cli.ts

@@ -10,12 +10,23 @@ import { registerReindexCommand } from "./commands/reindex.ts";
 import { registerServeCommand } from "./commands/serve.ts";
 import { registerSkillCommand } from "./commands/skill.ts";
 import { registerUpgradeCommand } from "./commands/upgrade.ts";
+import { EMBED_WORKER_SENTINEL } from "./constants.ts";
 import type { BuildContextOptions } from "./context.ts";
+import { runEmbedWorker } from "./ingest/embed-worker.ts";
 import { mountAsCommanderCommand } from "./mount/commander.ts";
 import { OPERATIONS } from "./operations/index.ts";
 import { logger } from "./output/logger.ts";
 import { maybeCheckForUpdate } from "./update/background.ts";
 
+// Hidden worker mode: the EmbedderPool re-execs this binary with the sentinel
+// as argv[2] (or argv[1] when `bun run src/cli.ts <sentinel>` is invoked
+// directly during tests). We bypass commander entirely and run the worker
+// stdin/stdout protocol loop instead.
+if (process.argv.includes(EMBED_WORKER_SENTINEL)) {
+	await runEmbedWorker();
+	process.exit(0);
+}
+
 program
 	.name("membot")
 	.description("Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.")

package/src/config/schemas.ts

@@ -7,6 +7,10 @@ export const ChunkerConfigSchema = z.object({
 	max_chars: z.number().int().positive().default(DEFAULTS.CHUNKER_MAX_CHARS),
 });
 
+export const ConvertersConfigSchema = z.object({
+	max_inline_image_captions: z.number().int().nonnegative().default(DEFAULTS.MAX_INLINE_IMAGE_CAPTIONS),
+});
+
 export const LlmConfigSchema = z.object({
 	anthropic_api_key: z.string().meta({ secret: true }).default(""),
 	converter_model: z.string().default(DEFAULTS.CONVERTER_MODEL),
@@ -19,6 +23,18 @@ export const DaemonConfigSchema = z.object({
 	tick_interval_sec: z.number().int().positive().default(DEFAULTS.DAEMON_TICK_SEC),
 });
 
+/**
+ * Embedding parallelism. `workers = null` (the default) resolves to
+ * `max(1, cpus()-1)` at context-build time so the pool grows with the host
+ * machine. Setting `workers = 1` disables the subprocess pool entirely
+ * and runs embedding inline in the parent (the original single-thread
+ * behaviour). Each worker loads its own copy of the WASM model
+ * (~50MB resident), so cap this on RAM-constrained machines.
+ */
+export const EmbeddingConfigSchema = z.object({
+	workers: z.number().int().min(1).nullable().default(null),
+});
+
 export const LinearDownloaderConfigSchema = z.object({
 	api_key: z.string().meta({ secret: true }).default(""),
 });
@@ -43,6 +59,8 @@ export const MembotConfigSchema = z.object({
 	embedding_model: z.string().default(EMBEDDING_MODEL),
 	embedding_dimension: z.number().int().positive().default(EMBEDDING_DIMENSION),
 	chunker: ChunkerConfigSchema.default(() => ChunkerConfigSchema.parse({})),
+	embedding: EmbeddingConfigSchema.default(() => EmbeddingConfigSchema.parse({})),
+	converters: ConvertersConfigSchema.default(() => ConvertersConfigSchema.parse({})),
 	llm: LlmConfigSchema.default(() => LlmConfigSchema.parse({})),
 	downloaders: DownloadersConfigSchema.default(() => DownloadersConfigSchema.parse({})),
 	daemon: DaemonConfigSchema.default(() => DaemonConfigSchema.parse({})),
@@ -52,6 +70,8 @@ export const MembotConfigSchema = z.object({
 
 export type MembotConfig = z.infer<typeof MembotConfigSchema>;
 export type ChunkerConfig = z.infer<typeof ChunkerConfigSchema>;
+export type EmbeddingConfig = z.infer<typeof EmbeddingConfigSchema>;
+export type ConvertersConfig = z.infer<typeof ConvertersConfigSchema>;
 export type LlmConfig = z.infer<typeof LlmConfigSchema>;
 export type DownloadersConfig = z.infer<typeof DownloadersConfigSchema>;
 export type LinearDownloaderConfig = z.infer<typeof LinearDownloaderConfigSchema>;

package/src/constants.ts

@@ -28,6 +28,14 @@ export const EMBEDDING_DIMENSION = 384;
  */
 export const EMBEDDING_BATCH_SIZE = 16;
 
+/**
+ * Hidden first-arg sentinel that re-execs the membot binary as an embed
+ * worker. The pool spawns `process.execPath <sentinel>` so the same compiled
+ * binary serves both the user-facing CLI and the worker subprocess; cli.ts
+ * checks this argv slot before commander sees it.
+ */
+export const EMBED_WORKER_SENTINEL = "__embed_worker";
+
 export const DEFAULTS = {
 	CHUNKER_MODE: "deterministic" as const,
 	CHUNKER_TARGET_CHARS: 4_000,
@@ -40,6 +48,13 @@ export const DEFAULTS = {
 	VISION_MODEL: "claude-haiku-4-5-20251001",
 	UPDATE_CHECK_INTERVAL_MS: 24 * 60 * 60 * 1000,
 	UPDATE_CHECK_TIMEOUT_MS: 5_000,
+	/**
+	 * Per-document cap on Claude vision caption calls when expanding inline
+	 * images during DOCX/HTML conversion. Beyond this, images get a small
+	 * deterministic placeholder so a slide-deck-shaped doc with hundreds of
+	 * embedded images doesn't fan out into hundreds of vision requests.
+	 */
+	MAX_INLINE_IMAGE_CAPTIONS: 20,
 } as const;
 
 export const FILES = {

package/src/context.ts

@@ -1,3 +1,4 @@
+import { cpus } from "node:os";
 import { join } from "node:path";
 import { loadConfig } from "./config/loader.ts";
 import type { MembotConfig } from "./config/schemas.ts";
@@ -25,11 +26,34 @@ export interface BuildContextOptions {
 	noInteractive?: boolean;
 }
 
+/**
+ * Resolve `config.embedding.workers` to a concrete worker count. Precedence:
+ *   1. An explicit numeric value in the config wins (user opt-in).
+ *   2. `MEMBOT_EMBEDDING_WORKERS` env var, if set to a positive integer.
+ *      The test harness sets this to `1` so unit tests doing tiny writes
+ *      don't pay the per-pool subprocess-spawn cost on slow CI runners.
+ *   3. Otherwise `null`/missing → `max(1, cpus()-1)`. The minus-one leaves
+ *      a core for the parent process (DB writes, IO, the spinner).
+ */
+export function resolveEmbeddingWorkers(configured: number | null | undefined): number {
+	if (typeof configured === "number" && configured >= 1) return configured;
+	const envOverride = process.env.MEMBOT_EMBEDDING_WORKERS;
+	if (envOverride) {
+		const n = Number(envOverride);
+		if (Number.isFinite(n) && n >= 1) return Math.floor(n);
+	}
+	return Math.max(1, cpus().length - 1);
+}
+
 /**
  * Build the AppContext used by every operation handler. Initializes:
  *  - output mode (TTY/JSON/color detection — frozen for the rest of the run)
  *  - config (~/.membot/config.json with env overrides)
  *  - DuckDB connection (~/.membot/index.duckdb), running migrations on first open
+ *
+ * The embedder worker pool is NOT created here — it's per-command,
+ * spawned by `withEmbedderPool()` at the top of bulk-embedding handlers
+ * (`add`, `refresh`, `write`) and disposed before they return.
  */
 export async function buildContext(options: BuildContextOptions = {}): Promise<AppContext> {
 	setMode(detectMode({ json: options.json, verbose: options.verbose, noColor: options.noColor }));

package/src/ingest/converter/docx.ts

@@ -1,15 +1,57 @@
 import mammoth from "mammoth";
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { type CapturedImage, inlineImageCaptions, MEMBOT_IMG_PREFIX } from "./images-inline.ts";
 
 const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
 
+/**
+ * Mammoth's image element wears an `altText` field that isn't reflected in
+ * the published `.d.ts`. We declare the bits we actually touch so the rest
+ * of the module can stay strict-typed.
+ */
+interface MammothImage {
+	contentType: string;
+	altText?: string;
+	readAsBuffer: () => Promise<Buffer>;
+}
+
 /**
  * Convert a DOCX file to markdown. Mammoth gives us HTML; we then run that
- * through turndown to get clean markdown. Any conversion warnings are
- * silently dropped — they're typically about styles we don't preserve.
+ * through turndown to get clean markdown. Embedded images (which mammoth
+ * would otherwise inline as 5MB base64 `data:` URIs) are intercepted and
+ * replaced with `membot-img://<id>` placeholders, then expanded into Claude
+ * vision captions by `inlineImageCaptions`. Conversion warnings from
+ * mammoth are silently dropped — they're typically about styles we don't
+ * preserve.
  */
-export async function convertDocx(bytes: Uint8Array): Promise<string> {
+export async function convertDocx(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const buf = Buffer.from(bytes);
-	const result = await mammoth.convertToHtml({ buffer: buf });
-	return turndown.turndown(result.value).trim();
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+
+	const result = await mammoth.convertToHtml(
+		{ buffer: buf },
+		{
+			convertImage: mammoth.images.imgElement(async (image) => {
+				const img = image as unknown as MammothImage;
+				const id = `img-${counter++}`;
+				try {
+					const buffer = await img.readAsBuffer();
+					images.set(id, {
+						bytes: new Uint8Array(buffer),
+						mimeType: img.contentType,
+						altText: img.altText,
+					});
+				} catch {
+					// If we can't read the image bytes, still emit the placeholder so
+					// turndown doesn't fall back to a giant inline data URI.
+				}
+				return { src: `${MEMBOT_IMG_PREFIX}${id}` };
+			}),
+		},
+	);
+
+	const md = turndown.turndown(result.value).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/html.ts

@@ -1,4 +1,6 @@
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { extractDataUriImages, inlineImageCaptions } from "./images-inline.ts";
 
 const turndown = new TurndownService({
 	headingStyle: "atx",
@@ -8,13 +10,18 @@ const turndown = new TurndownService({
 
 /**
  * Convert HTML bytes to markdown using turndown. Strips script/style blocks
- * before conversion so they don't leak into the chunker.
+ * before conversion so they don't leak into the chunker. Inline data-URI
+ * images are extracted into their bytes and replaced with vision captions
+ * via `inlineImageCaptions`; external `<img src="https://…">` references
+ * are left for turndown to render normally.
  */
-export function convertHtml(bytes: Uint8Array): string {
+export async function convertHtml(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const html = new TextDecoder("utf-8").decode(bytes);
 	const cleaned = html
 		.replace(/<script[\s\S]*?<\/script>/gi, "")
 		.replace(/<style[\s\S]*?<\/style>/gi, "")
 		.replace(/<noscript[\s\S]*?<\/noscript>/gi, "");
-	return turndown.turndown(cleaned).trim();
+	const { html: rewritten, images } = extractDataUriImages(cleaned);
+	const md = turndown.turndown(rewritten).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/image.ts

@@ -12,6 +12,13 @@ Output the caption only, no preamble.`;
 
 const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
 
+/** Anthropic vision rejects images > 5MB; stay under that with margin. */
+const VISION_MAX_BYTES = 4 * 1024 * 1024;
+/** Tesseract is roughly linear in pixel count; bail past this byte size to avoid pathological hangs. */
+const OCR_MAX_BYTES = 8 * 1024 * 1024;
+/** Hard wall-clock for either subtask so a stuck network call never freezes ingest. */
+const SUBTASK_TIMEOUT_MS = 60_000;
+
 /**
  * Build the markdown surrogate for an image: an LLM-generated caption
  * (when an API key is available) folded together with any text recovered
@@ -19,17 +26,47 @@ const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/web
  * when no API key is set.
  */
 export async function convertImage(bytes: Uint8Array, mimeType: string, llm: LlmConfig): Promise<string> {
-	const captionPromise = describeImage(bytes, mimeType, llm);
-	const ocrPromise = ocrImage(bytes);
+	const captionPromise =
+		bytes.byteLength <= VISION_MAX_BYTES
+			? withTimeout(describeImage(bytes, mimeType, llm), SUBTASK_TIMEOUT_MS, "vision")
+			: Promise.resolve("");
+	const ocrPromise =
+		bytes.byteLength <= OCR_MAX_BYTES ? withTimeout(ocrImage(bytes), SUBTASK_TIMEOUT_MS, "ocr") : Promise.resolve("");
 	const [caption, ocrText] = await Promise.all([captionPromise, ocrPromise]);
 
 	const sections: string[] = [];
 	if (caption) sections.push(caption);
 	if (ocrText) sections.push(`## Text detected via OCR\n\n${ocrText}`);
-	if (sections.length === 0) sections.push(`(image, ${mimeType}, no caption available)`);
+	if (sections.length === 0) {
+		const note =
+			bytes.byteLength > VISION_MAX_BYTES
+				? `(image, ${mimeType}, ${bytes.byteLength} bytes — exceeds vision size limit, no caption available)`
+				: `(image, ${mimeType}, no caption available)`;
+		sections.push(note);
+	}
 	return sections.join("\n\n");
 }
 
+/**
+ * Race a promise against a timer so a stuck network call (vision) or a
+ * pathological CPU-bound job (OCR on a multi-megapixel image) never freezes
+ * the whole conversion pipeline. Logs a warning when the timer wins.
+ */
+async function withTimeout<T extends string>(p: Promise<T>, ms: number, label: string): Promise<T | ""> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const timeout = new Promise<"">((resolve) => {
+		timer = setTimeout(() => {
+			logger.warn(`image: ${label} timed out after ${ms}ms`);
+			resolve("");
+		}, ms);
+	});
+	try {
+		return await Promise.race([p, timeout]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
 /**
  * Single-shot vision call asking Claude to caption an image. Returns the
  * caption text or an empty string when the API key is missing or the

package/src/ingest/converter/images-inline.ts

@@ -0,0 +1,132 @@
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { logger } from "../../output/logger.ts";
+import { convertImage } from "./image.ts";
+
+/**
+ * Bytes captured from an embedded image during DOCX/HTML conversion. The
+ * image-inlining helpers run `convertImage` over each one to produce a
+ * markdown caption that gets spliced back into the document body in place
+ * of the original `<img>` reference.
+ */
+export interface CapturedImage {
+	bytes: Uint8Array;
+	mimeType: string;
+	altText?: string;
+}
+
+/** URI scheme used to mark images that the inliner should expand. */
+export const MEMBOT_IMG_PREFIX = "membot-img://";
+
+/**
+ * Match `![alt](membot-img://<id>)` markdown image references. The id may
+ * contain any non-whitespace, non-`)` character so we don't accidentally
+ * stop at characters mammoth/turndown might emit inside an id.
+ */
+const TOKEN_RE = /!\[([^\]]*)\]\(membot-img:\/\/([^)\s]+)\)/g;
+
+/**
+ * Extract data-URI images from raw HTML and rewrite each `<img src="data:…">`
+ * to `<img src="membot-img://<id>">`. The captured bytes flow through the
+ * shared `inlineImageCaptions` step so HTML and DOCX share one captioning
+ * code path. Non-data `<img>` references are left untouched.
+ */
+export function extractDataUriImages(html: string): { html: string; images: Map<string, CapturedImage> } {
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+	const rewritten = html.replace(
+		/<img\b([^>]*?)\bsrc\s*=\s*(?:"data:([^";]+);base64,([^"]*)"|'data:([^';]+);base64,([^']*)')([^>]*)>/gi,
+		(
+			_match,
+			beforeSrc: string,
+			mimeDouble: string | undefined,
+			b64Double: string | undefined,
+			mimeSingle: string | undefined,
+			b64Single: string | undefined,
+			afterSrc: string,
+		) => {
+			const mimeType = (mimeDouble ?? mimeSingle ?? "image/png").trim();
+			const b64 = (b64Double ?? b64Single ?? "").replace(/\s+/g, "");
+			const id = `img-${counter++}`;
+			try {
+				const bytes = new Uint8Array(Buffer.from(b64, "base64"));
+				images.set(id, { bytes, mimeType });
+			} catch (err) {
+				logger.warn(
+					`images-inline: failed to decode embedded image (${err instanceof Error ? err.message : String(err)})`,
+				);
+				return `<img${beforeSrc} src=""${afterSrc}>`;
+			}
+			return `<img${beforeSrc} src="${MEMBOT_IMG_PREFIX}${id}"${afterSrc}>`;
+		},
+	);
+	return { html: rewritten, images };
+}
+
+/**
+ * Replace each `![alt](membot-img://<id>)` token in `markdown` with the
+ * caption produced by `convertImage`. Captures are processed in document
+ * order; once `max_inline_image_captions` (from `ConvertersConfig`) has been
+ * reached, the remaining tokens get a tiny deterministic placeholder rather
+ * than an LLM call so a doc full of embedded images doesn't fan out into
+ * hundreds of vision requests.
+ *
+ * No-ops on a markdown string with no `membot-img://` references; safe to
+ * call unconditionally from the converters.
+ */
+export async function inlineImageCaptions(
+	markdown: string,
+	images: Map<string, CapturedImage>,
+	llm: LlmConfig,
+	converters: ConvertersConfig,
+): Promise<string> {
+	if (images.size === 0) return markdown;
+
+	const captions = new Map<string, string>();
+	const overflow = new Set<string>();
+	let captioned = 0;
+
+	for (const match of markdown.matchAll(TOKEN_RE)) {
+		const alt = match[1] ?? "";
+		const id = match[2];
+		if (!id || captions.has(id) || overflow.has(id)) continue;
+		const img = images.get(id);
+		if (!img) continue;
+
+		if (captioned >= converters.max_inline_image_captions) {
+			overflow.add(id);
+			continue;
+		}
+		captioned++;
+		try {
+			const caption = await convertImage(img.bytes, img.mimeType, llm);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", caption));
+		} catch (err) {
+			logger.warn(`images-inline: caption failed for ${id} (${err instanceof Error ? err.message : String(err)})`);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", `(image, ${img.mimeType}, no caption available)`));
+		}
+	}
+
+	return markdown.replace(TOKEN_RE, (_match, alt: string, id: string) => {
+		const cached = captions.get(id);
+		if (cached) return cached;
+		const img = images.get(id);
+		if (!img) return formatCaptionBlock(alt, "(image, no caption available)");
+		return formatCaptionBlock(
+			alt || img.altText || "",
+			`(image, ${img.mimeType}, ${img.bytes.byteLength} bytes — caption skipped, exceeded max_inline_image_captions)`,
+		);
+	});
+}
+
+/**
+ * Render a captioned image as its own markdown paragraph block. Wrapping the
+ * caption in blank lines guarantees the deterministic chunker sees it as a
+ * paragraph boundary; an HTML comment with the alt text keeps the original
+ * positional cue without polluting search snippets.
+ */
+function formatCaptionBlock(alt: string, caption: string): string {
+	const trimmed = caption.trim();
+	const header = alt.trim() ? `<!-- image: ${alt.trim()} -->` : `<!-- image -->`;
+	const body = trimmed.length > 0 ? trimmed : "(image, no caption available)";
+	return `\n\n${header}\n\n${body}\n\n`;
+}

package/src/ingest/converter/index.ts

@@ -1,4 +1,4 @@
-import type { LlmConfig } from "../../config/schemas.ts";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
 import { convertDocx } from "./docx.ts";
 import { convertHtml } from "./html.ts";
 import { convertImage } from "./image.ts";
@@ -44,6 +44,7 @@ export async function convert(
 	mimeType: string,
 	source: string,
 	llm: LlmConfig,
+	converters: ConvertersConfig,
 ): Promise<ConvertResult> {
 	const mt = mimeType.toLowerCase();
 
@@ -52,11 +53,11 @@ export async function convert(
 	}
 
 	if (HTML_MIMES.has(mt)) {
-		return { markdown: convertHtml(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertHtml(bytes, llm, converters), contentMimeType: "text/markdown" };
 	}
 
 	if (DOCX_MIMES.has(mt)) {
-		return { markdown: await convertDocx(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertDocx(bytes, llm, converters), contentMimeType: "text/markdown" };
 	}
 
 	if (XLSX_MIMES.has(mt)) {