membot 0.6.0 → 0.7.0

package/README.md

@@ -136,6 +136,7 @@ Add `--watch` (and optional `--tick <sec>`) to also run the refresh daemon, whic
   membot config list                                            # show every value (secrets masked)
   membot config set llm.anthropic_api_key sk-ant-...            # enable LLM-fallback paths
   membot config set chunker.target_chars 800                    # tweak any nested value
+  membot config set converters.max_inline_image_captions 50     # raise per-doc cap on vision captions for embedded images
   membot config get llm.anthropic_api_key --show-secrets        # reveal the masked key
   membot config unset chunker.target_chars                      # back to schema default
   membot config path                                            # print the absolute config path

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.6.0",
+	"version": "0.7.0",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {

package/scripts/build-test-docx.ts

@@ -0,0 +1,84 @@
+#!/usr/bin/env bun
+/**
+ * One-shot generator for `test/fixtures/sample-with-image.docx`. Run this
+ * (`bun scripts/build-test-docx.ts`) when the fixture is missing or when
+ * the embedded test image needs to change. The DOCX itself is committed
+ * to the repo so test runs don't depend on jszip-as-transitive-dep.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+// jszip ships transitively via mammoth; this script is run by hand, not in tests.
+import JSZip from "../node_modules/jszip/lib/index.js";
+
+const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+
+const documentXml = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+            xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
+            xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"
+            xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"
+            xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture">
+  <w:body>
+    <w:p><w:r><w:t>Lead paragraph before the diagram.</w:t></w:r></w:p>
+    <w:p><w:r><w:drawing>
+      <wp:inline>
+        <wp:extent cx="635" cy="635"/>
+        <wp:docPr id="1" name="Picture 1" descr="architecture diagram"/>
+        <a:graphic>
+          <a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/picture">
+            <pic:pic>
+              <pic:nvPicPr>
+                <pic:cNvPr id="1" name="img.png" descr="architecture diagram"/>
+                <pic:cNvPicPr/>
+              </pic:nvPicPr>
+              <pic:blipFill>
+                <a:blip r:embed="rId1"/>
+                <a:stretch><a:fillRect/></a:stretch>
+              </pic:blipFill>
+              <pic:spPr>
+                <a:xfrm><a:off x="0" y="0"/><a:ext cx="635" cy="635"/></a:xfrm>
+                <a:prstGeom prst="rect"><a:avLst/></a:prstGeom>
+              </pic:spPr>
+            </pic:pic>
+          </a:graphicData>
+        </a:graphic>
+      </wp:inline>
+    </w:drawing></w:r></w:p>
+    <w:p><w:r><w:t>Trailing paragraph after the diagram.</w:t></w:r></w:p>
+  </w:body>
+</w:document>`;
+
+const documentRels = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/image1.png"/>
+</Relationships>`;
+
+const rootRels = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+</Relationships>`;
+
+const contentTypes = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+  <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+  <Default Extension="xml" ContentType="application/xml"/>
+  <Default Extension="png" ContentType="image/png"/>
+  <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+</Types>`;
+
+async function main(): Promise<void> {
+	const zip = new JSZip();
+	zip.file("[Content_Types].xml", contentTypes);
+	zip.file("_rels/.rels", rootRels);
+	zip.file("word/document.xml", documentXml);
+	zip.file("word/_rels/document.xml.rels", documentRels);
+	zip.file("word/media/image1.png", Buffer.from(TINY_PNG_BASE64, "base64"));
+
+	const buffer = await zip.generateAsync({ type: "nodebuffer" });
+	const out = "test/fixtures/sample-with-image.docx";
+	mkdirSync(dirname(out), { recursive: true });
+	writeFileSync(out, buffer);
+	console.log(`wrote ${out} (${buffer.byteLength} bytes)`);
+}
+
+await main();

package/src/config/schemas.ts

@@ -7,6 +7,10 @@ export const ChunkerConfigSchema = z.object({
 	max_chars: z.number().int().positive().default(DEFAULTS.CHUNKER_MAX_CHARS),
 });
 
+export const ConvertersConfigSchema = z.object({
+	max_inline_image_captions: z.number().int().nonnegative().default(DEFAULTS.MAX_INLINE_IMAGE_CAPTIONS),
+});
+
 export const LlmConfigSchema = z.object({
 	anthropic_api_key: z.string().meta({ secret: true }).default(""),
 	converter_model: z.string().default(DEFAULTS.CONVERTER_MODEL),
@@ -43,6 +47,7 @@ export const MembotConfigSchema = z.object({
 	embedding_model: z.string().default(EMBEDDING_MODEL),
 	embedding_dimension: z.number().int().positive().default(EMBEDDING_DIMENSION),
 	chunker: ChunkerConfigSchema.default(() => ChunkerConfigSchema.parse({})),
+	converters: ConvertersConfigSchema.default(() => ConvertersConfigSchema.parse({})),
 	llm: LlmConfigSchema.default(() => LlmConfigSchema.parse({})),
 	downloaders: DownloadersConfigSchema.default(() => DownloadersConfigSchema.parse({})),
 	daemon: DaemonConfigSchema.default(() => DaemonConfigSchema.parse({})),
@@ -52,6 +57,7 @@ export const MembotConfigSchema = z.object({
 
 export type MembotConfig = z.infer<typeof MembotConfigSchema>;
 export type ChunkerConfig = z.infer<typeof ChunkerConfigSchema>;
+export type ConvertersConfig = z.infer<typeof ConvertersConfigSchema>;
 export type LlmConfig = z.infer<typeof LlmConfigSchema>;
 export type DownloadersConfig = z.infer<typeof DownloadersConfigSchema>;
 export type LinearDownloaderConfig = z.infer<typeof LinearDownloaderConfigSchema>;

package/src/constants.ts

@@ -40,6 +40,13 @@ export const DEFAULTS = {
 	VISION_MODEL: "claude-haiku-4-5-20251001",
 	UPDATE_CHECK_INTERVAL_MS: 24 * 60 * 60 * 1000,
 	UPDATE_CHECK_TIMEOUT_MS: 5_000,
+	/**
+	 * Per-document cap on Claude vision caption calls when expanding inline
+	 * images during DOCX/HTML conversion. Beyond this, images get a small
+	 * deterministic placeholder so a slide-deck-shaped doc with hundreds of
+	 * embedded images doesn't fan out into hundreds of vision requests.
+	 */
+	MAX_INLINE_IMAGE_CAPTIONS: 20,
 } as const;
 
 export const FILES = {

package/src/ingest/converter/docx.ts

@@ -1,15 +1,57 @@
 import mammoth from "mammoth";
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { type CapturedImage, inlineImageCaptions, MEMBOT_IMG_PREFIX } from "./images-inline.ts";
 
 const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
 
+/**
+ * Mammoth's image element wears an `altText` field that isn't reflected in
+ * the published `.d.ts`. We declare the bits we actually touch so the rest
+ * of the module can stay strict-typed.
+ */
+interface MammothImage {
+	contentType: string;
+	altText?: string;
+	readAsBuffer: () => Promise<Buffer>;
+}
+
 /**
  * Convert a DOCX file to markdown. Mammoth gives us HTML; we then run that
- * through turndown to get clean markdown. Any conversion warnings are
- * silently dropped — they're typically about styles we don't preserve.
+ * through turndown to get clean markdown. Embedded images (which mammoth
+ * would otherwise inline as 5MB base64 `data:` URIs) are intercepted and
+ * replaced with `membot-img://<id>` placeholders, then expanded into Claude
+ * vision captions by `inlineImageCaptions`. Conversion warnings from
+ * mammoth are silently dropped — they're typically about styles we don't
+ * preserve.
  */
-export async function convertDocx(bytes: Uint8Array): Promise<string> {
+export async function convertDocx(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const buf = Buffer.from(bytes);
-	const result = await mammoth.convertToHtml({ buffer: buf });
-	return turndown.turndown(result.value).trim();
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+
+	const result = await mammoth.convertToHtml(
+		{ buffer: buf },
+		{
+			convertImage: mammoth.images.imgElement(async (image) => {
+				const img = image as unknown as MammothImage;
+				const id = `img-${counter++}`;
+				try {
+					const buffer = await img.readAsBuffer();
+					images.set(id, {
+						bytes: new Uint8Array(buffer),
+						mimeType: img.contentType,
+						altText: img.altText,
+					});
+				} catch {
+					// If we can't read the image bytes, still emit the placeholder so
+					// turndown doesn't fall back to a giant inline data URI.
+				}
+				return { src: `${MEMBOT_IMG_PREFIX}${id}` };
+			}),
+		},
+	);
+
+	const md = turndown.turndown(result.value).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/html.ts

@@ -1,4 +1,6 @@
 import TurndownService from "turndown";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { extractDataUriImages, inlineImageCaptions } from "./images-inline.ts";
 
 const turndown = new TurndownService({
 	headingStyle: "atx",
@@ -8,13 +10,18 @@ const turndown = new TurndownService({
 
 /**
  * Convert HTML bytes to markdown using turndown. Strips script/style blocks
- * before conversion so they don't leak into the chunker.
+ * before conversion so they don't leak into the chunker. Inline data-URI
+ * images are extracted into their bytes and replaced with vision captions
+ * via `inlineImageCaptions`; external `<img src="https://…">` references
+ * are left for turndown to render normally.
  */
-export function convertHtml(bytes: Uint8Array): string {
+export async function convertHtml(bytes: Uint8Array, llm: LlmConfig, converters: ConvertersConfig): Promise<string> {
 	const html = new TextDecoder("utf-8").decode(bytes);
 	const cleaned = html
 		.replace(/<script[\s\S]*?<\/script>/gi, "")
 		.replace(/<style[\s\S]*?<\/style>/gi, "")
 		.replace(/<noscript[\s\S]*?<\/noscript>/gi, "");
-	return turndown.turndown(cleaned).trim();
+	const { html: rewritten, images } = extractDataUriImages(cleaned);
+	const md = turndown.turndown(rewritten).trim();
+	return inlineImageCaptions(md, images, llm, converters);
 }

package/src/ingest/converter/image.ts

@@ -12,6 +12,13 @@ Output the caption only, no preamble.`;
 
 const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
 
+/** Anthropic vision rejects images > 5MB; stay under that with margin. */
+const VISION_MAX_BYTES = 4 * 1024 * 1024;
+/** Tesseract is roughly linear in pixel count; bail past this byte size to avoid pathological hangs. */
+const OCR_MAX_BYTES = 8 * 1024 * 1024;
+/** Hard wall-clock for either subtask so a stuck network call never freezes ingest. */
+const SUBTASK_TIMEOUT_MS = 60_000;
+
 /**
  * Build the markdown surrogate for an image: an LLM-generated caption
  * (when an API key is available) folded together with any text recovered
@@ -19,17 +26,47 @@ const VISION_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/web
  * when no API key is set.
  */
 export async function convertImage(bytes: Uint8Array, mimeType: string, llm: LlmConfig): Promise<string> {
-	const captionPromise = describeImage(bytes, mimeType, llm);
-	const ocrPromise = ocrImage(bytes);
+	const captionPromise =
+		bytes.byteLength <= VISION_MAX_BYTES
+			? withTimeout(describeImage(bytes, mimeType, llm), SUBTASK_TIMEOUT_MS, "vision")
+			: Promise.resolve("");
+	const ocrPromise =
+		bytes.byteLength <= OCR_MAX_BYTES ? withTimeout(ocrImage(bytes), SUBTASK_TIMEOUT_MS, "ocr") : Promise.resolve("");
 	const [caption, ocrText] = await Promise.all([captionPromise, ocrPromise]);
 
 	const sections: string[] = [];
 	if (caption) sections.push(caption);
 	if (ocrText) sections.push(`## Text detected via OCR\n\n${ocrText}`);
-	if (sections.length === 0) sections.push(`(image, ${mimeType}, no caption available)`);
+	if (sections.length === 0) {
+		const note =
+			bytes.byteLength > VISION_MAX_BYTES
+				? `(image, ${mimeType}, ${bytes.byteLength} bytes — exceeds vision size limit, no caption available)`
+				: `(image, ${mimeType}, no caption available)`;
+		sections.push(note);
+	}
 	return sections.join("\n\n");
 }
 
+/**
+ * Race a promise against a timer so a stuck network call (vision) or a
+ * pathological CPU-bound job (OCR on a multi-megapixel image) never freezes
+ * the whole conversion pipeline. Logs a warning when the timer wins.
+ */
+async function withTimeout<T extends string>(p: Promise<T>, ms: number, label: string): Promise<T | ""> {
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const timeout = new Promise<"">((resolve) => {
+		timer = setTimeout(() => {
+			logger.warn(`image: ${label} timed out after ${ms}ms`);
+			resolve("");
+		}, ms);
+	});
+	try {
+		return await Promise.race([p, timeout]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
 /**
  * Single-shot vision call asking Claude to caption an image. Returns the
  * caption text or an empty string when the API key is missing or the

package/src/ingest/converter/images-inline.ts

@@ -0,0 +1,132 @@
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
+import { logger } from "../../output/logger.ts";
+import { convertImage } from "./image.ts";
+
+/**
+ * Bytes captured from an embedded image during DOCX/HTML conversion. The
+ * image-inlining helpers run `convertImage` over each one to produce a
+ * markdown caption that gets spliced back into the document body in place
+ * of the original `<img>` reference.
+ */
+export interface CapturedImage {
+	bytes: Uint8Array;
+	mimeType: string;
+	altText?: string;
+}
+
+/** URI scheme used to mark images that the inliner should expand. */
+export const MEMBOT_IMG_PREFIX = "membot-img://";
+
+/**
+ * Match `![alt](membot-img://<id>)` markdown image references. The id may
+ * contain any non-whitespace, non-`)` character so we don't accidentally
+ * stop at characters mammoth/turndown might emit inside an id.
+ */
+const TOKEN_RE = /!\[([^\]]*)\]\(membot-img:\/\/([^)\s]+)\)/g;
+
+/**
+ * Extract data-URI images from raw HTML and rewrite each `<img src="data:…">`
+ * to `<img src="membot-img://<id>">`. The captured bytes flow through the
+ * shared `inlineImageCaptions` step so HTML and DOCX share one captioning
+ * code path. Non-data `<img>` references are left untouched.
+ */
+export function extractDataUriImages(html: string): { html: string; images: Map<string, CapturedImage> } {
+	const images = new Map<string, CapturedImage>();
+	let counter = 0;
+	const rewritten = html.replace(
+		/<img\b([^>]*?)\bsrc\s*=\s*(?:"data:([^";]+);base64,([^"]*)"|'data:([^';]+);base64,([^']*)')([^>]*)>/gi,
+		(
+			_match,
+			beforeSrc: string,
+			mimeDouble: string | undefined,
+			b64Double: string | undefined,
+			mimeSingle: string | undefined,
+			b64Single: string | undefined,
+			afterSrc: string,
+		) => {
+			const mimeType = (mimeDouble ?? mimeSingle ?? "image/png").trim();
+			const b64 = (b64Double ?? b64Single ?? "").replace(/\s+/g, "");
+			const id = `img-${counter++}`;
+			try {
+				const bytes = new Uint8Array(Buffer.from(b64, "base64"));
+				images.set(id, { bytes, mimeType });
+			} catch (err) {
+				logger.warn(
+					`images-inline: failed to decode embedded image (${err instanceof Error ? err.message : String(err)})`,
+				);
+				return `<img${beforeSrc} src=""${afterSrc}>`;
+			}
+			return `<img${beforeSrc} src="${MEMBOT_IMG_PREFIX}${id}"${afterSrc}>`;
+		},
+	);
+	return { html: rewritten, images };
+}
+
+/**
+ * Replace each `![alt](membot-img://<id>)` token in `markdown` with the
+ * caption produced by `convertImage`. Captures are processed in document
+ * order; once `max_inline_image_captions` (from `ConvertersConfig`) has been
+ * reached, the remaining tokens get a tiny deterministic placeholder rather
+ * than an LLM call so a doc full of embedded images doesn't fan out into
+ * hundreds of vision requests.
+ *
+ * No-ops on a markdown string with no `membot-img://` references; safe to
+ * call unconditionally from the converters.
+ */
+export async function inlineImageCaptions(
+	markdown: string,
+	images: Map<string, CapturedImage>,
+	llm: LlmConfig,
+	converters: ConvertersConfig,
+): Promise<string> {
+	if (images.size === 0) return markdown;
+
+	const captions = new Map<string, string>();
+	const overflow = new Set<string>();
+	let captioned = 0;
+
+	for (const match of markdown.matchAll(TOKEN_RE)) {
+		const alt = match[1] ?? "";
+		const id = match[2];
+		if (!id || captions.has(id) || overflow.has(id)) continue;
+		const img = images.get(id);
+		if (!img) continue;
+
+		if (captioned >= converters.max_inline_image_captions) {
+			overflow.add(id);
+			continue;
+		}
+		captioned++;
+		try {
+			const caption = await convertImage(img.bytes, img.mimeType, llm);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", caption));
+		} catch (err) {
+			logger.warn(`images-inline: caption failed for ${id} (${err instanceof Error ? err.message : String(err)})`);
+			captions.set(id, formatCaptionBlock(alt || img.altText || "", `(image, ${img.mimeType}, no caption available)`));
+		}
+	}
+
+	return markdown.replace(TOKEN_RE, (_match, alt: string, id: string) => {
+		const cached = captions.get(id);
+		if (cached) return cached;
+		const img = images.get(id);
+		if (!img) return formatCaptionBlock(alt, "(image, no caption available)");
+		return formatCaptionBlock(
+			alt || img.altText || "",
+			`(image, ${img.mimeType}, ${img.bytes.byteLength} bytes — caption skipped, exceeded max_inline_image_captions)`,
+		);
+	});
+}
+
+/**
+ * Render a captioned image as its own markdown paragraph block. Wrapping the
+ * caption in blank lines guarantees the deterministic chunker sees it as a
+ * paragraph boundary; an HTML comment with the alt text keeps the original
+ * positional cue without polluting search snippets.
+ */
+function formatCaptionBlock(alt: string, caption: string): string {
+	const trimmed = caption.trim();
+	const header = alt.trim() ? `<!-- image: ${alt.trim()} -->` : `<!-- image -->`;
+	const body = trimmed.length > 0 ? trimmed : "(image, no caption available)";
+	return `\n\n${header}\n\n${body}\n\n`;
+}

package/src/ingest/converter/index.ts

@@ -1,4 +1,4 @@
-import type { LlmConfig } from "../../config/schemas.ts";
+import type { ConvertersConfig, LlmConfig } from "../../config/schemas.ts";
 import { convertDocx } from "./docx.ts";
 import { convertHtml } from "./html.ts";
 import { convertImage } from "./image.ts";
@@ -44,6 +44,7 @@ export async function convert(
 	mimeType: string,
 	source: string,
 	llm: LlmConfig,
+	converters: ConvertersConfig,
 ): Promise<ConvertResult> {
 	const mt = mimeType.toLowerCase();
 
@@ -52,11 +53,11 @@ export async function convert(
 	}
 
 	if (HTML_MIMES.has(mt)) {
-		return { markdown: convertHtml(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertHtml(bytes, llm, converters), contentMimeType: "text/markdown" };
 	}
 
 	if (DOCX_MIMES.has(mt)) {
-		return { markdown: await convertDocx(bytes), contentMimeType: "text/markdown" };
+		return { markdown: await convertDocx(bytes, llm, converters), contentMimeType: "text/markdown" };
 	}
 
 	if (XLSX_MIMES.has(mt)) {

package/src/ingest/ingest.ts

@@ -383,7 +383,7 @@ async function pipelineForBytes(
 	});
 
 	onPhase?.("converting");
-	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm);
+	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm, ctx.config.converters);
 	const markdown = conversion.markdown;
 	const contentSha = sha256Hex(new TextEncoder().encode(markdown));
 

package/src/refresh/runner.ts

@@ -221,7 +221,7 @@ async function runPipelineForRefresh(
 	});
 
 	onPhase?.("converting");
-	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm);
+	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm, ctx.config.converters);
 	const markdown = conversion.markdown;
 	onPhase?.("describing");
 	const description = await describe(p.logicalPath, p.mime, markdown, ctx.config.llm);