membot 0.7.0 → 0.8.0

package/.claude/skills/membot.md

@@ -64,6 +64,7 @@ membot read <logical_path>                       # current markdown surrogate
 membot read <logical_path> --bytes               # original bytes (base64) — PDF/DOCX/image as ingested
 membot read <logical_path> --version <ts>        # historical snapshot
 membot info <logical_path>                       # metadata only (no content)
+membot stats [prefix]                            # whole-index summary; optional prefix scopes the aggregates
 membot versions <logical_path>                   # every version, newest first
 membot diff <logical_path> --a <ts> [--b <ts>]   # unified diff between versions
 ```
@@ -129,6 +130,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |
 | `membot info <path>`                  | Inspect metadata (source, downloader, refresh schedule, digests) without content |
+| `membot stats [prefix]`               | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes |
 | `membot versions <path>`              | List every version newest-first with version_id and change notes               |
 | `membot diff <path> --a <ts>`         | Unified diff between two versions                                              |
 | `membot mv <old> <new>`               | Rename a logical_path (history preserved)                                      |
@@ -161,4 +163,5 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Data lives in `~/.membot/index.duckdb` (override via `MEMBOT_HOME`).
 - Optional `ANTHROPIC_API_KEY` enables LLM fallback for messy/binary input. Without it, conversion degrades to deterministic native output.
+- `embedding.workers` (config key) caps the per-command embed-worker subprocess pool spawned at the top of `add` / `refresh` / `write`. Default `null` resolves to `cpus()-1`; set `1` to disable the pool.
 - Config file: `~/.membot/config.json` (see `membot --help` for the global flags).

package/.cursor/rules/membot.mdc

@@ -64,6 +64,7 @@ membot read <logical_path>                       # current markdown surrogate
 membot read <logical_path> --bytes               # original bytes (base64) — PDF/DOCX/image as ingested
 membot read <logical_path> --version <ts>        # historical snapshot
 membot info <logical_path>                       # metadata only (no content)
+membot stats [prefix]                            # whole-index summary; optional prefix scopes the aggregates
 membot versions <logical_path>                   # every version, newest first
 membot diff <logical_path> --a <ts> [--b <ts>]   # unified diff between versions
 ```
@@ -129,6 +130,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |
 | `membot info <path>`                  | Inspect metadata (source, downloader, refresh schedule, digests) without content |
+| `membot stats [prefix]`               | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes |
 | `membot versions <path>`              | List every version newest-first with version_id and change notes               |
 | `membot diff <path> --a <ts>`         | Unified diff between two versions                                              |
 | `membot mv <old> <new>`               | Rename a logical_path (history preserved)                                      |
@@ -161,4 +163,5 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Data lives in `~/.membot/index.duckdb` (override via `MEMBOT_HOME`).
 - Optional `ANTHROPIC_API_KEY` enables LLM fallback for messy/binary input. Without it, conversion degrades to deterministic native output.
+- `embedding.workers` (config key) caps the per-command embed-worker subprocess pool spawned at the top of `add` / `refresh` / `write`. Default `null` resolves to `cpus()-1`; set `1` to disable the pool.
 - Config file: `~/.membot/config.json` (see `membot --help` for the global flags).

package/README.md

@@ -83,6 +83,7 @@ The skill files describe the discover → ingest → search → read → write w
 | `membot read <path>`            | Read the markdown surrogate (or `--bytes` for original bytes, base64)             |
 | `membot search <query>`         | Hybrid search (semantic + BM25); `--include-history` searches older versions      |
 | `membot info <path>`            | Inspect metadata (source, fetcher, schedule, digests) without content             |
+| `membot stats [prefix]`         | Summarize the index (file/version/chunk/blob counts, on-disk size, refresh health, mime/source/downloader breakdowns); optional prefix scopes the aggregates |
 | `membot versions <path>`        | List every version newest-first                                                   |
 | `membot diff <path> <a> [b]`    | Unified diff between two versions                                                 |
 | `membot write <path>`           | Write inline agent-authored markdown as a new version                             |
@@ -136,12 +137,15 @@ Add `--watch` (and optional `--tick <sec>`) to also run the refresh daemon, whic
   membot config list                                            # show every value (secrets masked)
   membot config set llm.anthropic_api_key sk-ant-...            # enable LLM-fallback paths
   membot config set chunker.target_chars 800                    # tweak any nested value
+  membot config set embedding.workers 4                         # cap parallel embed workers
   membot config set converters.max_inline_image_captions 50     # raise per-doc cap on vision captions for embedded images
   membot config get llm.anthropic_api_key --show-secrets        # reveal the masked key
   membot config unset chunker.target_chars                      # back to schema default
   membot config path                                            # print the absolute config path
   ```
 
+  **Parallel embedding:** `embedding.workers` (default `null` → `cpus()-1`) controls how many subprocess workers fan out the WASM embedding work. The pool is **per-command** — spawned at the start of `add` / `refresh` / `write` and killed before the command returns, so membot doesn't keep idle workers around between invocations. Each worker loads its own ~50MB copy of the model, so on RAM-constrained machines drop it to a small fixed number (e.g. `4`); set `1` to disable the pool entirely and embed inline.
+
   Values are written with file mode `0600`. `ANTHROPIC_API_KEY` set in the environment still wins on read, so existing env-var setups keep working.
 - **Browser session:** `~/.membot/auth/browser-profile/` (Playwright persistent profile — cookies, localStorage, IndexedDB). Captured by `membot login`; cookie-based downloaders (Google) reuse it on every fetch. Delete the directory to force a fresh login.
 - **API keys:** stored under `downloaders.<service>.api_key` in `~/.membot/config.json`. Read by API-based downloaders (GitHub, Linear).

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.7.0",
+	"version": "0.8.0",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {

package/src/cli.ts

@@ -10,12 +10,23 @@ import { registerReindexCommand } from "./commands/reindex.ts";
 import { registerServeCommand } from "./commands/serve.ts";
 import { registerSkillCommand } from "./commands/skill.ts";
 import { registerUpgradeCommand } from "./commands/upgrade.ts";
+import { EMBED_WORKER_SENTINEL } from "./constants.ts";
 import type { BuildContextOptions } from "./context.ts";
+import { runEmbedWorker } from "./ingest/embed-worker.ts";
 import { mountAsCommanderCommand } from "./mount/commander.ts";
 import { OPERATIONS } from "./operations/index.ts";
 import { logger } from "./output/logger.ts";
 import { maybeCheckForUpdate } from "./update/background.ts";
 
+// Hidden worker mode: the EmbedderPool re-execs this binary with the sentinel
+// as argv[2] (or argv[1] when `bun run src/cli.ts <sentinel>` is invoked
+// directly during tests). We bypass commander entirely and run the worker
+// stdin/stdout protocol loop instead.
+if (process.argv.includes(EMBED_WORKER_SENTINEL)) {
+	await runEmbedWorker();
+	process.exit(0);
+}
+
 program
 	.name("membot")
 	.description("Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.")

package/src/config/schemas.ts

@@ -23,6 +23,18 @@ export const DaemonConfigSchema = z.object({
 	tick_interval_sec: z.number().int().positive().default(DEFAULTS.DAEMON_TICK_SEC),
 });
 
+/**
+ * Embedding parallelism. `workers = null` (the default) resolves to
+ * `max(1, cpus()-1)` at context-build time so the pool grows with the host
+ * machine. Setting `workers = 1` disables the subprocess pool entirely
+ * and runs embedding inline in the parent (the original single-thread
+ * behaviour). Each worker loads its own copy of the WASM model
+ * (~50MB resident), so cap this on RAM-constrained machines.
+ */
+export const EmbeddingConfigSchema = z.object({
+	workers: z.number().int().min(1).nullable().default(null),
+});
+
 export const LinearDownloaderConfigSchema = z.object({
 	api_key: z.string().meta({ secret: true }).default(""),
 });
@@ -47,6 +59,7 @@ export const MembotConfigSchema = z.object({
 	embedding_model: z.string().default(EMBEDDING_MODEL),
 	embedding_dimension: z.number().int().positive().default(EMBEDDING_DIMENSION),
 	chunker: ChunkerConfigSchema.default(() => ChunkerConfigSchema.parse({})),
+	embedding: EmbeddingConfigSchema.default(() => EmbeddingConfigSchema.parse({})),
 	converters: ConvertersConfigSchema.default(() => ConvertersConfigSchema.parse({})),
 	llm: LlmConfigSchema.default(() => LlmConfigSchema.parse({})),
 	downloaders: DownloadersConfigSchema.default(() => DownloadersConfigSchema.parse({})),
@@ -57,6 +70,7 @@ export const MembotConfigSchema = z.object({
 
 export type MembotConfig = z.infer<typeof MembotConfigSchema>;
 export type ChunkerConfig = z.infer<typeof ChunkerConfigSchema>;
+export type EmbeddingConfig = z.infer<typeof EmbeddingConfigSchema>;
 export type ConvertersConfig = z.infer<typeof ConvertersConfigSchema>;
 export type LlmConfig = z.infer<typeof LlmConfigSchema>;
 export type DownloadersConfig = z.infer<typeof DownloadersConfigSchema>;

package/src/constants.ts

@@ -28,6 +28,14 @@ export const EMBEDDING_DIMENSION = 384;
  */
 export const EMBEDDING_BATCH_SIZE = 16;
 
+/**
+ * Hidden first-arg sentinel that re-execs the membot binary as an embed
+ * worker. The pool spawns `process.execPath <sentinel>` so the same compiled
+ * binary serves both the user-facing CLI and the worker subprocess; cli.ts
+ * checks this argv slot before commander sees it.
+ */
+export const EMBED_WORKER_SENTINEL = "__embed_worker";
+
 export const DEFAULTS = {
 	CHUNKER_MODE: "deterministic" as const,
 	CHUNKER_TARGET_CHARS: 4_000,

package/src/context.ts

@@ -1,3 +1,4 @@
+import { cpus } from "node:os";
 import { join } from "node:path";
 import { loadConfig } from "./config/loader.ts";
 import type { MembotConfig } from "./config/schemas.ts";
@@ -25,11 +26,34 @@ export interface BuildContextOptions {
 	noInteractive?: boolean;
 }
 
+/**
+ * Resolve `config.embedding.workers` to a concrete worker count. Precedence:
+ *   1. An explicit numeric value in the config wins (user opt-in).
+ *   2. `MEMBOT_EMBEDDING_WORKERS` env var, if set to a positive integer.
+ *      The test harness sets this to `1` so unit tests doing tiny writes
+ *      don't pay the per-pool subprocess-spawn cost on slow CI runners.
+ *   3. Otherwise `null`/missing → `max(1, cpus()-1)`. The minus-one leaves
+ *      a core for the parent process (DB writes, IO, the spinner).
+ */
+export function resolveEmbeddingWorkers(configured: number | null | undefined): number {
+	if (typeof configured === "number" && configured >= 1) return configured;
+	const envOverride = process.env.MEMBOT_EMBEDDING_WORKERS;
+	if (envOverride) {
+		const n = Number(envOverride);
+		if (Number.isFinite(n) && n >= 1) return Math.floor(n);
+	}
+	return Math.max(1, cpus().length - 1);
+}
+
 /**
  * Build the AppContext used by every operation handler. Initializes:
  *  - output mode (TTY/JSON/color detection — frozen for the rest of the run)
  *  - config (~/.membot/config.json with env overrides)
  *  - DuckDB connection (~/.membot/index.duckdb), running migrations on first open
+ *
+ * The embedder worker pool is NOT created here — it's per-command,
+ * spawned by `withEmbedderPool()` at the top of bulk-embedding handlers
+ * (`add`, `refresh`, `write`) and disposed before they return.
  */
 export async function buildContext(options: BuildContextOptions = {}): Promise<AppContext> {
 	setMode(detectMode({ json: options.json, verbose: options.verbose, noColor: options.noColor }));

package/src/ingest/embed-worker.ts

@@ -0,0 +1,74 @@
+import { createInterface } from "node:readline";
+import { asHelpful, isHelpfulError } from "../errors.ts";
+import { embed } from "./embedder.ts";
+
+/**
+ * Wire-format message exchanged between the parent EmbedderPool and a worker
+ * subprocess over the worker's stdin/stdout. Each message is a single JSON
+ * object terminated by `\n` — a robust, language-agnostic encoding that
+ * survives partial reads on either end. There is no init/ready handshake;
+ * the worker lazy-loads the WASM pipeline on its first `embed` request.
+ */
+interface EmbedRequest {
+	type: "embed";
+	id: number;
+	model: string;
+	texts: string[];
+}
+
+interface EmbedResponse {
+	type: "embed-response";
+	id: number;
+	vectors?: number[][];
+	error?: { kind: string; message: string; hint: string };
+}
+
+/** Atomic JSON-line write to stdout — the protocol channel back to the parent. */
+function send(msg: EmbedResponse): void {
+	process.stdout.write(`${JSON.stringify(msg)}\n`);
+}
+
+/**
+ * Drive the embed-worker subprocess: read newline-delimited JSON requests
+ * from stdin, dispatch them to the local `embed()` (which uses the WASM
+ * pipeline), and write JSON responses to stdout. Diagnostics (logger output)
+ * go to stderr, so the protocol channel on stdout stays clean.
+ *
+ * The worker exits naturally when stdin closes (parent died or sent EOF).
+ */
+export async function runEmbedWorker(): Promise<void> {
+	const rl = createInterface({ input: process.stdin, crlfDelay: Number.POSITIVE_INFINITY });
+	for await (const line of rl) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		let req: EmbedRequest;
+		try {
+			req = JSON.parse(trimmed) as EmbedRequest;
+		} catch {
+			// A malformed line on stdin is almost certainly a bug in the parent —
+			// log to stderr and keep the worker alive so the parent's other
+			// requests still get served.
+			process.stderr.write(`embed-worker: ignoring malformed stdin line: ${trimmed.slice(0, 200)}\n`);
+			continue;
+		}
+		if (req.type !== "embed") continue;
+		await handleEmbed(req);
+	}
+}
+
+/** Run one embed request and reply with vectors or a serialisable error. */
+async function handleEmbed(req: EmbedRequest): Promise<void> {
+	try {
+		const vectors = await embed(req.texts, req.model);
+		send({ type: "embed-response", id: req.id, vectors });
+	} catch (err) {
+		const helpful = isHelpfulError(err)
+			? err
+			: asHelpful(err, "in embed worker", "Inspect the parent process stderr for the full stack trace.");
+		send({
+			type: "embed-response",
+			id: req.id,
+			error: { kind: helpful.kind, message: helpful.message, hint: helpful.hint },
+		});
+	}
+}

package/src/ingest/embedder-pool.ts

@@ -0,0 +1,391 @@
+import type { Subprocess } from "bun";
+import { EMBED_WORKER_SENTINEL, EMBEDDING_BATCH_SIZE, EMBEDDING_MODEL } from "../constants.ts";
+import { asHelpful, HelpfulError } from "../errors.ts";
+import { logger } from "../output/logger.ts";
+import { type EmbedOptions, setEmbedderPool } from "./embedder.ts";
+
+interface PendingRequest {
+	id: number;
+	resolve: (vectors: number[][]) => void;
+	reject: (err: unknown) => void;
+}
+
+interface Worker {
+	proc: Subprocess<"pipe", "pipe", "inherit">;
+	busy: boolean;
+	pending: PendingRequest | null;
+	exited: boolean;
+}
+
+interface EmbedResponseLine {
+	type: "embed-response";
+	id: number;
+	vectors?: number[][];
+	error?: { kind: string; message: string; hint: string };
+}
+
+/**
+ * A short-lived pool of embed-worker subprocesses. Created at the start of
+ * a bulk-embedding command (`add` / `refresh` / `write`), kept alive only
+ * for the duration of that command, and disposed before the command
+ * returns. Workers spawn lazily — they don't pre-load the WASM pipeline;
+ * the model is loaded on-demand inside the worker the first time a batch
+ * arrives. Each worker holds its own ~50MB WASM heap, so the parallelism
+ * comes for free in CPU but costs proportional RAM while the command runs.
+ *
+ * The pool is plugged in via `setEmbedderPool()` so the existing `embed()`
+ * call sites in the ingest pipeline transparently fan out without code
+ * changes.
+ */
+export class EmbedderPool {
+	private readonly workerCount: number;
+	private readonly model: string;
+	private workers: Worker[] = [];
+	private acquireQueue: Array<(w: Worker) => void> = [];
+	private nextRequestId = 1;
+	private spawned = false;
+	private disposed = false;
+
+	constructor(workerCount: number, model: string = EMBEDDING_MODEL) {
+		if (workerCount < 1 || !Number.isInteger(workerCount)) {
+			throw new HelpfulError({
+				kind: "input_error",
+				message: `EmbedderPool worker count must be a positive integer, got ${workerCount}`,
+				hint: "Set config.embedding.workers to a positive integer (or null for auto = cpus-1).",
+			});
+		}
+		this.workerCount = workerCount;
+		this.model = model;
+	}
+
+	/** Number of worker subprocesses this pool manages. */
+	get size(): number {
+		return this.workerCount;
+	}
+
+	/**
+	 * Spawn the worker subprocesses. Returns immediately — workers load the
+	 * WASM model lazily when the first batch arrives, so this is a cheap
+	 * operation. The first batch a worker receives pays the ~hundreds-of-ms
+	 * load cost; subsequent batches in the same worker are fast.
+	 */
+	spawn(): void {
+		if (this.spawned) return;
+		this.spawned = true;
+		logger.info(`embedder-pool: spawning ${this.workerCount} workers (model=${this.model})`);
+		for (let i = 0; i < this.workerCount; i++) {
+			this.workers.push(this.spawnWorker(i));
+		}
+	}
+
+	/**
+	 * Embed `texts` using the worker pool. Splits into batches of
+	 * `EMBEDDING_BATCH_SIZE`, dispatches each batch to whichever worker is
+	 * free, and reassembles vectors in original order. `opts.onProgress` is
+	 * called once per completed batch with `(done, total)` chunk counts.
+	 */
+	async embed(texts: string[], model?: string, opts: EmbedOptions = {}): Promise<number[][]> {
+		if (this.disposed) {
+			throw new HelpfulError({
+				kind: "internal_error",
+				message: "EmbedderPool: embed() called after dispose()",
+				hint: "The pool is per-command — wrap your work in `withEmbedderPool()` so a fresh pool is created.",
+			});
+		}
+		if (!this.spawned) this.spawn();
+		if (texts.length === 0) return [];
+
+		const targetModel = model ?? this.model;
+		const out = new Array<number[]>(texts.length);
+		let done = 0;
+
+		const batches: Array<{ start: number; texts: string[] }> = [];
+		for (let i = 0; i < texts.length; i += EMBEDDING_BATCH_SIZE) {
+			batches.push({ start: i, texts: texts.slice(i, i + EMBEDDING_BATCH_SIZE) });
+		}
+
+		await Promise.all(
+			batches.map(async (batch) => {
+				const vectors = await this.dispatchBatch(batch.texts, targetModel);
+				for (let i = 0; i < vectors.length; i++) {
+					const vec = vectors[i];
+					if (!vec) {
+						throw new HelpfulError({
+							kind: "internal_error",
+							message: `embedder-pool: worker returned undefined vector at batch index ${i}`,
+							hint: "Re-run with --verbose; check the worker stderr for a transformers/WASM error.",
+						});
+					}
+					out[batch.start + i] = vec;
+				}
+				done += vectors.length;
+				opts.onProgress?.(done, texts.length);
+			}),
+		);
+		return out;
+	}
+
+	/**
+	 * Tear down every worker subprocess. Idempotent. Pending requests are
+	 * rejected so any in-flight `embed()` callers see a HelpfulError instead
+	 * of hanging forever.
+	 */
+	async dispose(): Promise<void> {
+		if (this.disposed) return;
+		this.disposed = true;
+		if (this.spawned) {
+			logger.info(`embedder-pool: tearing down ${this.workers.length} workers`);
+		}
+		const disposeError = () =>
+			new HelpfulError({
+				kind: "internal_error",
+				message: "EmbedderPool disposed while a request was in flight",
+				hint: "This is usually fine on shutdown; if it appears mid-run, file an issue with the preceding stderr.",
+			});
+		for (const w of this.workers) {
+			if (w.pending) {
+				w.pending.reject(disposeError());
+				w.pending = null;
+			}
+			try {
+				w.proc.stdin.end();
+			} catch {
+				// stdin may already be closed; ignore.
+			}
+			try {
+				w.proc.kill();
+			} catch {
+				// process may already be dead; ignore.
+			}
+		}
+		// Anyone waiting on acquire() will never get a worker — release them.
+		const queue = this.acquireQueue;
+		this.acquireQueue = [];
+		for (const resolver of queue) {
+			// Fabricate an "already exited" worker so dispatchBatch's disposed
+			// guard fires and rejects with a clear error.
+			resolver({
+				proc: null as unknown as Subprocess<"pipe", "pipe", "inherit">,
+				busy: true,
+				pending: null,
+				exited: true,
+			});
+		}
+		await Promise.all(
+			this.workers.map(async (w) => {
+				try {
+					await w.proc.exited;
+				} catch {
+					// best effort
+				}
+			}),
+		);
+		this.workers = [];
+	}
+
+	/** Send one batch to a free worker and resolve with its vectors. */
+	private async dispatchBatch(texts: string[], model: string): Promise<number[][]> {
+		const worker = await this.acquire();
+		try {
+			if (this.disposed || worker.exited) {
+				throw new HelpfulError({
+					kind: "internal_error",
+					message: "EmbedderPool disposed before batch could be dispatched",
+					hint: "The pool was torn down mid-call — wrap your work in `withEmbedderPool()` for a fresh per-command pool.",
+				});
+			}
+			const id = this.nextRequestId++;
+			return await new Promise<number[][]>((resolve, reject) => {
+				worker.pending = { id, resolve, reject };
+				try {
+					worker.proc.stdin.write(`${JSON.stringify({ type: "embed", id, model, texts })}\n`);
+					worker.proc.stdin.flush();
+				} catch (err) {
+					worker.pending = null;
+					reject(
+						asHelpful(
+							err,
+							"while writing to embed worker stdin",
+							"The worker subprocess likely crashed. Set config.embedding.workers=1 to bypass the pool while debugging.",
+						),
+					);
+				}
+			});
+		} finally {
+			this.release(worker);
+		}
+	}
+
+	/** Wait for a free worker; first-come, first-served via the acquireQueue. */
+	private acquire(): Promise<Worker> {
+		const free = this.workers.find((w) => !w.exited && !w.busy);
+		if (free) {
+			free.busy = true;
+			return Promise.resolve(free);
+		}
+		return new Promise((resolve) => {
+			this.acquireQueue.push((w) => {
+				w.busy = true;
+				resolve(w);
+			});
+		});
+	}
+
+	/**
+	 * Hand a finished worker to the next waiter, or mark it idle. Called from
+	 * `dispatchBatch`'s finally block so it runs whether the request resolved
+	 * or rejected.
+	 */
+	private release(w: Worker): void {
+		w.pending = null;
+		w.busy = false;
+		if (w.exited) return;
+		const next = this.acquireQueue.shift();
+		if (next) next(w);
+	}
+
+	/**
+	 * Build the spawn command for one worker. Two regimes:
+	 *  - Compiled binary (`./dist/membot`): `process.execPath` is the membot
+	 *    binary itself, so we just hand it the sentinel arg and the early
+	 *    branch in `cli.ts` takes over before commander sees it.
+	 *  - Bun dev / `bun add -g`: `process.execPath` is the `bun` binary; we
+	 *    must point it at `cli.ts` explicitly. Resolve the path relative to
+	 *    this module so it survives whatever working directory the user
+	 *    invoked membot from.
+	 */
+	private resolveSpawnCommand(): string[] {
+		const exec = process.execPath;
+		const isBun = /[\\/]bunx?(\.exe)?$/.test(exec);
+		if (!isBun) {
+			return [exec, EMBED_WORKER_SENTINEL];
+		}
+		const cliPath = new URL("../cli.ts", import.meta.url).pathname;
+		return [exec, cliPath, EMBED_WORKER_SENTINEL];
+	}
+
+	/**
+	 * Spawn one worker subprocess and start its stdout reader. The worker
+	 * lazy-loads the WASM pipeline on its first `embed` request, so spawn is
+	 * cheap (no init handshake, no preload).
+	 */
+	private spawnWorker(index: number): Worker {
+		const proc = Bun.spawn(this.resolveSpawnCommand(), {
+			stdio: ["pipe", "pipe", "inherit"],
+		}) as Subprocess<"pipe", "pipe", "inherit">;
+
+		const worker: Worker = {
+			proc,
+			busy: false,
+			pending: null,
+			exited: false,
+		};
+
+		// Watch for premature exit and surface it to any in-flight request.
+		void proc.exited
+			.then((code) => {
+				worker.exited = true;
+				if (worker.pending) {
+					worker.pending.reject(
+						new HelpfulError({
+							kind: "internal_error",
+							message: `embed worker ${index} exited (code=${code}) with a request in flight`,
+							hint: "Run with --verbose; the worker's stderr was inherited and should explain the crash.",
+						}),
+					);
+					worker.pending = null;
+				}
+			})
+			.catch(() => {
+				// Bun's exited promise shouldn't reject, but guard anyway.
+			});
+
+		void this.readWorker(worker, index);
+		return worker;
+	}
+
+	/**
+	 * Newline-delimited JSON reader for one worker's stdout. Matches every
+	 * `{type:"embed-response", id}` to its pending request.
+	 */
+	private async readWorker(worker: Worker, index: number): Promise<void> {
+		const reader = worker.proc.stdout.getReader();
+		const decoder = new TextDecoder();
+		let buffer = "";
+		try {
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				buffer += decoder.decode(value, { stream: true });
+				while (true) {
+					const nl = buffer.indexOf("\n");
+					if (nl === -1) break;
+					const line = buffer.slice(0, nl);
+					buffer = buffer.slice(nl + 1);
+					if (!line.trim()) continue;
+					this.handleWorkerLine(worker, index, line);
+				}
+			}
+		} catch (err) {
+			logger.debug(`embedder-pool: worker ${index} stdout read failed: ${(err as Error).message}`);
+		}
+	}
+
+	/** Parse + dispatch one JSON line emitted by a worker. */
+	private handleWorkerLine(worker: Worker, index: number, line: string): void {
+		let parsed: EmbedResponseLine;
+		try {
+			parsed = JSON.parse(line) as EmbedResponseLine;
+		} catch {
+			logger.debug(`embedder-pool: worker ${index} emitted unparseable line: ${line.slice(0, 200)}`);
+			return;
+		}
+		if (parsed.type !== "embed-response") return;
+		const pending = worker.pending;
+		if (!pending) {
+			logger.debug(`embedder-pool: worker ${index} returned response with no pending request`);
+			return;
+		}
+		if (parsed.error) {
+			pending.reject(
+				new HelpfulError({
+					kind: "internal_error",
+					message: `embed worker ${index} failed: ${parsed.error.message}`,
+					hint: parsed.error.hint || "Inspect parent stderr for the full error.",
+				}),
+			);
+		} else if (parsed.vectors) {
+			pending.resolve(parsed.vectors);
+		} else {
+			pending.reject(
+				new HelpfulError({
+					kind: "internal_error",
+					message: `embed worker ${index} returned response with neither vectors nor error`,
+					hint: "This is a worker protocol bug — file an issue with the preceding stderr.",
+				}),
+			);
+		}
+	}
+}
+
+/**
+ * Run `fn` with a fresh `EmbedderPool` registered as the global embedder. The
+ * pool is created, plugged in via `setEmbedderPool()`, and disposed
+ * (subprocesses killed) before `fn`'s promise resolves — so the workers only
+ * exist for the duration of one bulk-embedding command (`add` / `refresh` /
+ * `write` / a daemon tick). When `workers <= 1` the helper short-circuits
+ * and runs `fn` inline against the single-process embedder, with no spawn
+ * overhead.
+ */
+export async function withEmbedderPool<T>(workerCount: number, model: string, fn: () => Promise<T>): Promise<T> {
+	if (workerCount <= 1) return fn();
+	const pool = new EmbedderPool(workerCount, model);
+	pool.spawn();
+	setEmbedderPool(pool);
+	try {
+		return await fn();
+	} finally {
+		setEmbedderPool(null);
+		await pool.dispose();
+	}
+}