membot 0.4.1 → 0.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.4.1",
+	"version": "0.4.2",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {

package/src/ingest/embedder.ts

@@ -63,6 +63,16 @@ async function getPipeline(model: string): Promise<FeatureExtractionPipeline> {
 	return p;
 }
 
+/**
+ * Options for `embed()`. `onProgress` fires once after each batch finishes
+ * with `(done, total)` chunk counts so callers can drive a spinner / progress
+ * bar — ONNX WASM holds the JS thread for hundreds of ms per batch and would
+ * otherwise leave nanospinner's setInterval starved between updates.
+ */
+export interface EmbedOptions {
+	onProgress?: (done: number, total: number) => void;
+}
+
 /**
  * Embed an array of texts to L2-normalized vectors with the configured
  * model. Throws a HelpfulError when the model's dimension doesn't match
@@ -71,8 +81,16 @@ async function getPipeline(model: string): Promise<FeatureExtractionPipeline> {
  * Inputs are sliced into windows of EMBEDDING_BATCH_SIZE so a single
  * forward pass never has to allocate activations for arbitrarily many
  * chunks — large files (hundreds of chunks) otherwise OOM the WASM heap.
+ *
+ * Between batches we yield a macrotask (`setTimeout(0)`) so the event loop
+ * can flush nanospinner renders and stderr writes — without that, the spinner
+ * visibly freezes for the entire embed phase on large files.
  */
-export async function embed(texts: string[], model: string = EMBEDDING_MODEL): Promise<number[][]> {
+export async function embed(
+	texts: string[],
+	model: string = EMBEDDING_MODEL,
+	opts: EmbedOptions = {},
+): Promise<number[][]> {
 	if (texts.length === 0) return [];
 	const extractor = await getPipeline(model);
 	const out: number[][] = [];
@@ -88,6 +106,10 @@ export async function embed(texts: string[], model: string = EMBEDDING_MODEL): P
 			});
 		}
 		for (const vec of data) out.push(vec);
+		opts.onProgress?.(out.length, texts.length);
+		// Yield a macrotask so nanospinner's setInterval and any queued
+		// stderr writes get a chance to run between batches.
+		await new Promise<void>((resolve) => setTimeout(resolve, 0));
 	}
 	return out;
 }

package/src/ingest/ingest.ts

@@ -54,6 +54,13 @@ export interface IngestResult {
 export interface IngestCallbacks {
 	onEntryStart?: (label: string) => void;
 	onEntryComplete?: (entry: IngestEntryResult) => void;
+	/**
+	 * Fires for sub-step progress within a single entry (e.g. "embedding
+	 * 32/168"). The callback runs many times per entry and is intended for
+	 * driving an interactive spinner — non-interactive callers should ignore
+	 * it to avoid log spam.
+	 */
+	onEntryProgress?: (label: string, sublabel: string) => void;
 }
 
 /**
@@ -140,23 +147,27 @@ async function ingestInline(
 		source_sha256: sha,
 	};
 	try {
-		const versionId = await persistVersion(ctx, {
-			logicalPath,
-			sourceType: "inline",
-			sourcePath: null,
-			sourceMtimeMs: null,
-			sourceSha: sha,
-			blobSha: null,
-			mime: "text/markdown",
-			bytes: null,
-			markdown: text,
-			fetcher: "inline",
-			fetcherServer: null,
-			fetcherTool: null,
-			fetcherArgs: null,
-			refreshSec,
-			changeNote: input.change_note ?? null,
-		});
+		const versionId = await persistVersion(
+			ctx,
+			{
+				logicalPath,
+				sourceType: "inline",
+				sourcePath: null,
+				sourceMtimeMs: null,
+				sourceSha: sha,
+				blobSha: null,
+				mime: "text/markdown",
+				bytes: null,
+				markdown: text,
+				fetcher: "inline",
+				fetcherServer: null,
+				fetcherTool: null,
+				fetcherArgs: null,
+				refreshSec,
+				changeNote: input.change_note ?? null,
+			},
+			(done, total) => callbacks?.onEntryProgress?.(logicalPath, `embedding ${done}/${total}`),
+		);
 		result.version_id = versionId;
 	} catch (err) {
 		result.status = "failed";
@@ -217,22 +228,26 @@ async function ingestUrl(
 			}
 		}
 
-		const versionId = await pipelineForBytes(ctx, {
-			logicalPath,
-			bytes: fetched.bytes,
-			mime: fetched.mimeType,
-			source: url,
-			sourceType: "remote",
-			sourcePath: url,
-			sourceMtimeMs: null,
-			sourceSha: fetched.sha256,
-			fetcher: fetched.fetcher,
-			fetcherServer: fetched.fetcherServer,
-			fetcherTool: fetched.fetcherTool,
-			fetcherArgs: fetched.fetcherArgs,
-			refreshSec,
-			changeNote: input.change_note ?? null,
-		});
+		const versionId = await pipelineForBytes(
+			ctx,
+			{
+				logicalPath,
+				bytes: fetched.bytes,
+				mime: fetched.mimeType,
+				source: url,
+				sourceType: "remote",
+				sourcePath: url,
+				sourceMtimeMs: null,
+				sourceSha: fetched.sha256,
+				fetcher: fetched.fetcher,
+				fetcherServer: fetched.fetcherServer,
+				fetcherTool: fetched.fetcherTool,
+				fetcherArgs: fetched.fetcherArgs,
+				refreshSec,
+				changeNote: input.change_note ?? null,
+			},
+			(done, total) => callbacks?.onEntryProgress?.(url, `embedding ${done}/${total}`),
+		);
 		result.version_id = versionId;
 	} catch (err) {
 		result.status = "failed";
@@ -299,22 +314,26 @@ async function ingestLocalFiles(
 				}
 			}
 
-			const versionId = await pipelineForBytes(ctx, {
-				logicalPath,
-				bytes: local.bytes,
-				mime: local.mimeType,
-				source: entry.absPath,
-				sourceType: "local",
-				sourcePath: entry.absPath,
-				sourceMtimeMs: local.mtimeMs,
-				sourceSha: local.sha256,
-				fetcher: "local",
-				fetcherServer: null,
-				fetcherTool: null,
-				fetcherArgs: null,
-				refreshSec,
-				changeNote: input.change_note ?? null,
-			});
+			const versionId = await pipelineForBytes(
+				ctx,
+				{
+					logicalPath,
+					bytes: local.bytes,
+					mime: local.mimeType,
+					source: entry.absPath,
+					sourceType: "local",
+					sourcePath: entry.absPath,
+					sourceMtimeMs: local.mtimeMs,
+					sourceSha: local.sha256,
+					fetcher: "local",
+					fetcherServer: null,
+					fetcherTool: null,
+					fetcherArgs: null,
+					refreshSec,
+					changeNote: input.change_note ?? null,
+				},
+				(done, total) => callbacks?.onEntryProgress?.(entry.relPathFromBase, `embedding ${done}/${total}`),
+			);
 			result.version_id = versionId;
 		} catch (err) {
 			result.status = "failed";
@@ -353,9 +372,14 @@ interface PipelineParams {
  * Run the bytes-in / version-out pipeline: store the blob, convert to
  * markdown, describe, chunk, embed, and write a new files row + chunks
  * rows under a fresh version_id. Returns the version_id so callers can
- * report it back.
+ * report it back. The optional `onEmbedProgress` is forwarded to the
+ * embedder so callers can drive a spinner during the slow phase.
  */
-async function pipelineForBytes(ctx: AppContext, p: PipelineParams): Promise<string> {
+async function pipelineForBytes(
+	ctx: AppContext,
+	p: PipelineParams,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<string> {
 	await upsertBlob(ctx.db, {
 		sha256: p.sourceSha,
 		mime_type: p.mime,
@@ -367,24 +391,28 @@ async function pipelineForBytes(ctx: AppContext, p: PipelineParams): Promise<str
 	const markdown = conversion.markdown;
 	const contentSha = sha256Hex(new TextEncoder().encode(markdown));
 
-	return persistVersion(ctx, {
-		logicalPath: p.logicalPath,
-		sourceType: p.sourceType,
-		sourcePath: p.sourcePath,
-		sourceMtimeMs: p.sourceMtimeMs,
-		sourceSha: p.sourceSha,
-		blobSha: p.sourceSha,
-		mime: p.mime,
-		bytes: p.bytes,
-		markdown,
-		contentSha,
-		fetcher: p.fetcher,
-		fetcherServer: p.fetcherServer,
-		fetcherTool: p.fetcherTool,
-		fetcherArgs: p.fetcherArgs,
-		refreshSec: p.refreshSec,
-		changeNote: p.changeNote,
-	});
+	return persistVersion(
+		ctx,
+		{
+			logicalPath: p.logicalPath,
+			sourceType: p.sourceType,
+			sourcePath: p.sourcePath,
+			sourceMtimeMs: p.sourceMtimeMs,
+			sourceSha: p.sourceSha,
+			blobSha: p.sourceSha,
+			mime: p.mime,
+			bytes: p.bytes,
+			markdown,
+			contentSha,
+			fetcher: p.fetcher,
+			fetcherServer: p.fetcherServer,
+			fetcherTool: p.fetcherTool,
+			fetcherArgs: p.fetcherArgs,
+			refreshSec: p.refreshSec,
+			changeNote: p.changeNote,
+		},
+		onEmbedProgress,
+	);
 }
 
 interface PersistParams {
@@ -412,13 +440,17 @@ interface PersistParams {
  * embedded text per chunk is `<path>\n<description>\n\n<body>`, stored
  * verbatim as `chunks.search_text` and later FTS-indexed.
  */
-async function persistVersion(ctx: AppContext, p: PersistParams): Promise<string> {
+async function persistVersion(
+	ctx: AppContext,
+	p: PersistParams,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<string> {
 	const description = await describe(p.logicalPath, p.mime, p.markdown, ctx.config.llm);
 	const chunks = chunkDeterministic(p.markdown, ctx.config.chunker);
 	const searchTexts = chunks.map((c) => buildSearchText(p.logicalPath, description, c.content));
 	let embeddings: number[][];
 	try {
-		embeddings = await embed(searchTexts, ctx.config.embedding_model);
+		embeddings = await embed(searchTexts, ctx.config.embedding_model, { onProgress: onEmbedProgress });
 	} catch (err) {
 		throw asHelpful(
 			err,

package/src/operations/add.ts

@@ -138,6 +138,7 @@ Pass \`logical_path\` to override. For a multi-source / directory / glob walk it
 		const callbacks: IngestCallbacks = {
 			onEntryStart: (label) => ctx.progress.tick(label),
 			onEntryComplete: (entry) => ctx.progress.entry(formatEntryLine(entry)),
+			onEntryProgress: (_label, sublabel) => ctx.progress.update(sublabel),
 		};
 
 		for (const outcome of outcomes) {

package/src/operations/refresh.ts

@@ -60,7 +60,9 @@ export const refreshOperation = defineOperation({
 		for (const path of targets) {
 			ctx.progress.tick(path);
 			try {
-				const r = await refreshOne(ctx, path, input.force);
+				const r = await refreshOne(ctx, path, input.force, (done, total) =>
+					ctx.progress.update(`embedding ${done}/${total}`),
+				);
 				out.push(r);
 			} catch (err) {
 				out.push({ logical_path: path, status: "failed", error: err instanceof Error ? err.message : String(err) });

package/src/output/progress.ts

@@ -15,6 +15,13 @@ import { isSilent, useSpinner } from "./tty.ts";
 export interface Progress {
 	start(total: number, label?: string): void;
 	tick(label: string): void;
+	/**
+	 * Re-render the active spinner with the most recent `tick` label plus an
+	 * extra suffix (e.g. "embedding 32/168") without advancing the counter.
+	 * No-op in non-interactive / silent / JSON modes — sub-step progress is
+	 * deliberately TTY-only so CI logs don't get one line per inner batch.
+	 */
+	update(suffix: string): void;
 	entry(line: string): void;
 	done(summary?: string): void;
 	fail(summary?: string): void;
@@ -51,25 +58,28 @@ function truncateLabel(label: string, max = LABEL_MAX): string {
 export function createProgress(): Progress {
 	let total = 0;
 	let count = 0;
+	let lastLabel = "";
 	let spinner: ReturnType<typeof logger.startSpinner> | null = null;
 
 	const interactive = useSpinner();
 	const silent = isSilent();
 
-	const renderSpinnerText = (label: string): string => {
+	const renderSpinnerText = (label: string, suffix?: string): string => {
 		const bar = renderBar(count, total);
 		const pct = total > 0 ? Math.floor((count / total) * 100) : 0;
-		const tail = label ? ` — ${truncateLabel(label)}` : "";
-		return `${bar} ${count}/${total} (${pct}%)${tail}`;
+		const labelTail = label ? ` — ${truncateLabel(label)}` : "";
+		const suffixTail = suffix ? ` — ${suffix}` : "";
+		return `${bar} ${count}/${total} (${pct}%)${labelTail}${suffixTail}`;
 	};
 
 	return {
 		start(t: number, label?: string) {
 			total = t;
 			count = 0;
+			lastLabel = label ?? "";
 			if (silent) return;
 			if (interactive) {
-				const initial = renderSpinnerText(label ?? "");
+				const initial = renderSpinnerText(lastLabel);
 				spinner = logger.startSpinner(initial);
 			} else if (label) {
 				logger.info(`${label}: 0/${total}`);
@@ -77,6 +87,7 @@ export function createProgress(): Progress {
 		},
 		tick(label: string) {
 			count += 1;
+			lastLabel = label;
 			if (silent) return;
 			if (interactive && spinner) {
 				spinner.update(renderSpinnerText(label));
@@ -84,6 +95,11 @@ export function createProgress(): Progress {
 				logger.info(`[${count}/${total}] ${label}`);
 			}
 		},
+		update(suffix: string) {
+			if (silent) return;
+			if (!interactive || !spinner) return;
+			spinner.update(renderSpinnerText(lastLabel, suffix));
+		},
 		entry(line: string) {
 			if (silent) return;
 			logger.info(line);

package/src/refresh/runner.ts

@@ -24,9 +24,16 @@ export interface RefreshOutcome {
  * via the persisted mcpx invocation), and creates a new version only if
  * the source bytes changed. Always updates `refreshed_at` and
  * `last_refresh_status` on the row. Returns a per-path outcome — never
- * throws unless the path doesn't exist.
+ * throws unless the path doesn't exist. The optional `onEmbedProgress`
+ * callback is forwarded to the embedder so interactive callers (e.g. the
+ * `refresh` operation) can drive a spinner during the slow phase.
  */
-export async function refreshOne(ctx: AppContext, logicalPath: string, force = false): Promise<RefreshOutcome> {
+export async function refreshOne(
+	ctx: AppContext,
+	logicalPath: string,
+	force = false,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<RefreshOutcome> {
 	const cur = await getCurrent(ctx.db, logicalPath);
 	if (!cur) {
 		throw new HelpfulError({
@@ -42,10 +49,10 @@ export async function refreshOne(ctx: AppContext, logicalPath: string, force = f
 
 	try {
 		if (cur.source_type === "local") {
-			return await refreshLocal(ctx, cur, force);
+			return await refreshLocal(ctx, cur, force, onEmbedProgress);
 		}
 		if (cur.source_type === "remote") {
-			return await refreshRemote(ctx, cur, force);
+			return await refreshRemote(ctx, cur, force, onEmbedProgress);
 		}
 	} catch (err) {
 		const message = err instanceof Error ? err.message : String(err);
@@ -74,7 +81,12 @@ interface CurrentRow {
 }
 
 /** Local-file refresh: stat-then-sha gate before re-running the pipeline. */
-async function refreshLocal(ctx: AppContext, cur: CurrentRow, force: boolean): Promise<RefreshOutcome> {
+async function refreshLocal(
+	ctx: AppContext,
+	cur: CurrentRow,
+	force: boolean,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<RefreshOutcome> {
 	if (!cur.source_path) {
 		throw new HelpfulError({
 			kind: "input_error",
@@ -92,26 +104,35 @@ async function refreshLocal(ctx: AppContext, cur: CurrentRow, force: boolean): P
 		return { logical_path: cur.logical_path, status: "unchanged" };
 	}
 
-	const versionId = await runPipelineForRefresh(ctx, {
-		logicalPath: cur.logical_path,
-		bytes: local.bytes,
-		mime: local.mimeType,
-		source: cur.source_path,
-		sourceType: "local",
-		sourcePath: cur.source_path,
-		sourceMtimeMs: local.mtimeMs,
-		sourceSha: local.sha256,
-		fetcher: "local",
-		fetcherServer: null,
-		fetcherTool: null,
-		fetcherArgs: null,
-		refreshSec: cur.refresh_frequency_sec,
-	});
+	const versionId = await runPipelineForRefresh(
+		ctx,
+		{
+			logicalPath: cur.logical_path,
+			bytes: local.bytes,
+			mime: local.mimeType,
+			source: cur.source_path,
+			sourceType: "local",
+			sourcePath: cur.source_path,
+			sourceMtimeMs: local.mtimeMs,
+			sourceSha: local.sha256,
+			fetcher: "local",
+			fetcherServer: null,
+			fetcherTool: null,
+			fetcherArgs: null,
+			refreshSec: cur.refresh_frequency_sec,
+		},
+		onEmbedProgress,
+	);
 	return { logical_path: cur.logical_path, status: "ok", new_version_id: versionId };
 }
 
 /** Remote refresh: replay the persisted mcpx invocation, or plain HTTP. */
-async function refreshRemote(ctx: AppContext, cur: CurrentRow, force: boolean): Promise<RefreshOutcome> {
+async function refreshRemote(
+	ctx: AppContext,
+	cur: CurrentRow,
+	force: boolean,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<RefreshOutcome> {
 	if (!cur.source_path) {
 		throw new HelpfulError({
 			kind: "input_error",
@@ -129,21 +150,25 @@ async function refreshRemote(ctx: AppContext, cur: CurrentRow, force: boolean):
 		return { logical_path: cur.logical_path, status: "unchanged" };
 	}
 
-	const versionId = await runPipelineForRefresh(ctx, {
-		logicalPath: cur.logical_path,
-		bytes: fetched.bytes,
-		mime: fetched.mimeType,
-		source: cur.source_path,
-		sourceType: "remote",
-		sourcePath: cur.source_path,
-		sourceMtimeMs: null,
-		sourceSha: fetched.sha256,
-		fetcher: cur.fetcher === "mcpx" ? "mcpx" : "http",
-		fetcherServer: fetched.fetcherServer,
-		fetcherTool: fetched.fetcherTool,
-		fetcherArgs: fetched.fetcherArgs,
-		refreshSec: cur.refresh_frequency_sec,
-	});
+	const versionId = await runPipelineForRefresh(
+		ctx,
+		{
+			logicalPath: cur.logical_path,
+			bytes: fetched.bytes,
+			mime: fetched.mimeType,
+			source: cur.source_path,
+			sourceType: "remote",
+			sourcePath: cur.source_path,
+			sourceMtimeMs: null,
+			sourceSha: fetched.sha256,
+			fetcher: cur.fetcher === "mcpx" ? "mcpx" : "http",
+			fetcherServer: fetched.fetcherServer,
+			fetcherTool: fetched.fetcherTool,
+			fetcherArgs: fetched.fetcherArgs,
+			refreshSec: cur.refresh_frequency_sec,
+		},
+		onEmbedProgress,
+	);
 	return { logical_path: cur.logical_path, status: "ok", new_version_id: versionId };
 }
 
@@ -237,7 +262,11 @@ interface PipelineParams {
  * fields (`change_note='refresh: source updated'`) aren't accidentally
  * applied to first-time ingests.
  */
-async function runPipelineForRefresh(ctx: AppContext, p: PipelineParams): Promise<string> {
+async function runPipelineForRefresh(
+	ctx: AppContext,
+	p: PipelineParams,
+	onEmbedProgress?: (done: number, total: number) => void,
+): Promise<string> {
 	await upsertBlob(ctx.db, {
 		sha256: p.sourceSha,
 		mime_type: p.mime,
@@ -250,7 +279,7 @@ async function runPipelineForRefresh(ctx: AppContext, p: PipelineParams): Promis
 	const description = await describe(p.logicalPath, p.mime, markdown, ctx.config.llm);
 	const chunks = chunkDeterministic(markdown, ctx.config.chunker);
 	const searchTexts = chunks.map((c) => buildSearchText(p.logicalPath, description, c.content));
-	const embeddings = await embed(searchTexts, ctx.config.embedding_model);
+	const embeddings = await embed(searchTexts, ctx.config.embedding_model, { onProgress: onEmbedProgress });
 
 	const versionId = millisIso(Date.now());
 	const contentSha = sha256Hex(new TextEncoder().encode(markdown));