@zenera/rag 1.1.10 → 1.1.11

package/dist/docs/pool.d.ts

@@ -0,0 +1,27 @@
+import { type Chunk, type ChunkOptions } from './chunk.ts';
+import type { FileOutline } from './files.ts';
+export interface ParseInput {
+    name: string;
+    text: string;
+    format: 'markdown' | 'text';
+}
+export interface Parsed {
+    chunks: Chunk[];
+    outline: FileOutline;
+}
+/** One short of the machine, so the thread doing everything else keeps a core. */
+export declare const poolSize: () => number;
+export interface ParseAllOptions {
+    chunk?: ChunkOptions;
+    /** how many threads to allow; 1 keeps everything here */
+    workers?: number;
+    onProgress?: (done: number, total: number, pending: readonly string[]) => void;
+}
+export declare function parseAll(inputs: readonly ParseInput[], options?: ParseAllOptions): Promise<Parsed[]>;
+/**
+ * The threaded path on its own. `parseAll` falls back to this thread when the
+ * workers cannot run, which is right for a build and useless for a test: a
+ * worker that never starts would look exactly like a fast one. Tests call this.
+ */
+export declare function parseThreaded(inputs: readonly ParseInput[], chunk: ChunkOptions, size: number, onProgress?: (done: number, total: number, pending: readonly string[]) => void): Promise<Parsed[]>;
+//# sourceMappingURL=pool.d.ts.map

package/dist/docs/pool.js

@@ -0,0 +1,133 @@
+import { availableParallelism } from 'node:os';
+import { Worker } from 'node:worker_threads';
+import { chunkDocument } from "./chunk.js";
+import { outlineOf } from "./outline.js";
+import { parseDocument } from "./parse.js";
+// ---------------------------------------------------------------------------
+// Parsing several documents at once
+//
+// Remark is synchronous and holds the thread for as long as a document takes,
+// so a corpus is parsed strictly one file at a time no matter how many cores
+// are idle. A pool fixes that and nothing else: the work is pure, the inputs are
+// independent, and the answers are placed by index, so the result is the same
+// list in the same order that one thread would have produced.
+//
+// It is not always the faster choice. Starting a thread costs more than parsing
+// a small document, so a handful of files are done here; and a caller that
+// supplied its own `tokenCount` cannot be helped at all, because a function is
+// not something `postMessage` can clone. Every one of those paths runs the same
+// three functions in this thread instead, which is why the fallback is safe
+// rather than merely convenient.
+// ---------------------------------------------------------------------------
+/** Below this, starting threads costs more than the parsing they would save. */
+const MIN_DOCUMENTS = 8;
+/** More than this and the threads contend for memory bandwidth, not for work. */
+const MAX_WORKERS = 8;
+/** One short of the machine, so the thread doing everything else keeps a core. */
+export const poolSize = () => Math.max(1, Math.min(MAX_WORKERS, availableParallelism() - 1));
+const EXTENSION = import.meta.url.endsWith('.ts') ? '.ts' : '.js';
+// Built from `import.meta.url` because the extension changes on the way to
+// `dist/`, and `rewriteRelativeImportExtensions` does not touch string literals.
+const WORKER = new URL(`./parse-worker${EXTENSION}`, import.meta.url);
+/** How many documents run between yields, so a long parse still narrates. */
+const YIELD_EVERY = 32;
+export async function parseAll(inputs, options = {}) {
+    const chunk = options.chunk ?? {};
+    const workers = Math.min(options.workers ?? poolSize(), inputs.length);
+    const out = new Array(inputs.length);
+    if (workers > 1 && inputs.length >= MIN_DOCUMENTS && typeof chunk.tokenCount !== 'function') {
+        try {
+            await threaded(out, inputs, chunk, workers, options.onProgress);
+        }
+        catch {
+            // A thread that would not start, or died. Whatever it did finish is
+            // in `out` already and is not parsed again; `here` fills the rest.
+        }
+    }
+    return here(out, inputs, chunk, options.onProgress);
+}
+async function here(out, inputs, chunk, onProgress) {
+    let done = out.reduce((n, parsed) => (parsed ? n + 1 : n), 0);
+    for (const [at, input] of inputs.entries()) {
+        if (out[at]) {
+            continue;
+        }
+        const parsed = parseDocument(input.text, input.name, input.format);
+        const chunks = chunkDocument(parsed, chunk);
+        out[at] = { chunks, outline: outlineOf(parsed, chunks.length) };
+        onProgress?.(++done, inputs.length, [input.name]);
+        if (done % YIELD_EVERY === 0) {
+            // Remark is synchronous, so a corpus parsed here would hold the loop
+            // for minutes and freeze the very report saying it is still working.
+            await new Promise((resume) => setImmediate(resume));
+        }
+    }
+    return out;
+}
+/**
+ * The threaded path on its own. `parseAll` falls back to this thread when the
+ * workers cannot run, which is right for a build and useless for a test: a
+ * worker that never starts would look exactly like a fast one. Tests call this.
+ */
+export async function parseThreaded(inputs, chunk, size, onProgress) {
+    const out = new Array(inputs.length);
+    await threaded(out, inputs, chunk, size, onProgress);
+    return out;
+}
+async function threaded(out, inputs, chunk, size, onProgress) {
+    const workers = Array.from({ length: size }, () => new Worker(WORKER, { workerData: chunk }));
+    // What each thread is on, so a corpus with two pathological documents in it
+    // can name them instead of looking stalled a hair short of the total.
+    const busy = new Map();
+    let next = 0;
+    let done = 0;
+    const pump = async (worker) => {
+        while (next < inputs.length) {
+            const at = next++;
+            busy.set(worker, inputs[at].name);
+            out[at] = await ask(worker, { at, ...inputs[at] });
+            busy.delete(worker);
+            onProgress?.(++done, inputs.length, [...busy.values()]);
+        }
+    };
+    try {
+        await Promise.all(workers.map(pump));
+    }
+    finally {
+        await Promise.all(workers.map((worker) => worker.terminate()));
+    }
+}
+function ask(worker, job) {
+    return new Promise((resolve, reject) => {
+        const done = () => {
+            worker.off('message', onMessage);
+            worker.off('error', onError);
+            worker.off('exit', onExit);
+        };
+        const onMessage = (reply) => {
+            if (reply.at !== job.at) {
+                return;
+            }
+            done();
+            if (reply.error !== undefined) {
+                reject(new Error(`${job.name}: ${reply.error}`));
+            }
+            else {
+                resolve({ chunks: reply.chunks, outline: reply.outline });
+            }
+        };
+        const onError = (err) => {
+            done();
+            reject(err);
+        };
+        const onExit = (code) => {
+            done();
+            reject(new Error(`parse worker stopped with code ${code}`));
+        };
+        worker.on('message', onMessage);
+        worker.on('error', onError);
+        worker.on('exit', onExit);
+        worker.postMessage(job);
+    });
+}
+//# sourceMappingURL=pool.js.map

package/dist/docs/readme.js

@@ -1,11 +1,16 @@
 import { basename } from 'node:path';
 import { INTERVAL_MS, } from "../common/progress.js";
-import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+import { breakdown, duration, fields, grid, message, plural, searched } from "../common/prose.js";
 export const PHASES = {
     reading: 'reading the documents and cutting them into chunks',
     embedding: 'embedding',
     writing: 'writing the store',
 };
+/** What the running count is counting, which is not the same in every phase. */
+const COUNTING = {
+    reading: 'parsed',
+    embedding: 'embedded',
+};
 export const DOCS_REPORT = { building, complete, failed };
 function building(state) {
     const rows = [
@@ -21,9 +26,18 @@ function building(state) {
         rows.push(['found', counted(state.summary)]);
     }
     if (state.total > 0) {
-        const percent = Math.round((state.done / state.total) * 100);
-        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+        // Rounded down: 13619 of 13621 is not 100%, and a report that says it is
+        // turns two slow documents into a build that looks hung.
+        const percent = Math.floor((state.done / state.total) * 100);
+        rows.push([
+            COUNTING[state.phase] ?? 'done',
+            `${state.done} of ${state.total} · ${percent}%`,
+        ]);
+    }
+    if (state.pending.length > 0) {
+        rows.push(['still on', waiting(state.pending)]);
     }
+    rows.push(['timing', breakdown(state.timings)]);
     rows.push(['updated', new Date(state.now).toISOString()]);
     return [
         '# Document index — being built',
@@ -57,6 +71,8 @@ function complete(state) {
         `${counted(manifest.counts)},`,
         `${searched(manifest.indexes)}.`,
         '',
+        `Time: ${breakdown(state.timings)}.`,
+        '',
         '## Files',
         '',
         ...fields([
@@ -92,6 +108,7 @@ function failed(state) {
             ['step', state.step],
             ['reason', message(state.reason)],
             ['started', new Date(state.started).toISOString()],
+            ['timing', breakdown(state.timings)],
             [
                 'failed',
                 `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
@@ -101,6 +118,9 @@ function failed(state) {
     ].join('\n');
 }
 // ---------------------------------------------------------------------------
+/** Names are only worth reading once there are few enough to act on. */
+const MOST_NAMES = 3;
+const waiting = (pending) => pending.length <= MOST_NAMES ? pending.join(', ') : `${pending.length} documents`;
 const HEADERS = ['document', 'format', 'lines', 'sections', 'tables', 'chunks'];
 const documentTable = (sources) => grid(HEADERS, sources.map((s) => [
     s.name,

package/dist/docs/store.d.ts

@@ -1,4 +1,13 @@
 import { type Connection, type Table } from '@lancedb/lancedb';
+/**
+ * Rows per write.
+ *
+ * Arrow addresses a batch's buffers with 32-bit offsets, so one batch carrying
+ * more than 2 GiB does not error — it panics inside the reader, on a thread
+ * whose panic never reaches this one. At 3072 dimensions the vector column
+ * alone crosses that at about 175k rows, which a documentation tree reaches.
+ */
+export declare const WRITE_BATCH = 8192;
 export interface ChunkRecord {
     /** `${path}#c${ordinal}` — the fusion key, and stable across compaction */
     id: string;
@@ -37,7 +46,22 @@ export interface WriteResult {
     fts: boolean;
     vector: boolean;
 }
-export declare function writeChunks(dir: string, rows: readonly ChunkRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+/**
+ * A table built a window at a time.
+ *
+ * The corpus used to be embedded into one array and handed over in one call,
+ * which made peak memory a function of corpus size and, past 2 GiB of vectors,
+ * a panic. Rows go in as they are paid for instead, so what is resident is one
+ * window rather than all of it.
+ */
+export interface ChunkWriter {
+    add(rows: readonly ChunkRecord[], vectors: readonly Float32Array[]): Promise<void>;
+    /** Builds the indexes and closes. Throws if nothing was ever added. */
+    finish(): Promise<WriteResult>;
+    /** For a build that failed, so the connection does not outlive it. */
+    close(): void;
+}
+export declare function openChunks(dir: string): Promise<ChunkWriter>;
 export declare class ChunkStore {
     #private;
     constructor(db: Connection, table: Table);

package/dist/docs/store.js

@@ -27,28 +27,71 @@ import { lancePath } from "./files.js";
 const TABLE = 'chunks';
 /** Below this an IVF index has nothing to train on, and a flat scan is faster. */
 const VECTOR_INDEX_MIN_ROWS = 2000;
+/**
+ * Rows per write.
+ *
+ * Arrow addresses a batch's buffers with 32-bit offsets, so one batch carrying
+ * more than 2 GiB does not error — it panics inside the reader, on a thread
+ * whose panic never reaches this one. At 3072 dimensions the vector column
+ * alone crosses that at about 175k rows, which a documentation tree reaches.
+ */
+export const WRITE_BATCH = 8192;
 const KINDS = new Set(CHUNK_KINDS);
 /** What may go into a string literal in a predicate, and nothing else. */
 const SAFE = /^[\w.:/ -]+$/;
-export async function writeChunks(dir, rows, vectors) {
-    if (rows.length === 0) {
-        throw new CliError('the documents hold nothing to index', EXIT.invalid, 'they are empty, or every one of them is blank');
+export async function openChunks(dir) {
+    return new Writer(await connect(lancePath(dir)));
+}
+class Writer {
+    #db;
+    #table;
+    #rows = 0;
+    constructor(db) {
+        this.#db = db;
     }
-    const db = await connect(lancePath(dir));
-    // Every column is always populated — never null — so the Arrow schema is
-    // inferred from the first row without a declaration to keep in step.
-    const table = await db.createTable(TABLE, rows.map((row, i) => ({ ...row, vector: vectors[i] })), { mode: 'overwrite' });
-    await table.createIndex('text', { config: Index.fts() });
-    await table.createIndex('kind', { config: Index.bitmap() });
-    for (const column of ['path', 'structurePath']) {
-        await table.createIndex(column, { config: Index.btree() });
+    async add(rows, vectors) {
+        for (let from = 0; from < rows.length; from += WRITE_BATCH) {
+            const batch = rows.slice(from, from + WRITE_BATCH).map((row, i) => ({
+                ...row,
+                vector: vectors[from + i],
+            }));
+            if (this.#table) {
+                await this.#table.add(batch);
+            }
+            else {
+                // Every column is always populated — never null — so the Arrow
+                // schema is inferred from the first row without a declaration.
+                this.#table = await this.#db.createTable(TABLE, batch, { mode: 'overwrite' });
+            }
+            this.#rows += batch.length;
+        }
     }
-    const vector = rows.length >= VECTOR_INDEX_MIN_ROWS;
-    if (vector) {
-        await table.createIndex('vector');
+    async finish() {
+        const table = this.#table;
+        if (!table) {
+            throw new CliError('the documents hold nothing to index', EXIT.invalid, 'they are empty, or every one of them is blank');
+        }
+        // A writer that panicked did not reject, so a short table is the only
+        // evidence left that the rows never landed.
+        const stored = await table.countRows();
+        if (stored !== this.#rows) {
+            throw new CliError(`the table holds ${stored} of ${this.#rows} chunks`, EXIT.failed, 'the write did not finish, and a partial index answers as if it were whole');
+        }
+        await table.createIndex('text', { config: Index.fts() });
+        await table.createIndex('kind', { config: Index.bitmap() });
+        for (const column of ['path', 'structurePath']) {
+            await table.createIndex(column, { config: Index.btree() });
+        }
+        const vector = this.#rows >= VECTOR_INDEX_MIN_ROWS;
+        if (vector) {
+            await table.createIndex('vector');
+        }
+        this.#db.close();
+        return { rows: this.#rows, fts: true, vector };
+    }
+    close() {
+        this.#db.close();
     }
-    db.close();
-    return { rows: rows.length, fts: true, vector };
 }
 export class ChunkStore {
     #db;

package/dist/schema/build.d.ts

@@ -1,4 +1,5 @@
 import type { Embedder } from '@zenera/neo';
+import { type PhaseTiming } from '../common/progress.ts';
 import { type EntityRecord } from './entities.ts';
 import { type Counts, type Manifest, type SourceRecord } from './files.ts';
 export interface BuildOptions {
@@ -9,10 +10,19 @@ export interface BuildOptions {
     embeddingRef?: string;
     /** told the manifest, so a store can say what wrote it */
     indexer: string;
-    /** texts sent to the embedder at once */
-    batch?: number;
     /** keep a bundled copy of each document in the index. On by default. */
     sources?: boolean;
+    /**
+     * The width asked of the embedder, when one was asked for. Part of the cache
+     * key, because a truncated vector is a different vector; left undefined when
+     * nobody asked, because that is a different key again from asking for the
+     * number the model would have chosen anyway.
+     */
+    dimensions?: number;
+    /** reuse vectors this machine already has; on by default */
+    cache?: boolean;
+    /** keep them somewhere other than the shared store */
+    cacheDir?: string;
     signal?: AbortSignal;
     /** what the documents turned out to hold, before a vector has been paid for */
     onRead?: (summary: BuildSummary) => void;
@@ -25,6 +35,10 @@ export interface BuildSummary {
 export interface BuildResult {
     manifest: Manifest;
     entities: EntityRecord[];
+    /** what each phase cost, so a slow build can say which part was slow */
+    timings: readonly PhaseTiming[];
+    /** vectors that came out of the shared cache instead of being paid for again */
+    reused: number;
 }
 export declare function buildIndex(options: BuildOptions): Promise<BuildResult>;
 //# sourceMappingURL=build.d.ts.map

package/dist/schema/build.js

@@ -1,11 +1,11 @@
+import { embedStream, NO_CACHE, openCache } from "../common/cache.js";
 import { beginBuild } from "../common/progress.js";
 import { toEntities } from "./entities.js";
 import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
 import { buildGraph } from "./graph.js";
 import { PHASES, SCHEMA_REPORT } from "./readme.js";
 import { loadSpecs } from "./spec.js";
-import { writeStore } from "./store.js";
-const DEFAULT_BATCH = 96;
+import { openStore } from "./store.js";
 export async function buildIndex(options) {
     const journal = beginBuild({
         dir: options.out,
@@ -15,6 +15,15 @@ export async function buildIndex(options) {
         phases: PHASES,
         report: SCHEMA_REPORT,
     });
+    const ref = options.embeddingRef ?? options.embedder.id;
+    const cache = options.cache === false
+        ? NO_CACHE
+        : openCache(options.embedder, {
+            ref,
+            dir: options.cacheDir,
+            dimensions: options.dimensions,
+        });
+    let writer;
     try {
         const corpus = await loadSpecs(options.files);
         journal.phase('graph');
@@ -33,18 +42,20 @@ export async function buildIndex(options) {
         journal.read(summary.counts, summary.counts.entities);
         options.onRead?.(summary);
         journal.phase('embedding');
-        const vectors = await embedAll(entities, options, journal);
+        writer = await openStore(options.out);
+        const dimensions = await embedAll(entities, options, journal, cache, writer);
         journal.phase('writing');
-        const written = await writeStore(options.out, entities, vectors);
+        const written = await writer.finish();
         const manifest = {
             version: INDEX_VERSION,
             kind: 'schema',
             createdAt: new Date().toISOString(),
             indexer: options.indexer,
             embedding: {
-                ref: options.embeddingRef ?? options.embedder.id,
+                ref,
                 id: options.embedder.id,
-                dimensions: vectors[0]?.length ?? 0,
+                dimensions,
+                requested: options.dimensions,
             },
             sources: summary.sources,
             counts: summary.counts,
@@ -57,32 +68,33 @@ export async function buildIndex(options) {
             operations: corpus.operations,
             documents: keep ? corpus.documents : {},
         });
+        cache.commit();
         journal.finish(manifest);
-        return { manifest, entities };
+        return { manifest, entities, timings: journal.timings, reused: cache.hits };
     }
     catch (err) {
+        writer?.close();
+        cache.abandon();
         journal.fail(err);
         throw err;
     }
 }
-async function embedAll(entities, options, journal) {
-    const size = options.batch ?? DEFAULT_BATCH;
-    const out = [];
-    for (let at = 0; at < entities.length; at += size) {
-        const slice = entities.slice(at, at + size);
-        const response = await options.embedder.embed({
-            input: slice.map((e) => e.text),
-            taskType: 'document',
-            signal: options.signal,
-        });
-        if (response.vectors.length !== slice.length) {
-            throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
-        }
-        out.push(...response.vectors.map((v) => Float32Array.from(v)));
-        journal.progress(out.length, entities.length);
-        options.onProgress?.(out.length, entities.length);
-    }
-    return out;
+async function embedAll(entities, options, journal, cache, writer) {
+    // A window at a time. How many texts fit in a request, and how many
+    // requests may be in flight, are still the embedder's to answer — it knows
+    // the model's caps and it is the one that sees the 429s.
+    return embedStream({
+        embedder: options.embedder,
+        cache,
+        records: entities,
+        textOf: (entity) => entity.text,
+        signal: options.signal,
+        onProgress: (done, total) => {
+            journal.progress(done, total);
+            options.onProgress?.(done, total);
+        },
+        onWindow: (window, vectors) => writer.add(window, vectors),
+    });
 }
 /**
  * Counted off the entities rather than the corpus, so a schema the document

package/dist/schema/command.js

@@ -1,9 +1,10 @@
-import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, table, usageError, write, } from '@zenera/cli/lib';
+import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
 import { relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
 import { assertSameEmbedding } from "../common/manifest.js";
 import { isGlob, loose, matcher, PatternError, wildcard } from "../common/match.js";
+import { breakdown } from "../common/prose.js";
 import { buildIndex } from "./build.js";
 import { openIndex, readManifest, readSource, SCHEMA_INDEX } from "./files.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
@@ -68,11 +69,17 @@ export const command = {
                 '  -o, --out <dir>',
                 dim(`Where the index goes. Default ${DEFAULT_DIR}, or ${DIR_ENV}.`),
             ],
+            ['  --batch <n>', dim("Texts per embedding request. Default: the model's own cap.")],
             [
-                '  --batch <n>',
-                dim('Texts per embedding request, and how often progress prints. Default 96.'),
+                '  --dimensions <n>',
+                dim("Narrower vectors, if the model allows it. Default: the model's own width."),
             ],
             ['  --no-sources', dim('Do not keep a copy of each document in the index.')],
+            [
+                '  --no-cache',
+                dim('Embed everything again, ignoring vectors this machine already has.'),
+            ],
+            ['  --cache-dir <dir>', dim('Keep the vectors somewhere other than the shared cache.')],
         ]),
         '',
         'Search terms (repeatable)',
@@ -189,34 +196,48 @@ async function index(args, ctx) {
         out: { type: 'string', short: 'o' },
         embedding: { type: 'string' },
         batch: { type: 'string' },
+        dimensions: { type: 'string' },
         'no-sources': { type: 'boolean' },
+        'no-cache': { type: 'boolean' },
+        'cache-dir': { type: 'string' },
         quiet: { type: 'boolean' },
     }, INDEX_USAGE);
     if (positionals.length === 0) {
         throw usageError('no document given', INDEX_USAGE);
     }
     const out = outputDir(ctx.cwd, values.out, SCHEMA_INDEX);
+    const cacheDir = values['cache-dir'] ? resolve(ctx.cwd, values['cache-dir']) : paths.cache();
     const loud = !values.quiet && !ctx.json;
-    const chosen = await resolveEmbedder(values.embedding);
+    // Undefined when unasked, all the way to the cache key: a width nobody
+    // named is not the same key as the width the model happens to default to,
+    // and resolving it here would miss every vector already paid for.
+    const dimensions = values.dimensions ? count(values.dimensions, '--dimensions') : undefined;
+    const chosen = await resolveEmbedder(values.embedding, {
+        maxBatch: values.batch ? count(values.batch, '--batch') : undefined,
+        dimensions,
+    });
     const started = Date.now();
-    const { manifest } = await buildIndex({
+    const { manifest, timings, reused } = await buildIndex({
         files: positionals.map((file) => resolve(ctx.cwd, file)),
         out,
         embedder: chosen,
         embeddingRef: values.embedding,
         indexer: 'zenera-rag',
-        batch: values.batch ? count(values.batch, '--batch') : undefined,
         sources: !values['no-sources'],
+        dimensions,
+        cache: !values['no-cache'],
+        cacheDir: values['cache-dir'] ? cacheDir : undefined,
         onRead: loud
             ? (summary) => {
                 printSources(summary.sources);
-                // The first batch can take a while and says nothing while it
-                // does; this is the line that makes that a wait, not a hang.
+                // Embedding is one call now, and a long one; this is the line
+                // that makes the wait before the first progress report a wait
+                // rather than a hang.
                 note(dim(`  embedding ${summary.counts.entities} entities with ${chosen.id} …`));
             }
             : undefined,
         onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
+            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
             : undefined,
     });
     if (ctx.json) {
@@ -229,6 +250,9 @@ async function index(args, ctx) {
     write(out);
     note(`  wrote ${bold(String(manifest.counts.entities))} entities to ${bold(out)}, ` +
         `embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
+    note(dim(`  ${breakdown(timings)}`));
+    note(dim(`  reused ${reused}/${manifest.counts.entities} vectors` +
+        `${values['no-cache'] ? ' (--no-cache)' : ` from ${cacheDir}`}`));
     const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
     note(dim(`  search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
 }
@@ -315,7 +339,9 @@ async function search(args, ctx) {
     const manifest = await readManifest(dir);
     const ref = values.embedding ?? manifest.embedding.ref;
     assertSameEmbedding(manifest, ref);
-    const index = await SchemaIndex.open(dir, await resolveEmbedder(ref));
+    // A query has to be asked at the width the entities were written at, and
+    // the ref alone does not say what that was.
+    const index = await SchemaIndex.open(dir, await resolveEmbedder(ref, { dimensions: manifest.embedding.requested }));
     try {
         if (values.interactive) {
             await repl(index, query, { format, ...options });

package/dist/schema/readme.js

@@ -1,6 +1,6 @@
 import { basename } from 'node:path';
 import { INTERVAL_MS } from "../common/progress.js";
-import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+import { breakdown, duration, fields, grid, message, plural, searched } from "../common/prose.js";
 export const PHASES = {
     reading: 'reading the documents',
     graph: 'building the graph',
@@ -22,9 +22,10 @@ function building(state) {
         rows.push(['found', entities(state.summary)]);
     }
     if (state.total > 0) {
-        const percent = Math.round((state.done / state.total) * 100);
+        const percent = Math.floor((state.done / state.total) * 100);
         rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
     }
+    rows.push(['timing', breakdown(state.timings)]);
     rows.push(['updated', new Date(state.now).toISOString()]);
     return [
         '# Schema index — being built',
@@ -60,6 +61,8 @@ function complete(state) {
         `${entities(manifest.counts)},`,
         `${searched(manifest.indexes)}.`,
         '',
+        `Time: ${breakdown(state.timings)}.`,
+        '',
         '## Files',
         '',
         ...fields([
@@ -97,6 +100,7 @@ function failed(state) {
             ['step', state.step],
             ['reason', message(state.reason)],
             ['started', new Date(state.started).toISOString()],
+            ['timing', breakdown(state.timings)],
             [
                 'failed',
                 `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,

package/dist/schema/store.d.ts

@@ -18,7 +18,15 @@ export interface WriteResult {
     fts: boolean;
     vector: boolean;
 }
-export declare function writeStore(dir: string, rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+/** A table built a window at a time, so peak memory is not a function of size. */
+export interface EntityWriter {
+    add(rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<void>;
+    /** Builds the indexes and closes. Throws if nothing was ever added. */
+    finish(): Promise<WriteResult>;
+    /** For a build that failed, so the connection does not outlive it. */
+    close(): void;
+}
+export declare function openStore(dir: string): Promise<EntityWriter>;
 export declare class EntityStore {
     #private;
     constructor(db: Connection, table: Table, sources?: readonly string[]);