@zenera/rag 1.1.10 → 1.1.11

package/README.md

@@ -376,6 +376,47 @@ The copies are a record, not an input: rebuilding reads the files you name, not
 the ones in `sources/`. A **document** index has no `--no-sources`, because
 there the copies are what every quoted line is read from.
 
+**Vectors are paid for once.** Embedding is the whole cost of a build — the
+reading and the writing are milliseconds, the round trips are minutes, and they
+are the only part anyone is billed for. Almost none of it is new work: editing
+one paragraph re-embeds every other one unchanged, and a build killed at 90%
+starts again from nothing. So the vectors are kept in this machine's shared
+cache, `~/.zenera/neo/cache/vectors/`, keyed by a hash of the exact text that
+produced them together with the model that made them. Re-indexing an unchanged
+corpus embeds nothing; re-indexing after an edit pays for the chunks that
+changed.
+
+The store is the machine's rather than the index's, which is the point: the same
+corpus indexed into a second directory costs nothing the second time, and two
+projects that quote the same handbook pay for it once between them.
+
+The key is the text, never the chunk's position — inserting a sentence at the
+top of a document shifts every later chunk's ordinal without changing a word of
+it. Two documents that share a paragraph share its vector. Vectors another model
+made are not evicted, they are simply never asked for, because the model is part
+of the key. It is a cache, so every failure in it is a miss and nothing more.
+`--no-cache` ignores it; `--cache-dir <dir>` keeps the work somewhere else.
+
+Nothing is ever evicted by a build: what a corpus stops referring to is still
+work somebody paid for. Getting rid of it is [`zen cache`](../cli/README.md)'s
+job — `zen cache ls` to see what is there, `zen cache prune --older-than 30d` to
+drop what has gone unread.
+
+**Parsing is spread across cores, and also cached.** With the vectors cached,
+parsing is what is left: reading a document, cutting it into chunks, and
+recording its shape. It is CPU-bound and every document is independent of every
+other, so a **document** index does it on a pool of worker threads — one per
+core, less the one running the build. The pool is skipped for a handful of
+documents, where starting the threads costs more than it saves.
+
+The result is cached the same way the vectors are, under
+`~/.zenera/neo/cache/docs-parse/`. The key covers the file's bytes, the name
+stamped on its chunks, and every setting that decides where a chunk ends, so
+changing `--chunk-tokens` misses on all of it and editing one file misses on one
+file. Both caches are governed by the same `--no-cache` and the same
+`--cache-dir`, and both fall back to doing the work when anything about them
+looks wrong.
+
 ## Notes
 
 - Documents are **bundled, not dereferenced**: `#/components/schemas/User` is

package/dist/common/cache.d.ts

@@ -0,0 +1,66 @@
+import type { Embedder } from '@zenera/neo';
+export declare const VECTOR_KIND = "vectors";
+export interface VectorCache {
+    /** the vector this text already has, if it has one */
+    get(text: string): Float32Array | undefined;
+    /** keeps a vector for next time */
+    put(text: string, vector: Float32Array): void;
+    /** says the entries this build read are still wanted, so age means unused */
+    commit(): void;
+    /** for a build that failed; entries land as they are paid for, so nothing unwinds */
+    abandon(): void;
+    readonly hits: number;
+}
+/** A cache that remembers nothing, for `--no-cache`. */
+export declare const NO_CACHE: VectorCache;
+export interface VectorCacheOptions {
+    /** the reference as it was typed, which is part of what the vectors mean */
+    ref: string;
+    /** somewhere other than the shared store */
+    dir?: string;
+    /** when the caller asked for a width the model does not default to */
+    dimensions?: number;
+}
+export declare function openCache(embedder: Embedder, options: VectorCacheOptions): VectorCache;
+export interface CachedEmbedOptions {
+    embedder: Embedder;
+    cache: VectorCache;
+    texts: readonly string[];
+    signal?: AbortSignal;
+    onProgress?: (done: number, total: number) => void;
+}
+/**
+ * Embeds only what the cache does not already have, and hands each request's
+ * answer to the cache as it lands rather than at the end, so a build killed
+ * half way keeps the half it paid for.
+ *
+ * Identical texts are embedded once. A corpus repeats itself more than it looks
+ * like it does — shared boilerplate, a table copied between two documents — and
+ * a duplicate is a whole vector's worth of request for an answer already held.
+ */
+export declare function embedCached(options: CachedEmbedOptions): Promise<Float32Array[]>;
+export interface EmbedStreamOptions<R> {
+    embedder: Embedder;
+    cache: VectorCache;
+    records: readonly R[];
+    textOf: (record: R) => string;
+    window?: number;
+    signal?: AbortSignal;
+    /** counted over every record, not over the window being worked on */
+    onProgress?: (done: number, total: number) => void;
+    onWindow: (records: readonly R[], vectors: readonly Float32Array[]) => Promise<void>;
+}
+/**
+ * Embeds in windows and hands each one straight to whatever stores it, so that
+ * peak memory is the window rather than the corpus. A corpus of 200k chunks at
+ * 3072 dimensions held about 12 GB of vectors this way round; it now holds
+ * whatever one window is, however large the corpus gets.
+ *
+ * Duplicate texts spanning two windows still cost one embedding, because the
+ * first window has already written them to the cache by the time the second
+ * asks. Under `--no-cache` they cost two, which is what `--no-cache` means.
+ *
+ * Returns the width the model answered with, for the manifest.
+ */
+export declare function embedStream<R>(options: EmbedStreamOptions<R>): Promise<number>;
+//# sourceMappingURL=cache.d.ts.map

package/dist/common/cache.js

@@ -0,0 +1,172 @@
+import { Cache, cacheKey } from '@zenera/cli/lib';
+// ---------------------------------------------------------------------------
+// Not paying twice for the same vector
+//
+// Embedding is the whole cost of a build: the reading and the writing are
+// seconds, the round trips are minutes and the only part anyone is billed for.
+// And almost none of it is new work. Re-indexing a corpus after editing one
+// paragraph re-embeds every other paragraph unchanged, and a build killed at
+// 90% starts again from nothing.
+//
+// The vectors live in the machine's shared cache, so two indexes built from the
+// same corpus — or one corpus indexed twice into different directories — pay
+// for it once between them.
+//
+// The key is the *text*, together with the model that would embed it. Never the
+// chunk's position: inserting a sentence at the top of a document shifts every
+// later chunk's ordinal without changing a word of it, and an ordinal key would
+// miss all of them. Two documents that happen to share a paragraph share its
+// vector; a different model shares nothing, because it asks a different
+// question and so has a different key. That is also why there is nothing here
+// that invalidates anything.
+//
+// It is a cache, so every error is a miss, and the shared store guarantees that
+// much on its own: nothing in this file can fail a build.
+// ---------------------------------------------------------------------------
+export const VECTOR_KIND = 'vectors';
+/** Bumped only if what a key means changes; the model is already in the key. */
+const VECTOR_VERSION = 'v1';
+/** A cache that remembers nothing, for `--no-cache`. */
+export const NO_CACHE = {
+    get: () => undefined,
+    put: () => { },
+    commit: () => { },
+    abandon: () => { },
+    hits: 0,
+};
+export function openCache(embedder, options) {
+    return new StoredVectors(new Cache(VECTOR_KIND, { dir: options.dir }), embedder, options);
+}
+class StoredVectors {
+    #store;
+    #prefix;
+    constructor(store, embedder, options) {
+        this.#store = store;
+        this.#prefix = [VECTOR_VERSION, options.ref, embedder.id, options.dimensions];
+    }
+    get hits() {
+        return this.#store.hits;
+    }
+    /** Everything the vector is a function of, ending with the text itself. */
+    #key(text) {
+        return cacheKey(...this.#prefix, text);
+    }
+    get(text) {
+        const found = this.#store.get(this.#key(text));
+        if (typeof found !== 'string' || found.length === 0) {
+            return undefined;
+        }
+        const bytes = Buffer.from(found, 'base64');
+        if (bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) {
+            return undefined;
+        }
+        // Buffer.from can land at any offset in the shared pool, and a
+        // Float32Array needs a multiple of four. Slicing copies to its own.
+        return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
+    }
+    put(text, vector) {
+        if (vector.length > 0) {
+            const bytes = new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
+            this.#store.put(this.#key(text), Buffer.from(bytes).toString('base64'));
+        }
+    }
+    commit() {
+        this.#store.commit();
+    }
+    abandon() { }
+}
+/**
+ * Embeds only what the cache does not already have, and hands each request's
+ * answer to the cache as it lands rather than at the end, so a build killed
+ * half way keeps the half it paid for.
+ *
+ * Identical texts are embedded once. A corpus repeats itself more than it looks
+ * like it does — shared boilerplate, a table copied between two documents — and
+ * a duplicate is a whole vector's worth of request for an answer already held.
+ */
+export async function embedCached(options) {
+    const { cache, texts } = options;
+    const vectors = new Array(texts.length);
+    const wanted = new Map();
+    for (const [at, text] of texts.entries()) {
+        const hit = cache.get(text);
+        if (hit) {
+            vectors[at] = hit;
+            continue;
+        }
+        const waiting = wanted.get(text);
+        if (waiting) {
+            waiting.push(at);
+        }
+        else {
+            wanted.set(text, [at]);
+        }
+    }
+    const input = [...wanted.keys()];
+    const known = texts.length - [...wanted.values()].reduce((n, at) => n + at.length, 0);
+    if (input.length > 0) {
+        const response = await options.embedder.embed({
+            input,
+            taskType: 'document',
+            signal: options.signal,
+            onSlice: (at, slice) => {
+                for (const [i, vector] of slice.entries()) {
+                    cache.put(input[at + i], Float32Array.from(vector));
+                }
+            },
+            onProgress: (done) => options.onProgress?.(known + done, texts.length),
+        });
+        for (const [i, vector] of response.vectors.entries()) {
+            const shared = Float32Array.from(vector);
+            // `put` rewrites the same bytes, so what a slice already saved costs
+            // nothing here. Not every embedder reports slices, and the cache
+            // cannot depend on it.
+            cache.put(input[i], shared);
+            for (const at of wanted.get(input[i])) {
+                vectors[at] = shared;
+            }
+        }
+    }
+    options.onProgress?.(texts.length, texts.length);
+    return vectors;
+}
+/**
+ * How many records are embedded, written and let go of before the next are
+ * looked at. At 3072 dimensions a window costs about 250 MB while it is in
+ * flight, and the embedder still sees enough texts at once to keep every
+ * request slot busy.
+ */
+const WINDOW = 4096;
+/**
+ * Embeds in windows and hands each one straight to whatever stores it, so that
+ * peak memory is the window rather than the corpus. A corpus of 200k chunks at
+ * 3072 dimensions held about 12 GB of vectors this way round; it now holds
+ * whatever one window is, however large the corpus gets.
+ *
+ * Duplicate texts spanning two windows still cost one embedding, because the
+ * first window has already written them to the cache by the time the second
+ * asks. Under `--no-cache` they cost two, which is what `--no-cache` means.
+ *
+ * Returns the width the model answered with, for the manifest.
+ */
+export async function embedStream(options) {
+    const size = options.window ?? WINDOW;
+    const total = options.records.length;
+    let dimensions = 0;
+    for (let at = 0; at < total; at += size) {
+        const window = options.records.slice(at, at + size);
+        const vectors = await embedCached({
+            embedder: options.embedder,
+            cache: options.cache,
+            texts: window.map(options.textOf),
+            signal: options.signal,
+            onProgress: (done) => options.onProgress?.(at + done, total),
+        });
+        if (dimensions === 0) {
+            dimensions = vectors[0]?.length ?? 0;
+        }
+        await options.onWindow(window, vectors);
+    }
+    return dimensions;
+}
+//# sourceMappingURL=cache.js.map

package/dist/common/embedder.d.ts

@@ -1,3 +1,13 @@
 import { type Embedder } from '@zenera/neo';
-export declare function resolveEmbedder(ref: string | undefined): Promise<Embedder>;
+/**
+ * What a shorthand ref cannot say. Both are ceilings on what the embedder would
+ * otherwise work out for itself from the model and from what the provider
+ * refuses, so leaving them unset is the normal case.
+ */
+export interface EmbedderTuning {
+    maxBatch?: number;
+    /** narrower vectors, where the model is trained to be truncated */
+    dimensions?: number;
+}
+export declare function resolveEmbedder(ref: string | undefined, tuning?: EmbedderTuning): Promise<Embedder>;
 //# sourceMappingURL=embedder.d.ts.map

package/dist/common/embedder.js

@@ -1,20 +1,6 @@
 import { bold, CliError, CURATED, cyan, dim, ensureHome, envNames, form, KeyStore, note, PROVIDERS, table, usageError, } from '@zenera/cli/lib';
-import { createEmbedder } from '@zenera/neo';
-// ---------------------------------------------------------------------------
-// Getting an embedder, and saying what the choices are when there is none
-//
-// The keyring is materialised here, and only here. The `zen` frame does not do
-// it, and a command that forgets to looks exactly like a machine with no key:
-// `zen key ls` shows the credential live and the command still says the
-// environment variable is not set. Real env always wins, so a shell that names
-// a key is never overridden by one on disk.
-//
-// Nothing in here is about a particular index. Which model made the vectors is
-// recorded in the manifest and enforced on every later search, so this only has
-// to turn a reference into an embedder — or, given none, into a list worth
-// choosing from.
-// ---------------------------------------------------------------------------
-export async function resolveEmbedder(ref) {
+import { createEmbedder, defaultModels } from '@zenera/neo';
+export async function resolveEmbedder(ref, tuning) {
     ensureHome();
     const keys = await KeyStore.open();
     // Asked before materialising, because materialising is what erases the
@@ -24,7 +10,10 @@ export async function resolveEmbedder(ref) {
     if (!ref) {
         throw choices(keys, fromEnv);
     }
-    return createEmbedder(ref);
+    if (tuning?.maxBatch === undefined && tuning?.dimensions === undefined) {
+        return createEmbedder(ref);
+    }
+    return createEmbedder({ ...defaultModels.parseEmbedding(ref), ...tuning });
 }
 /**
  * Well-known embedding models per provider, read off the CLI's catalog table so

package/dist/common/manifest.d.ts

@@ -12,11 +12,17 @@ export interface IndexHead {
     kind: IndexKind;
     createdAt: string;
     indexer: string;
-    /** `ref` as it was typed, `id` as the embedder answers to it */
+    /**
+     * `ref` as it was typed, `id` as the embedder answers to it, `dimensions`
+     * as the vectors actually came back. `requested` only when a width was
+     * asked for out loud: a search has to ask for the same one, and asking for
+     * the model's own default is not the same as not asking.
+     */
     embedding: {
         ref: string;
         id: string;
         dimensions: number;
+        requested?: number;
     };
     /** whether the table carries an fts index, and whether it carries a vector one */
     indexes: {

package/dist/common/progress.d.ts

@@ -2,6 +2,11 @@ export declare const LOCK_FILE = ".lock";
 export declare const README_FILE = "README.md";
 /** The floor on how often README.md is rewritten. */
 export declare const INTERVAL_MS = 5000;
+/** How long one phase took, by its key rather than its sentence. */
+export interface PhaseTiming {
+    name: string;
+    ms: number;
+}
 export interface Building<S> {
     documents: readonly string[];
     embedding: string;
@@ -9,8 +14,14 @@ export interface Building<S> {
     now: number;
     /** the current phase, as it should be said out loud */
     step: string;
+    /** the current phase's key, for a report that counts different things per phase */
+    phase: string;
     done: number;
     total: number;
+    /** what the phase is still waiting on, when it is able to say */
+    pending: readonly string[];
+    /** every phase so far; the last is the one still running */
+    timings: readonly PhaseTiming[];
     /** what the documents turned out to hold, once they have been read */
     summary: S | undefined;
 }
@@ -18,12 +29,14 @@ export interface Completed<M> {
     dir: string;
     manifest: M;
     ms: number;
+    timings: readonly PhaseTiming[];
 }
 export interface Failed {
     documents: readonly string[];
     step: string;
     reason: unknown;
     started: number;
+    timings: readonly PhaseTiming[];
 }
 /** The three states a README can be in, written by whoever knows the subject. */
 export interface Report<S, M> {
@@ -45,9 +58,11 @@ export interface BuildPlan<S, M, P extends string> {
 export interface Journal<S, M, P extends string> {
     phase(name: P): void;
     read(summary: S, total: number): void;
-    progress(done: number, total: number): void;
+    progress(done: number, total: number, pending?: readonly string[]): void;
     finish(manifest: M): void;
     fail(reason: unknown): void;
+    /** what each phase cost, for a caller that wants to say so out loud */
+    readonly timings: readonly PhaseTiming[];
 }
 /**
  * Takes the directory, or refuses it. Two builds writing one index would

package/dist/common/progress.js

@@ -46,8 +46,18 @@ export function beginBuild(plan) {
     let summary;
     let done = 0;
     let total = 0;
+    let pending = [];
     let wroteAt = 0;
     let closed = false;
+    const timings = [];
+    let phaseAt = started;
+    /** Closes the running phase off. Called on every transition, and at the end. */
+    const mark = () => {
+        const at = Date.now();
+        timings.push({ name: phase, ms: at - phaseAt });
+        phaseAt = at;
+    };
+    const sofar = () => [...timings, { name: phase, ms: Date.now() - phaseAt }];
     const write = (body) => {
         // Through a temp name, so a reader never catches half a file.
         const target = join(plan.dir, README_FILE);
@@ -62,8 +72,11 @@ export function beginBuild(plan) {
         started,
         now: Date.now(),
         step: plan.phases[phase],
+        phase,
         done,
         total,
+        pending,
+        timings: sofar(),
         summary,
     }));
     const maybe = () => {
@@ -87,7 +100,9 @@ export function beginBuild(plan) {
     report();
     return {
         phase(name) {
+            mark();
             phase = name;
+            pending = [];
             maybe();
         },
         read(seen, count) {
@@ -95,16 +110,33 @@ export function beginBuild(plan) {
             total = count;
             maybe();
         },
-        progress(at, of) {
+        progress(at, of, waiting) {
             done = at;
             total = of;
+            pending = waiting ?? [];
             maybe();
         },
         finish(manifest) {
-            close(plan.report.complete({ dir: plan.dir, manifest, ms: Date.now() - started }));
+            mark();
+            close(plan.report.complete({
+                dir: plan.dir,
+                manifest,
+                ms: Date.now() - started,
+                timings,
+            }));
         },
         fail(reason) {
-            close(plan.report.failed({ documents, step: plan.phases[phase], reason, started }));
+            mark();
+            close(plan.report.failed({
+                documents,
+                step: plan.phases[phase],
+                reason,
+                started,
+                timings,
+            }));
+        },
+        get timings() {
+            return closed ? timings : sofar();
         },
     };
 }

package/dist/common/prose.d.ts

@@ -4,6 +4,13 @@ export declare function fields(rows: readonly string[][]): string[];
 export declare function grid(headers: readonly string[], rows: readonly (readonly string[])[]): string[];
 export declare function plural(n: number, one: string, many?: string): string;
 export declare function duration(ms: number): string;
+/** Sub-second, because a phase breakdown is read to compare phases against each other. */
+export declare function span(ms: number): string;
+/** Where the time went, which is the only way to know what is worth making faster. */
+export declare const breakdown: (timings: readonly {
+    name: string;
+    ms: number;
+}[]) => string;
 /** The first line only: a stack trace in a README helps nobody. */
 export declare function message(reason: unknown): string;
 export declare function searched(indexes: {

package/dist/common/prose.js

@@ -36,6 +36,19 @@ export function duration(ms) {
         ? `${minutes}m${seconds % 60}s`
         : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
 }
+/** Sub-second, because a phase breakdown is read to compare phases against each other. */
+export function span(ms) {
+    if (ms < 1000) {
+        return `${Math.round(ms)}ms`;
+    }
+    if (ms < 60_000) {
+        return `${(ms / 1000).toFixed(1)}s`;
+    }
+    const seconds = Math.round(ms / 1000);
+    return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, '0')}s`;
+}
+/** Where the time went, which is the only way to know what is worth making faster. */
+export const breakdown = (timings) => timings.map((t) => `${t.name} ${span(t.ms)}`).join(' · ');
 /** The first line only: a stack trace in a README helps nobody. */
 export function message(reason) {
     const text = reason instanceof Error ? reason.message : String(reason);

package/dist/docs/build.d.ts

@@ -1,4 +1,5 @@
 import type { Embedder } from '@zenera/neo';
+import { type PhaseTiming } from '../common/progress.ts';
 import { type ChunkOptions } from './chunk.ts';
 import { type Counts, type DocRecord, type Manifest } from './files.ts';
 import { type Corpus } from './load.ts';
@@ -13,12 +14,23 @@ export interface BuildOptions {
     embeddingRef?: string;
     /** told the manifest, so a store can say what wrote it */
     indexer: string;
-    /** texts sent to the embedder at once */
-    batch?: number;
     chunk?: ChunkOptions;
+    /**
+     * The width asked of the embedder, when one was asked for. Part of the cache
+     * key, because a truncated vector is a different vector; left undefined when
+     * nobody asked, because that is a different key again from asking for the
+     * number the model would have chosen anyway.
+     */
+    dimensions?: number;
+    /** reuse vectors and parses this machine already has; on by default */
+    cache?: boolean;
+    /** keep them somewhere other than the shared store */
+    cacheDir?: string;
     signal?: AbortSignal;
     /** what the documents turned out to hold, before a vector has been paid for */
     onRead?: (summary: BuildSummary) => void;
+    /** documents parsed so far, and what the pool is still on */
+    onReading?: (done: number, total: number, pending: readonly string[]) => void;
     onProgress?: (done: number, total: number) => void;
 }
 export interface BuildSummary {
@@ -29,6 +41,14 @@ export interface BuildSummary {
 export interface BuildResult {
     manifest: Manifest;
     chunks: ChunkRecord[];
+    /** what each phase cost, so a slow build can say which part was slow */
+    timings: readonly PhaseTiming[];
+    /** what came out of the shared cache instead of being done again */
+    reused: Reused;
+}
+export interface Reused {
+    parses: number;
+    vectors: number;
 }
 export declare function buildIndex(options: BuildOptions): Promise<BuildResult>;
 //# sourceMappingURL=build.d.ts.map