@zenera/rag 1.1.21 → 1.1.22

package/README.md

@@ -113,6 +113,8 @@ standalone OpenAPI document.
 
 ```
 zen rag schema index <spec...>   Read the documents and write a searchable index.
+zen rag schema restore           Re-embed an index from its own copies of the documents.
+zen rag schema ready             Can this index answer? Exit 0 if it can, 3 if it cannot.
 zen rag schema search            Ask it something. --interactive for a prompt.
 zen rag schema list <what>       Every method, type or property. No ranking.
 zen rag schema grep <pattern>    Every literal match across the whole index.
@@ -306,6 +308,8 @@ in line numbers and the file can be edited from the answer.
 
 ```
 zen rag docs index <path...>     Read the documents and write a searchable index.
+zen rag docs restore             Re-embed an index from its own copies of the documents.
+zen rag docs ready               Can this index answer? Exit 0 if it can, 3 if it cannot.
 zen rag docs search [text]       Ask it something. --interactive for a prompt.
 zen rag docs list <what>         Every document, section or table. No ranking.
 zen rag docs grep <pattern>      Every matching line, with the section it sits in.
@@ -471,6 +475,17 @@ references; document indexes preserve the source material used for retrieval.
 For documents, search results are always grounded in the original text, with
 source names and line ranges available for inspection and follow-up work.
 
+Because the documents travel inside the index, it can be rebuilt from itself.
+Vector files are large and binary, so a project typically commits the index and
+git-ignores `lance/`; a fresh checkout then has an index that looks complete and
+cannot answer. `ready` reports that in an exit code, and `restore` re-embeds
+from the copies already present.
+
+```sh
+zen rag docs ready --dir assets/docs-db      # exit 0 searchable, 3 not
+zen rag docs restore --dir assets/docs-db    # re-embed from sources/
+```
+
 <details>
 <summary>Built for project workflows</summary>
 
@@ -493,6 +508,10 @@ zen rag schema index --embedding openai:text-embedding-3-small ./specs
 The selected provider stays associated with the index, ensuring search uses a
 compatible representation of its content.
 
+To change it later, `restore --embedding <ref>` re-embeds an existing index from
+its own copies and rewrites the manifest, so no source material has to be
+located again.
+
 ### Efficient rebuilds
 
 Rebuild an index as content changes without repeating work for unchanged files

package/dist/common/manifest.d.ts

@@ -1,4 +1,7 @@
 export declare const MANIFEST_FILE = "manifest.json";
+/** Where the vectors live, in both kinds of index. */
+export declare const LANCE_DIR = "lance";
+export declare const lancePath: (dir: string) => string;
 export type IndexKind = 'schema' | 'docs';
 /** How to name a kind when something has to be said about it. */
 export declare const SUBJECT: Record<IndexKind, {
@@ -53,4 +56,28 @@ export declare function readHead<T extends IndexHead>(dir: string, spec: IndexSp
  * something anyone should have to remember.
  */
 export declare function assertSameEmbedding(head: IndexHead, ref: string): void;
+export type IndexState = 'absent' | 'incomplete' | 'ready';
+interface Closable {
+    close(): void;
+}
+export interface IndexHealth<T extends IndexHead = IndexHead> {
+    dir: string;
+    state: IndexState;
+    /** absent when there is no manifest to read */
+    head?: T;
+    /** what is the matter, or what is there, in one line */
+    reason: string;
+    /** the command that would change the answer; empty when the answer is yes */
+    fix: string;
+}
+/**
+ * `open` is the subject's own store opener, so the readiness of an index is
+ * decided by the same code a search would use rather than by a second opinion
+ * about what a directory ought to contain.
+ *
+ * A manifest of the wrong kind or the wrong version still throws: those are not
+ * states of this index, they are the caller asking about a different one.
+ */
+export declare function inspectIndex<T extends IndexHead>(dir: string, spec: IndexSpec, open: (dir: string) => Promise<Closable>): Promise<IndexHealth<T>>;
+export {};
 //# sourceMappingURL=manifest.d.ts.map

package/dist/common/manifest.js

@@ -1,4 +1,5 @@
 import { CliError, EXIT } from '@zenera/cli/lib';
+import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 // ---------------------------------------------------------------------------
@@ -15,6 +16,9 @@ import { join } from 'node:path';
 // worth spending a field on.
 // ---------------------------------------------------------------------------
 export const MANIFEST_FILE = 'manifest.json';
+/** Where the vectors live, in both kinds of index. */
+export const LANCE_DIR = 'lance';
+export const lancePath = (dir) => join(dir, LANCE_DIR);
 /** How to name a kind when something has to be said about it. */
 export const SUBJECT = {
     schema: { label: 'a schema index', command: 'zen rag schema' },
@@ -56,7 +60,49 @@ export async function readHead(dir, spec) {
  */
 export function assertSameEmbedding(head, ref) {
     if (ref !== head.embedding.ref && ref !== head.embedding.id) {
-        throw new CliError(`this index was built with ${head.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${head.embedding.ref}, or rebuild it`);
+        throw new CliError(`this index was built with ${head.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${head.embedding.ref}, or move it to ${ref} with ` +
+            `\`${SUBJECT[head.kind ?? 'schema'].command} restore --embedding ${ref}\``);
     }
 }
+/**
+ * `open` is the subject's own store opener, so the readiness of an index is
+ * decided by the same code a search would use rather than by a second opinion
+ * about what a directory ought to contain.
+ *
+ * A manifest of the wrong kind or the wrong version still throws: those are not
+ * states of this index, they are the caller asking about a different one.
+ */
+export async function inspectIndex(dir, spec, open) {
+    const { command } = SUBJECT[spec.kind];
+    if (!existsSync(join(dir, MANIFEST_FILE))) {
+        return {
+            dir,
+            state: 'absent',
+            reason: `${dir} does not hold an index`,
+            fix: `build one with \`${command} index\``,
+        };
+    }
+    const head = await readHead(dir, spec);
+    let store;
+    try {
+        store = await open(dir);
+    }
+    catch {
+        return {
+            dir,
+            state: 'incomplete',
+            head,
+            reason: `${dir} has no vectors — \`${LANCE_DIR}/\` is missing or unreadable`,
+            fix: `re-embed them with \`${command} restore --dir ${dir}\``,
+        };
+    }
+    store.close();
+    return {
+        dir,
+        state: 'ready',
+        head,
+        reason: `${dir} is searchable — ${head.embedding.ref} (${head.embedding.dimensions}d)`,
+        fix: '',
+    };
+}
 //# sourceMappingURL=manifest.js.map

package/dist/common/prose.d.ts

@@ -11,6 +11,13 @@ export declare const breakdown: (timings: readonly {
     name: string;
     ms: number;
 }[]) => string;
+/**
+ * The silence after the last embedded row. Building the full-text and vector
+ * indexes is one call that reports nothing until it returns, and on a large
+ * corpus that is minutes — so it is announced, or it reads as a hang. Small
+ * indexes finish before the line is read, and are not promised a wait.
+ */
+export declare function indexing(rows: number, unit: string): string[];
 /** The first line only: a stack trace in a README helps nobody. */
 export declare function message(reason: unknown): string;
 export declare function searched(indexes: {

package/dist/common/prose.js

@@ -49,6 +49,22 @@ export function span(ms) {
 }
 /** Where the time went, which is the only way to know what is worth making faster. */
 export const breakdown = (timings) => timings.map((t) => `${t.name} ${span(t.ms)}`).join(' · ');
+/**
+ * The silence after the last embedded row. Building the full-text and vector
+ * indexes is one call that reports nothing until it returns, and on a large
+ * corpus that is minutes — so it is announced, or it reads as a hang. Small
+ * indexes finish before the line is read, and are not promised a wait.
+ */
+export function indexing(rows, unit) {
+    const lines = [`indexing ${rows} ${unit} for search: full text, then vectors …`];
+    if (rows >= 50_000) {
+        lines.push('this step reports no progress, and at this size it takes several minutes');
+    }
+    else if (rows >= 5_000) {
+        lines.push('this step reports no progress until it is done');
+    }
+    return lines;
+}
 /** The first line only: a stack trace in a README helps nobody. */
 export function message(reason) {
     const text = reason instanceof Error ? reason.message : String(reason);

package/dist/common/restore.d.ts

@@ -0,0 +1,30 @@
+import { type Context } from '@zenera/cli/lib';
+import type { IndexHead, IndexHealth } from './manifest.ts';
+export interface Staging {
+    /** where the rebuild is written */
+    dir: string;
+    /** the old index, kept until the new one is in place */
+    previous: string;
+}
+export declare function stagingFor(dir: string): Staging;
+/**
+ * Two renames, because a rename cannot replace a directory that has anything in
+ * it. If the second fails the first is undone, so the window in which neither
+ * directory is the index is as short as two syscalls and never ends with one
+ * missing.
+ */
+export declare function swapIn(dir: string, staging: Staging): Promise<void>;
+/**
+ * The answer to `ready` is the exit code, so that a setup step can be written
+ * as `if zen rag docs ready -d "$OUT" --quiet; then …` and never have to parse
+ * anything. With `--json` the same answer is a document, and it is printed for
+ * both outcomes: a caller that asked for JSON gets JSON whatever the news is.
+ */
+export declare function reportReady(health: IndexHealth, ctx: Context, quiet?: boolean): void;
+/**
+ * What a restore is about to do, said before it starts, because the two things
+ * worth objecting to — a different model, and chunking that was never recorded
+ * — are both cheap to say and expensive to discover afterwards.
+ */
+export declare function announce(head: IndexHead, ref: string, entities: string): void;
+//# sourceMappingURL=restore.d.ts.map

package/dist/common/restore.js

@@ -0,0 +1,65 @@
+import { CliError, dim, EXIT, json, note } from '@zenera/cli/lib';
+import { rename, rm } from 'node:fs/promises';
+import { basename, dirname, join } from 'node:path';
+export function stagingFor(dir) {
+    const parent = dirname(dir);
+    const name = basename(dir);
+    return {
+        dir: join(parent, `.${name}.restoring`),
+        previous: join(parent, `.${name}.previous`),
+    };
+}
+/**
+ * Two renames, because a rename cannot replace a directory that has anything in
+ * it. If the second fails the first is undone, so the window in which neither
+ * directory is the index is as short as two syscalls and never ends with one
+ * missing.
+ */
+export async function swapIn(dir, staging) {
+    await rm(staging.previous, { recursive: true, force: true });
+    await rename(dir, staging.previous);
+    try {
+        await rename(staging.dir, dir);
+    }
+    catch (err) {
+        await rename(staging.previous, dir).catch(() => undefined);
+        throw err;
+    }
+    await rm(staging.previous, { recursive: true, force: true });
+}
+/**
+ * The answer to `ready` is the exit code, so that a setup step can be written
+ * as `if zen rag docs ready -d "$OUT" --quiet; then …` and never have to parse
+ * anything. With `--json` the same answer is a document, and it is printed for
+ * both outcomes: a caller that asked for JSON gets JSON whatever the news is.
+ */
+export function reportReady(health, ctx, quiet = false) {
+    if (ctx.json) {
+        json({
+            ready: health.state === 'ready',
+            dir: health.dir,
+            state: health.state,
+            embedding: health.head?.embedding ?? null,
+            reason: health.reason,
+            ...(health.fix ? { fix: health.fix } : {}),
+        });
+    }
+    if (health.state !== 'ready') {
+        throw new CliError(health.reason, EXIT.invalid, health.fix);
+    }
+    if (!quiet && !ctx.json) {
+        note(dim(`  ${health.reason}`));
+    }
+}
+/**
+ * What a restore is about to do, said before it starts, because the two things
+ * worth objecting to — a different model, and chunking that was never recorded
+ * — are both cheap to say and expensive to discover afterwards.
+ */
+export function announce(head, ref, entities) {
+    if (ref !== head.embedding.ref) {
+        note(dim(`  embedder ${head.embedding.ref} → ${ref}; the manifest will be rewritten`));
+    }
+    note(dim(`  re-embedding ${entities} with ${ref} …`));
+}
+//# sourceMappingURL=restore.js.map

package/dist/docs/build.d.ts

@@ -15,6 +15,8 @@ export interface BuildOptions {
     /** told the manifest, so a store can say what wrote it */
     indexer: string;
     chunk?: ChunkOptions;
+    /** name the documents relative to this, rather than to their common root */
+    root?: string;
     /**
      * The width asked of the embedder, when one was asked for. Part of the cache
      * key, because a truncated vector is a different vector; left undefined when
@@ -32,6 +34,8 @@ export interface BuildOptions {
     /** documents parsed so far, and what the pool is still on */
     onReading?: (done: number, total: number, pending: readonly string[]) => void;
     onProgress?: (done: number, total: number) => void;
+    /** every vector is in and the search indexes are about to be built, silently */
+    onWriting?: (rows: number) => void;
 }
 export interface BuildSummary {
     sources: DocRecord[];

package/dist/docs/build.js

@@ -26,6 +26,7 @@ export async function buildIndex(options) {
     try {
         const corpus = await loadDocuments(options.files, options.cwd, {
             chunk: options.chunk,
+            root: options.root,
             cache: options.cache !== false,
             cacheDir: options.cacheDir,
             onProgress: (done, total, pending) => {
@@ -60,6 +61,7 @@ export async function buildIndex(options) {
         writer = await openChunks(options.out);
         const dimensions = await embedAll(chunks, options, journal, cache, writer);
         journal.phase('writing');
+        options.onWriting?.(chunks.length);
         const written = await writer.finish();
         const manifest = {
             version: INDEX_VERSION,
@@ -74,6 +76,7 @@ export async function buildIndex(options) {
             },
             sources,
             counts,
+            chunk: settingsOf(options.chunk),
             indexes: { fts: written.fts, vector: written.vector },
         };
         const outline = { files: corpus.docs.map((doc) => doc.outline) };
@@ -95,6 +98,25 @@ export async function buildIndex(options) {
         throw err;
     }
 }
+/**
+ * What the manifest keeps of the chunk settings: the numbers, and only the ones
+ * actually given. A `tokenCount` function cannot be written down, and a build
+ * handed one is a build a restore cannot reproduce from the manifest alone —
+ * which the omission says, rather than a recorded default pretending otherwise.
+ */
+function settingsOf(chunk) {
+    if (!chunk) {
+        return undefined;
+    }
+    const { chunkTokens, minChunkTokens, maxChunkTokens, tableSliceTokens } = chunk;
+    const settings = {
+        ...(chunkTokens === undefined ? {} : { chunkTokens }),
+        ...(minChunkTokens === undefined ? {} : { minChunkTokens }),
+        ...(maxChunkTokens === undefined ? {} : { maxChunkTokens }),
+        ...(tableSliceTokens === undefined ? {} : { tableSliceTokens }),
+    };
+    return Object.keys(settings).length > 0 ? settings : undefined;
+}
 /** One row per chunk, with the render set encoded and the document name on it. */
 function recordsOf(corpus) {
     return corpus.docs.flatMap((doc) => doc.chunks.map((chunk) => ({

package/dist/docs/command.js

@@ -1,20 +1,23 @@
 import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
-import { relative, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
-import { assertSameEmbedding } from "../common/manifest.js";
+import { assertSameEmbedding, inspectIndex } from "../common/manifest.js";
 import { PatternError } from "../common/match.js";
 import { INTERVAL_MS } from "../common/progress.js";
-import { breakdown, grid } from "../common/prose.js";
+import { breakdown, grid, indexing } from "../common/prose.js";
+import { announce, reportReady, stagingFor, swapIn } from "../common/restore.js";
 import { assemble, DEFAULT_MAX_LINES } from "./assemble.js";
 import { buildIndex } from "./build.js";
 import { CHUNK_KINDS } from "./chunk.js";
-import { DOCS_INDEX, readManifest } from "./files.js";
+import { DOCS_INDEX, readManifest, SOURCES_DIR } from "./files.js";
 import { DOC_EXTENSIONS } from "./load.js";
 import { grepLines, listFiles, listSections, listTables, readRange, readSection, } from "./lookup.js";
 import { MATCH_HEADERS, matchRows, renderAssembly } from "./render.js";
 import { repl } from "./repl.js";
 import { DEFAULT_LIMIT, DocsIndex, legsOf, SEARCH_MODES, } from "./search.js";
+import { ChunkStore } from "./store.js";
 const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = DOCS_INDEX;
 // ---------------------------------------------------------------------------
 // zen rag docs — a pile of documents, as something to search
@@ -39,8 +42,10 @@ const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = DOCS_INDEX;
 // answer "does the word `deprecated` appear anywhere" — the honest answer is
 // every match or none, and these give it with no model and no credential.
 // ---------------------------------------------------------------------------
-const USAGE = 'zen rag docs <index|search|list|grep|show|stats> [args...]';
+const USAGE = 'zen rag docs <index|restore|ready|search|list|grep|show|stats> [args...]';
 const INDEX_USAGE = 'zen rag docs index --embedding <ref> [--out <dir>] <path...>';
+const RESTORE_USAGE = 'zen rag docs restore [--dir <dir>] [--embedding <ref>]';
+const READY_USAGE = 'zen rag docs ready [--dir <dir>] [--quiet]';
 const SEARCH_USAGE = 'zen rag docs search [--dir <dir>] [query... | --text-query <text> --vector-query <text>]';
 const LIST_USAGE = 'zen rag docs list <files|sections|tables> [--dir <dir>]';
 const GREP_USAGE = 'zen rag docs grep <pattern> [--dir <dir>]';
@@ -55,6 +60,11 @@ export const command = {
                 '  index <path...>',
                 dim('Build or refresh an index before searching changed documents.'),
             ],
+            [
+                '  restore',
+                dim('Re-embed an index from its own copies, after a clone or a model change.'),
+            ],
+            ['  ready', dim('Ask whether an index can answer. Exit 0 if it can, 3 if it cannot.')],
             [
                 '  search [text]',
                 dim('Rank relevant passages with hybrid full-text and vector search.'),
@@ -101,6 +111,29 @@ export const command = {
         dim('  Every document is copied into the index, so it stays portable and'),
         dim('  every quoted line comes from the document rather than a rebuild of it.'),
         '',
+        'Restore — the vectors, from the copies the index already holds',
+        ...table([
+            ['  -d, --dir <dir>', dim(`Which index. Found from here if unset; see ${DIR_ENV}.`)],
+            ['  --embedding <ref>', dim('Move to another embedder. Default: the one recorded.')],
+            ['  --dimensions <n>', dim('Narrower vectors. Default: the width recorded.')],
+            ['  --chunk-tokens <n>', dim('Re-chunk at this size. Default: the size recorded.')],
+            ['  --batch <n>', dim("Texts per embedding request. Default: the model's own cap.")],
+            ['  --no-cache', dim('Embed everything again, ignoring what is already kept.')],
+            ['  --cache-dir <dir>', dim('Take them from somewhere other than the shared cache.')],
+            ['  --quiet', dim('No narration.')],
+        ]),
+        '',
+        dim('  `lance/` is binary and rebuildable, so it is usually git-ignored while the'),
+        dim('  rest of the index is committed. A clone then has an index that looks built'),
+        dim(`  and cannot answer: ${cyan('ready')} says so, and this puts the vectors back`),
+        dim('  without going to find the documents again.'),
+        '',
+        dim('  It is also how an index changes embedder: --embedding re-embeds everything'),
+        dim('  and rewrites the manifest, so later searches ask for the new model.'),
+        '',
+        dim('  The rebuild is written beside the index and renamed into place, so a failed'),
+        dim('  restore leaves the old index intact — and needs room for both meanwhile.'),
+        '',
         'Search',
         ...table([
             ['  <text>', dim('What to look for. One question, not a list of terms.')],
@@ -177,6 +210,10 @@ export const command = {
         switch (name) {
             case 'index':
                 return await index(tail, ctx);
+            case 'restore':
+                return await restore(tail, ctx);
+            case 'ready':
+                return await ready(tail, ctx);
             case 'search':
                 return await search(tail, ctx);
             case 'list':
@@ -250,6 +287,7 @@ async function index(args, ctx) {
         onProgress: loud
             ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
             : undefined,
+        onWriting: loud ? (rows) => notes(indexing(rows, 'chunks').map(aside)) : undefined,
     });
     if (ctx.json) {
         json({ out, manifest });
@@ -269,6 +307,91 @@ async function index(args, ctx) {
     const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
     note(dim(`  search it: ${cyan(`zen rag docs search${where} "what you are after"`)}`));
 }
+async function restore(args, ctx) {
+    const { values } = parse(args, {
+        dir: { type: 'string', short: 'd' },
+        embedding: { type: 'string' },
+        batch: { type: 'string' },
+        dimensions: { type: 'string' },
+        'chunk-tokens': { type: 'string' },
+        'no-cache': { type: 'boolean' },
+        'cache-dir': { type: 'string' },
+        quiet: { type: 'boolean' },
+    }, RESTORE_USAGE);
+    const dir = indexDir(ctx, values.dir);
+    const manifest = await readManifest(dir);
+    const sources = join(dir, SOURCES_DIR);
+    if (!existsSync(sources)) {
+        throw new CliError(`${dir} kept no copy of its documents`, EXIT.invalid, 'build it again from the documents with `zen rag docs index`');
+    }
+    const loud = !values.quiet && !ctx.json;
+    const cacheDir = values['cache-dir'] ? resolve(ctx.cwd, values['cache-dir']) : paths.cache();
+    // Every setting defaults to what the index recorded, so a restore that is
+    // asked for nothing produces the index it already had, minus the loss.
+    const ref = values.embedding ?? manifest.embedding.ref;
+    const dimensions = values.dimensions
+        ? count(values.dimensions, '--dimensions')
+        : manifest.embedding.requested;
+    const chunk = values['chunk-tokens']
+        ? { chunkTokens: count(values['chunk-tokens'], '--chunk-tokens') }
+        : manifest.chunk;
+    if (loud && !chunk) {
+        note(dim('  this index records no chunk settings; the defaults are used'));
+    }
+    const chosen = await resolveEmbedder(ref, {
+        maxBatch: values.batch ? count(values.batch, '--batch') : undefined,
+        dimensions,
+    });
+    const started = Date.now();
+    if (loud) {
+        announce(manifest, ref, `${manifest.counts.chunks} chunks`);
+    }
+    const staging = stagingFor(dir);
+    const built = await buildIndex({
+        files: [sources],
+        cwd: dir,
+        // The names are the identity — chunk ids, --file patterns, what `show`
+        // takes — so they are taken from `sources/` rather than from wherever
+        // the documents happened to sit on the machine that indexed them.
+        root: sources,
+        out: staging.dir,
+        embedder: chosen,
+        embeddingRef: ref,
+        // The index is the same index; what rebuilt its vectors is not a new
+        // author of it.
+        indexer: manifest.indexer,
+        chunk,
+        dimensions,
+        cache: !values['no-cache'],
+        cacheDir: values['cache-dir'] ? cacheDir : undefined,
+        onProgress: loud
+            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
+            : undefined,
+        onWriting: loud ? (rows) => notes(indexing(rows, 'chunks').map(aside)) : undefined,
+    });
+    await swapIn(dir, staging);
+    if (ctx.json) {
+        json({ dir, manifest: built.manifest });
+        return;
+    }
+    note();
+    write(dir);
+    note(`  restored ${bold(String(built.manifest.counts.chunks))} chunks from ` +
+        `${bold(String(built.manifest.counts.documents))} document(s) in ${bold(dir)}, ` +
+        `embedded with ${built.manifest.embedding.ref} ` +
+        `(${built.manifest.embedding.dimensions}d)`);
+    note(dim(`  ${breakdown(built.timings)}`));
+    note(dim(`  reused ${built.reused.vectors}/${built.manifest.counts.chunks} vectors` +
+        `${values['no-cache'] ? ' (--no-cache)' : ` from ${cacheDir}`}`));
+}
+// ---------------------------------------------------------------------------
+// ready
+// ---------------------------------------------------------------------------
+async function ready(args, ctx) {
+    const { values } = parse(args, { dir: { type: 'string', short: 'd' }, quiet: { type: 'boolean' } }, READY_USAGE);
+    const health = await inspectIndex(indexDir(ctx, values.dir), DOCS_INDEX, ChunkStore.open);
+    reportReady(health, ctx, values.quiet === true);
+}
 const SOURCE_HEADERS = ['LINES', 'SECTIONS', 'TABLES', 'CHUNKS'];
 function printSources(sources) {
     // A corpus can be thousands of documents and a wall of them is not a report.
@@ -531,8 +654,9 @@ async function stats(args, ctx) {
     const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag docs stats [--dir <dir>]');
     const dir = indexDir(ctx, values.dir);
     const manifest = await readManifest(dir);
+    const health = await inspectIndex(dir, DOCS_INDEX, ChunkStore.open);
     if (ctx.json) {
-        json(manifest);
+        json({ ...manifest, state: health.state });
         return;
     }
     note(bold(dir));
@@ -544,6 +668,9 @@ async function stats(args, ctx) {
             '  indexes',
             `fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
         ],
+        // `stats` reports and never refuses, so a missing store is a row
+        // here rather than the error `search` would raise.
+        ['  vectors', health.state === 'ready' ? 'present' : dim('missing — `restore` them')],
     ]));
     printSources(manifest.sources);
     notes(table([
@@ -619,5 +746,6 @@ function notes(lines) {
         note(line);
     }
 }
+const aside = (line) => dim(`  ${line}`);
 const yes = (value) => (value ? 'yes' : 'no');
 //# sourceMappingURL=command.js.map

package/dist/docs/files.d.ts

@@ -1,11 +1,12 @@
 import { type IndexHead, type IndexSpec } from '../common/manifest.ts';
+import type { ChunkOptions } from './chunk.ts';
 import type { DocFormat } from './parse.ts';
 export declare const INDEX_VERSION = 1;
 /** How a document index is found, read, and refused. */
 export declare const DOCS_INDEX: IndexSpec;
 export declare const OUTLINE_FILE = "outline.json";
-export declare const LANCE_DIR = "lance";
 export declare const SOURCES_DIR = "sources";
+export { LANCE_DIR, lancePath } from '../common/manifest.ts';
 export interface HeadingRecord {
     /** the heading's own line, 1-based */
     line: number;
@@ -63,9 +64,17 @@ export interface Counts {
     sections: number;
     tables: number;
 }
+/**
+ * The chunk settings a build was given, so a restore can cut the documents the
+ * same way rather than silently re-chunking them. Numbers only: `tokenCount` is
+ * a function and does not survive a manifest, and absent means the defaults.
+ */
+export type ChunkSettings = Omit<ChunkOptions, 'tokenCount'>;
 export interface Manifest extends IndexHead {
     sources: DocRecord[];
     counts: Counts;
+    /** absent in indexes built before this was recorded, which means the defaults */
+    chunk?: ChunkSettings;
 }
 export interface WrittenIndex {
     manifest: Manifest;
@@ -78,7 +87,6 @@ export interface OpenDocs {
     manifest: Manifest;
     outline: Outline;
 }
-export declare const lancePath: (dir: string) => string;
 export declare function writeIndex(dir: string, index: WrittenIndex): Promise<void>;
 export declare function openIndex(dir: string): Promise<OpenDocs>;
 export declare const readManifest: (dir: string) => Promise<Manifest>;

package/dist/docs/files.js

@@ -41,9 +41,8 @@ export const DOCS_INDEX = {
     envName: 'ZEN_DOCS_DB',
 };
 export const OUTLINE_FILE = 'outline.json';
-export const LANCE_DIR = 'lance';
 export const SOURCES_DIR = 'sources';
-export const lancePath = (dir) => join(dir, LANCE_DIR);
+export { LANCE_DIR, lancePath } from "../common/manifest.js";
 export async function writeIndex(dir, index) {
     await mkdir(dir, { recursive: true });
     await writeFile(join(dir, OUTLINE_FILE), JSON.stringify(index.outline));

package/dist/docs/load.d.ts

@@ -26,6 +26,13 @@ export interface Corpus {
 }
 export interface LoadOptions {
     chunk?: ChunkOptions;
+    /**
+     * Name documents relative to this, instead of relative to the common root of
+     * whatever was found. A restore pins it to the index's own `sources/`, so a
+     * corpus that is one nested document keeps the name it was indexed under
+     * rather than being renamed to its basename by `commonRoot`.
+     */
+    root?: string;
     /** remember what documents parse to; on by default */
     cache?: boolean;
     /** keep the parses somewhere other than the shared store */

package/dist/docs/load.js

@@ -39,7 +39,7 @@ export async function loadDocuments(inputs, cwd, options = {}) {
     if (found.length === 0) {
         throw new CliError('nothing to index', EXIT.invalid, `no ${DOC_EXTENSIONS.join(', ')} files were found under ${inputs.join(', ')}`);
     }
-    const root = commonRoot(found);
+    const root = options.root ? resolve(cwd, options.root) : commonRoot(found);
     const taken = new Set();
     const chunk = options.chunk ?? {};
     const cache = options.cache === false ? NO_PARSE_CACHE : openParseCache(options.cacheDir);

package/dist/docs/store.js

@@ -107,7 +107,7 @@ export class ChunkStore {
         }
         catch {
             db.close();
-            throw new CliError(`${dir} holds no searchable table`, EXIT.invalid, 'rebuild it with `zen rag docs index`');
+            throw new CliError(`${dir} holds no searchable table`, EXIT.invalid, 'put the vectors back with `zen rag docs restore`, or rebuild it with `zen rag docs index`');
         }
     }
     /** Nearest neighbours. Every row has a vector, so nothing has to be excluded. */

package/dist/schema/build.d.ts

@@ -27,6 +27,8 @@ export interface BuildOptions {
     /** what the documents turned out to hold, before a vector has been paid for */
     onRead?: (summary: BuildSummary) => void;
     onProgress?: (done: number, total: number) => void;
+    /** every vector is in and the search indexes are about to be built, silently */
+    onWriting?: (rows: number) => void;
 }
 export interface BuildSummary {
     sources: SourceRecord[];

package/dist/schema/build.js

@@ -45,6 +45,7 @@ export async function buildIndex(options) {
         writer = await openStore(options.out);
         const dimensions = await embedAll(entities, options, journal, cache, writer);
         journal.phase('writing');
+        options.onWriting?.(entities.length);
         const written = await writer.finish();
         const manifest = {
             version: INDEX_VERSION,

package/dist/schema/command.js

@@ -1,18 +1,21 @@
 import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
-import { relative, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
-import { assertSameEmbedding } from "../common/manifest.js";
+import { assertSameEmbedding, inspectIndex } from "../common/manifest.js";
 import { isGlob, loose, matcher, PatternError, wildcard } from "../common/match.js";
-import { breakdown } from "../common/prose.js";
+import { breakdown, indexing } from "../common/prose.js";
+import { announce, reportReady, stagingFor, swapIn } from "../common/restore.js";
 import { buildIndex } from "./build.js";
-import { openIndex, readManifest, readSource, SCHEMA_INDEX } from "./files.js";
+import { openIndex, readManifest, readSource, SCHEMA_INDEX, SOURCES_DIR, } from "./files.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
 import { isFormat, present } from "./present.js";
 import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { sourceTag } from "./render.js";
 import { repl } from "./repl.js";
 import { SchemaIndex } from "./search.js";
+import { EntityStore } from "./store.js";
 import { select, stitch } from "./subgraph.js";
 import { chainOf, DEFAULT_TRACE_HOPS, traceNodes } from "./trace.js";
 const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = SCHEMA_INDEX;
@@ -37,8 +40,10 @@ const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = SCHEMA_INDEX;
 // that question is every match or none, and these three give it without asking
 // a model or a credential for permission.
 // ---------------------------------------------------------------------------
-const USAGE = 'zen rag schema <index|search|list|grep|trace|show|stats> [spec...]';
+const USAGE = 'zen rag schema <index|restore|ready|search|list|grep|trace|show|stats> [spec...]';
 const INDEX_USAGE = 'zen rag schema index --embedding <ref> [--out <dir>] <spec...>';
+const RESTORE_USAGE = 'zen rag schema restore [--dir <dir>] [--embedding <ref>]';
+const READY_USAGE = 'zen rag schema ready [--dir <dir>] [--quiet]';
 const SEARCH_USAGE = 'zen rag schema search [--dir <dir>] [query...]';
 const LIST_USAGE = 'zen rag schema list <methods|types|properties> [--dir <dir>]';
 const GREP_USAGE = 'zen rag schema grep <pattern> [--dir <dir>]';
@@ -51,6 +56,8 @@ export const command = {
         'Commands',
         ...table([
             ['  index <spec...>', dim('Read the documents and write a searchable index.')],
+            ['  restore', dim('Re-embed an index from its own copies of the documents.')],
+            ['  ready', dim('Ask whether an index can answer. Exit 0 if it can, 3 if it cannot.')],
             ['  search', dim('Ask it something. --interactive for a prompt.')],
             ['  list <what>', dim('Every method, type or property matching a pattern.')],
             ['  grep <pattern>', dim('Every literal match, ranked by nothing.')],
@@ -82,6 +89,26 @@ export const command = {
             ['  --cache-dir <dir>', dim('Keep the vectors somewhere other than the shared cache.')],
         ]),
         '',
+        'Restore — the vectors, from the copies the index already holds',
+        ...table([
+            ['  -d, --dir <dir>', dim(`Which index. Found from here if unset; see ${DIR_ENV}.`)],
+            ['  --embedding <ref>', dim('Move to another embedder. Default: the one recorded.')],
+            ['  --dimensions <n>', dim('Narrower vectors. Default: the width recorded.')],
+            ['  --batch <n>', dim("Texts per embedding request. Default: the model's own cap.")],
+            ['  --no-cache', dim('Embed everything again, ignoring what this machine has.')],
+            ['  --cache-dir <dir>', dim('Take them from somewhere other than the shared cache.')],
+            ['  --quiet', dim('No narration.')],
+        ]),
+        '',
+        dim('  `lance/` is binary and rebuildable, so it is usually git-ignored while the'),
+        dim('  rest of the index is committed. A clone then has an index that looks built'),
+        dim(`  and cannot answer: ${cyan('ready')} says so, and this puts the vectors back`),
+        dim('  without going to find the documents again. It needs an index built with'),
+        dim('  sources — one made with --no-sources has nothing to rebuild from.'),
+        '',
+        dim('  It is also how an index changes embedder: --embedding re-embeds everything'),
+        dim('  and rewrites the manifest, so later searches ask for the new model.'),
+        '',
         'Search terms (repeatable)',
         ...table([
             ['  <text>', dim('A bare phrase, the same as --all.')],
@@ -174,6 +201,10 @@ export const command = {
         switch (name) {
             case 'index':
                 return await index(tail, ctx);
+            case 'restore':
+                return await restore(tail, ctx);
+            case 'ready':
+                return await ready(tail, ctx);
             case 'search':
                 return await search(tail, ctx);
             case 'list':
@@ -239,6 +270,7 @@ async function index(args, ctx) {
         onProgress: loud
             ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
             : undefined,
+        onWriting: loud ? (rows) => notes(indexing(rows, 'entities').map(aside)) : undefined,
     });
     if (ctx.json) {
         json({ out, manifest });
@@ -256,6 +288,90 @@ async function index(args, ctx) {
     const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
     note(dim(`  search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
 }
+async function restore(args, ctx) {
+    const { values } = parse(args, {
+        dir: { type: 'string', short: 'd' },
+        embedding: { type: 'string' },
+        batch: { type: 'string' },
+        dimensions: { type: 'string' },
+        'no-cache': { type: 'boolean' },
+        'cache-dir': { type: 'string' },
+        quiet: { type: 'boolean' },
+    }, RESTORE_USAGE);
+    const dir = indexDir(ctx, values.dir);
+    const manifest = await readManifest(dir);
+    const files = bundled(dir, manifest);
+    const loud = !values.quiet && !ctx.json;
+    const cacheDir = values['cache-dir'] ? resolve(ctx.cwd, values['cache-dir']) : paths.cache();
+    // Every setting defaults to what the index recorded, so a restore that is
+    // asked for nothing produces the index it already had, minus the loss.
+    const ref = values.embedding ?? manifest.embedding.ref;
+    const dimensions = values.dimensions
+        ? count(values.dimensions, '--dimensions')
+        : manifest.embedding.requested;
+    const chosen = await resolveEmbedder(ref, {
+        maxBatch: values.batch ? count(values.batch, '--batch') : undefined,
+        dimensions,
+    });
+    const started = Date.now();
+    if (loud) {
+        announce(manifest, ref, `${manifest.counts.entities} entities`);
+    }
+    const staging = stagingFor(dir);
+    const built = await buildIndex({
+        files,
+        out: staging.dir,
+        embedder: chosen,
+        embeddingRef: ref,
+        // The index is the same index; what rebuilt its vectors is not a new
+        // author of it.
+        indexer: manifest.indexer,
+        sources: true,
+        dimensions,
+        cache: !values['no-cache'],
+        cacheDir: values['cache-dir'] ? cacheDir : undefined,
+        onProgress: loud
+            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
+            : undefined,
+        onWriting: loud ? (rows) => notes(indexing(rows, 'entities').map(aside)) : undefined,
+    });
+    await swapIn(dir, staging);
+    if (ctx.json) {
+        json({ dir, manifest: built.manifest });
+        return;
+    }
+    note();
+    write(dir);
+    note(`  restored ${bold(String(built.manifest.counts.entities))} entities in ${bold(dir)}, ` +
+        `embedded with ${built.manifest.embedding.ref} ` +
+        `(${built.manifest.embedding.dimensions}d)`);
+    note(dim(`  ${breakdown(built.timings)}`));
+    note(dim(`  reused ${built.reused}/${built.manifest.counts.entities} vectors` +
+        `${values['no-cache'] ? ' (--no-cache)' : ` from ${cacheDir}`}`));
+}
+/**
+ * The documents to rebuild from, in the order the index recorded them — which
+ * is what keeps the slugs the same, since two documents with the same filename
+ * are told apart by which was read first.
+ */
+function bundled(dir, manifest) {
+    const missing = manifest.sources.filter((s) => !s.path || !existsSync(join(dir, s.path)));
+    if (missing.length > 0) {
+        throw new CliError(`${dir} has no copy of ${missing.map((s) => s.name).join(', ')}`, EXIT.invalid, manifest.sources.some((s) => s.path)
+            ? `the ${SOURCES_DIR}/ directory is incomplete; rebuild with \`zen rag schema index\``
+            : 'it was built with --no-sources; rebuild it from the documents with `zen rag schema index`');
+    }
+    return manifest.sources.map((s) => join(dir, s.path));
+}
+// ---------------------------------------------------------------------------
+// ready
+// ---------------------------------------------------------------------------
+async function ready(args, ctx) {
+    const { values } = parse(args, { dir: { type: 'string', short: 'd' }, quiet: { type: 'boolean' } }, READY_USAGE);
+    const dir = indexDir(ctx, values.dir);
+    const health = await inspectIndex(dir, SCHEMA_INDEX, (at) => EntityStore.open(at));
+    reportReady(health, ctx, values.quiet === true);
+}
 const HEADERS = ['PATHS', 'OPERATIONS', 'SCHEMAS', 'FIELDS'];
 function elapsed(since) {
     const seconds = Math.round((Date.now() - since) / 1000);
@@ -805,8 +921,9 @@ async function stats(args, ctx) {
     const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag schema stats [--dir <dir>]');
     const dir = indexDir(ctx, values.dir);
     const manifest = await readManifest(dir);
+    const health = await inspectIndex(dir, SCHEMA_INDEX, (at) => EntityStore.open(at));
     if (ctx.json) {
-        json(manifest);
+        json({ ...manifest, state: health.state });
         return;
     }
     note(bold(dir));
@@ -818,6 +935,9 @@ async function stats(args, ctx) {
             '  indexes',
             `fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
         ],
+        // `stats` reports and never refuses, so a missing store is a row
+        // here rather than the error `search` would raise.
+        ['  vectors', health.state === 'ready' ? 'present' : dim('missing — `restore` them')],
     ]));
     printSources(manifest.sources);
     notes(table([['  entities', String(manifest.counts.entities)]]).map(dim));
@@ -827,6 +947,7 @@ function notes(lines) {
         note(line);
     }
 }
+const aside = (line) => dim(`  ${line}`);
 const yes = (value) => (value ? 'yes' : 'no');
 // ---------------------------------------------------------------------------
 function formatOf(value) {

package/dist/schema/files.d.ts

@@ -8,8 +8,8 @@ export declare const SCHEMA_INDEX: IndexSpec;
 export declare const GRAPH_FILE = "graph.json";
 export declare const SCHEMAS_FILE = "schemas.json";
 export declare const OPERATIONS_FILE = "operations.json";
-export declare const LANCE_DIR = "lance";
 export declare const SOURCES_DIR = "sources";
+export { LANCE_DIR, lancePath } from '../common/manifest.ts';
 export interface SourceRecord {
     /** the document's name within the index: what every entity's `source` says */
     name: string;
@@ -53,7 +53,6 @@ export interface OpenIndex {
     /** the operations, likewise */
     operations(): Promise<Operation[]>;
 }
-export declare const lancePath: (dir: string) => string;
 export declare function writeIndex(dir: string, index: WrittenIndex): Promise<void>;
 export declare function openIndex(dir: string): Promise<OpenIndex>;
 /**

package/dist/schema/files.js

@@ -31,9 +31,8 @@ export const SCHEMA_INDEX = {
 export const GRAPH_FILE = 'graph.json';
 export const SCHEMAS_FILE = 'schemas.json';
 export const OPERATIONS_FILE = 'operations.json';
-export const LANCE_DIR = 'lance';
 export const SOURCES_DIR = 'sources';
-export const lancePath = (dir) => join(dir, LANCE_DIR);
+export { LANCE_DIR, lancePath } from "../common/manifest.js";
 export async function writeIndex(dir, index) {
     await mkdir(dir, { recursive: true });
     await writeFile(join(dir, GRAPH_FILE), JSON.stringify(index.graph.export()));

package/dist/schema/store.js

@@ -100,7 +100,7 @@ export class EntityStore {
         }
         catch {
             db.close();
-            throw new CliError(`${dir} holds no searchable table`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
+            throw new CliError(`${dir} holds no searchable table`, EXIT.invalid, 'put the vectors back with `zen rag schema restore`, or rebuild it with `zen rag schema index`');
         }
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.21",
+    "version": "1.1.22",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.21",
-        "@zenera/neo": "^1.1.21",
+        "@zenera/cli": "^1.1.22",
+        "@zenera/neo": "^1.1.22",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",