@zenera/rag 1.1.2 → 1.1.4

package/README.md

@@ -29,7 +29,7 @@ standalone OpenAPI document.
 Node.js 24+. Install it alongside the CLI:
 
 ```sh
-npm i -g @zenera/cli @zenera/rag openai
+npm i -g @zenera/cli @zenera/rag
 zen key add openai          # the keyring `zen` already uses
 ```
 
@@ -47,8 +47,8 @@ zen rag schema show Type:Invoice      # a named node, with no search in between
 ```
 
 Non-interactive search is a machine interface: every field is a flag, the whole
-query can arrive as one JSON object, `--json` is a stable shape, no terminal is
-required, and an empty result exits 0.
+query can arrive as one JSON object, the `--json` output keeps the same
+structure from run to run, no terminal is required, and an empty result exits 0.
 
 ```sh
 zen rag schema search --query - --format ts <<'JSON'
@@ -106,16 +106,38 @@ need the word explained again, it needs the list of types that have one.
 
 ```
 schema-db/
+├── README.md         what this index holds — a live progress report while it builds
 ├── manifest.json     written last — its absence means "not indexed"
 ├── graph.json        topology and light attributes, read whole
-├── schemas.json      the raw schemas, read on first hydrate
+├── schemas.json      the raw schemas, read the first time one is needed in full
 ├── operations.json   likewise, for the OpenAPI subset
+├── sources/          the documents themselves, bundled, exactly as indexed
 └── lance/            one table: a row per node, one text column, one vector
 ```
 
 The manifest records which embedder made the vectors, and a search with a
 different one is refused rather than answered with noise.
 
+Indexing a large document is minutes of silence, so the directory says what is
+happening to it. `README.md` appears first as a progress report — the documents,
+the embedder, the step, how many entities have been embedded of how many, and
+how long it has been going — rewritten at most every five seconds, and replaced
+on completion by a description of what the index turned out to hold. A build
+that dies leaves it saying so. While one runs, `.lock` names the process; a
+second build of the same directory is refused, unless the lock is stale.
+
+**An index is one portable thing.** Nothing in it names a path outside itself:
+a document is known by a short name taken from its filename, and the document
+itself is copied into `sources/` as it was indexed — bundled, so every external
+`$ref` is already resolved and the copy stands alone. The directory can be moved,
+committed or shipped whole and still says what it is made of. This matters
+because a project's `assets/` is mounted at `/assets` inside an agent's sandbox,
+so an index built here is read under a name this machine never sees. `--no-sources`
+leaves the copies out, for an index that will never travel.
+
+The copies are a record, not an input: rebuilding reads the files you name, not
+the ones in `sources/`.
+
 ## Notes
 
 - Documents are **bundled, not dereferenced**: `#/components/schemas/User` is
@@ -125,8 +147,8 @@ different one is refused rather than answered with noise.
 - A **query parameter is a property**, like any field in a body. Nobody should
   have to know in advance which one `page_size` is.
 - Every type carries a **direction** — `input`, `output` or `both` — worked out
-  by propagating from the operations through composition, so a shared DTO is
-  honestly both rather than whichever side was read last.
+  by propagating from the operations through composition, so a type used on both
+  sides is honestly both rather than whichever side was read last.
 - Filters reaching the store are **closed enums only**. Exclusion lists are
   applied in JavaScript afterwards, so nothing a model wrote ever reaches a SQL
   predicate.

package/dist/command.js

@@ -1,6 +1,6 @@
-import { relative, resolve } from 'node:path';
-import { bold, CliError, cyan, dim, ensureHome, EXIT, isInteractive, json, KeyStore, note, parse, PROVIDERS, SHAPES, table, usageError, write, } from '@zenera/cli/lib';
+import { bold, CliError, CURATED, cyan, dim, ensureHome, envNames, EXIT, form, isInteractive, json, KeyStore, note, parse, PROVIDERS, table, usageError, write, } from '@zenera/cli/lib';
 import { createEmbedder } from '@zenera/neo';
+import { relative, resolve } from 'node:path';
 import { isFormat, present } from "./present.js";
 import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { repl } from "./repl.js";
@@ -46,6 +46,7 @@ export const command = {
                 '  --batch <n>',
                 dim('Texts per embedding request, and how often progress prints. Default 96.'),
             ],
+            ['  --no-sources', dim('Do not keep a copy of each document in the index.')],
         ]),
         '',
         'Search terms (repeatable)',
@@ -107,6 +108,7 @@ async function index(args, ctx) {
         out: { type: 'string', short: 'o' },
         embedding: { type: 'string' },
         batch: { type: 'string' },
+        'no-sources': { type: 'boolean' },
         quiet: { type: 'boolean' },
     }, INDEX_USAGE);
     if (positionals.length === 0) {
@@ -123,9 +125,10 @@ async function index(args, ctx) {
         embeddingRef: values.embedding,
         indexer: 'zenera-rag',
         batch: values.batch ? count(values.batch, '--batch') : undefined,
+        sources: !values['no-sources'],
         onRead: loud
             ? (summary) => {
-                printSources(summary.sources, ctx.cwd);
+                printSources(summary.sources);
                 // The first batch can take a while and says nothing while it
                 // does; this is the line that makes that a wait, not a hang.
                 note(dim(`  embedding ${summary.counts.entities} entities with ${chosen.id} …`));
@@ -153,9 +156,9 @@ function elapsed(since) {
     const seconds = Math.round((Date.now() - since) / 1000);
     return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
 }
-function printSources(sources, cwd) {
+function printSources(sources) {
     const rows = sources.map((s) => ({
-        name: relative(cwd, s.path) || s.path,
+        name: s.file,
         dialect: s.dialect,
         cells: [s.paths, s.methods, s.types, s.properties],
     }));
@@ -359,7 +362,7 @@ async function stats(args, ctx) {
             `fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
         ],
     ]));
-    printSources(manifest.sources, ctx.cwd);
+    printSources(manifest.sources);
     notes(table([['  entities', String(manifest.counts.entities)]]).map(dim));
 }
 function notes(lines) {
@@ -375,7 +378,7 @@ async function embedder(ref) {
     const keys = await KeyStore.open();
     // Asked before materialising, because materialising is what erases the
     // difference between "the environment had it" and "the keyring supplied it".
-    const fromEnv = new Set(PROVIDERS.filter((p) => process.env[SHAPES[p].env]));
+    const fromEnv = new Set(PROVIDERS.filter((p) => envNames(p).some((n) => process.env[n])));
     keys.materialize();
     if (!ref) {
         throw choices(keys, fromEnv);
@@ -383,28 +386,24 @@ async function embedder(ref) {
     return createEmbedder(ref);
 }
 /**
- * Well-known embedding models per provider. A list rather than a lookup: any
- * ref the registry can parse works, and these are the ones worth typing.
- * Anthropic is absent because it publishes no embeddings API at all.
+ * Well-known embedding models per provider, read off the CLI's catalog table so
+ * there is one list rather than two that drift. Any ref the registry can parse
+ * works; these are the ones worth typing. Anthropic has none because it
+ * publishes no embeddings API at all.
  */
-const EMBEDDINGS = {
-    openai: ['text-embedding-3-small', 'text-embedding-3-large'],
-    google: ['gemini-embedding-001'],
-    vertex: ['gemini-embedding-001', 'text-embedding-005'],
-    openrouter: ['openai/text-embedding-3-small'],
-};
+const embeddingsOf = (provider) => CURATED[provider].filter((m) => m.roles.includes('embedding')).map((m) => m.id);
 /** What could be passed, with the ones this machine can actually use first. */
 function choices(keys, fromEnv) {
     const rows = [];
     const rest = [];
     for (const provider of PROVIDERS) {
-        for (const model of EMBEDDINGS[provider] ?? []) {
+        for (const model of embeddingsOf(provider)) {
             const source = fromEnv.has(provider)
                 ? 'environment'
                 : keys.active(provider)
                     ? 'keyring'
                     : '';
-            const row = [`  ${cyan(`${provider}:${model}`)}`, dim(source || SHAPES[provider].env)];
+            const row = [`  ${cyan(`${provider}:${model}`)}`, dim(source || form(provider).env)];
             (source ? rows : rest).push(row);
         }
     }
@@ -415,7 +414,9 @@ function choices(keys, fromEnv) {
         note(dim('  no provider on this machine has a credential — try: zen key add openai'));
         note('');
     }
-    return usageError('no embedder named', 'pass --embedding <ref>, one of the above');
+    // `pick` is the one that ends the question rather than restating it: it
+    // tries them and prints the first that answers.
+    return usageError('no embedder named', 'pass --embedding <ref>, or run: zen models pick --embedding');
 }
 function formatOf(value) {
     if (value === undefined) {

package/dist/schema/build.d.ts

@@ -11,6 +11,8 @@ export interface BuildOptions {
     indexer: string;
     /** texts sent to the embedder at once */
     batch?: number;
+    /** keep a bundled copy of each document in the index. On by default. */
+    sources?: boolean;
     signal?: AbortSignal;
     /** what the documents turned out to hold, before a vector has been paid for */
     onRead?: (summary: BuildSummary) => void;

package/dist/schema/build.js

@@ -1,42 +1,67 @@
 import { toEntities } from "./entities.js";
-import { INDEX_VERSION, writeIndex, } from "./files.js";
+import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
 import { buildGraph } from "./graph.js";
+import { beginBuild } from "./progress.js";
 import { loadSpecs } from "./spec.js";
 import { writeStore } from "./store.js";
 const DEFAULT_BATCH = 96;
 export async function buildIndex(options) {
-    const corpus = await loadSpecs(options.files);
-    const { graph, types } = buildGraph(corpus);
-    const entities = toEntities(graph);
-    const summary = {
-        sources: sourcesOf(corpus, entities, options.files),
-        counts: {
-            methods: entities.filter((e) => e.kind === 'method').length,
-            types: entities.filter((e) => e.kind === 'type').length,
-            properties: entities.filter((e) => e.kind === 'property').length,
-            entities: entities.length,
-        },
-    };
-    options.onRead?.(summary);
-    const vectors = await embedAll(entities, options);
-    const written = await writeStore(options.out, entities, vectors);
-    const manifest = {
-        version: INDEX_VERSION,
-        createdAt: new Date().toISOString(),
+    const journal = beginBuild({
+        dir: options.out,
+        files: options.files,
+        embedding: options.embeddingRef ?? options.embedder.id,
         indexer: options.indexer,
-        embedding: {
-            ref: options.embeddingRef ?? options.embedder.id,
-            id: options.embedder.id,
-            dimensions: vectors[0]?.length ?? 0,
-        },
-        sources: summary.sources,
-        counts: summary.counts,
-        indexes: { fts: written.fts, vector: written.vector },
-    };
-    await writeIndex(options.out, { manifest, graph, types, operations: corpus.operations });
-    return { manifest, entities };
+    });
+    try {
+        const corpus = await loadSpecs(options.files);
+        journal.phase('graph');
+        const { graph, types } = buildGraph(corpus);
+        const entities = toEntities(graph);
+        const keep = options.sources !== false;
+        const summary = {
+            sources: sourcesOf(corpus, entities, keep),
+            counts: {
+                methods: entities.filter((e) => e.kind === 'method').length,
+                types: entities.filter((e) => e.kind === 'type').length,
+                properties: entities.filter((e) => e.kind === 'property').length,
+                entities: entities.length,
+            },
+        };
+        journal.read(summary.counts);
+        options.onRead?.(summary);
+        journal.phase('embedding');
+        const vectors = await embedAll(entities, options, journal);
+        journal.phase('writing');
+        const written = await writeStore(options.out, entities, vectors);
+        const manifest = {
+            version: INDEX_VERSION,
+            createdAt: new Date().toISOString(),
+            indexer: options.indexer,
+            embedding: {
+                ref: options.embeddingRef ?? options.embedder.id,
+                id: options.embedder.id,
+                dimensions: vectors[0]?.length ?? 0,
+            },
+            sources: summary.sources,
+            counts: summary.counts,
+            indexes: { fts: written.fts, vector: written.vector },
+        };
+        await writeIndex(options.out, {
+            manifest,
+            graph,
+            types,
+            operations: corpus.operations,
+            documents: keep ? corpus.documents : {},
+        });
+        journal.finish(manifest);
+        return { manifest, entities };
+    }
+    catch (err) {
+        journal.fail(err);
+        throw err;
+    }
 }
-async function embedAll(entities, options) {
+async function embedAll(entities, options, journal) {
     const size = options.batch ?? DEFAULT_BATCH;
     const out = [];
     for (let at = 0; at < entities.length; at += size) {
@@ -50,6 +75,7 @@ async function embedAll(entities, options) {
             throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
         }
         out.push(...response.vectors.map((v) => Float32Array.from(v)));
+        journal.progress(out.length, entities.length);
         options.onProgress?.(out.length, entities.length);
     }
     return out;
@@ -59,12 +85,14 @@ async function embedAll(entities, options) {
  * never named — an inline request body, say — is credited to the file it came
  * from instead of going missing.
  */
-function sourcesOf(corpus, entities, files) {
-    return corpus.docs.map((doc, index) => {
+function sourcesOf(corpus, entities, keep) {
+    return corpus.docs.map((doc) => {
         const mine = entities.filter((e) => e.source === doc.source);
         const operations = corpus.operations.filter((op) => op.source === doc.source);
         return {
-            path: files[index] ?? doc.source,
+            name: doc.source,
+            file: doc.file,
+            path: keep ? `${SOURCES_DIR}/${doc.source}.json` : undefined,
             sha256: doc.sha256,
             dialect: doc.dialect,
             title: doc.title,

package/dist/schema/files.d.ts

@@ -1,14 +1,20 @@
 import type { ApiGraph } from './graph.ts';
 import type { Schema } from './schema.ts';
 import type { Operation } from './spec.ts';
-export declare const INDEX_VERSION = 1;
+export declare const INDEX_VERSION = 3;
 export declare const MANIFEST_FILE = "manifest.json";
 export declare const GRAPH_FILE = "graph.json";
 export declare const SCHEMAS_FILE = "schemas.json";
 export declare const OPERATIONS_FILE = "operations.json";
 export declare const LANCE_DIR = "lance";
+export declare const SOURCES_DIR = "sources";
 export interface SourceRecord {
-    path: string;
+    /** the document's name within the index: what every entity's `source` says */
+    name: string;
+    /** what the file was called on the machine that built this */
+    file: string;
+    /** the bundled copy, relative to the index; absent when none was kept */
+    path?: string;
     sha256: string;
     dialect: string;
     title: string;
@@ -47,6 +53,8 @@ export interface WrittenIndex {
     graph: ApiGraph;
     types: Readonly<Record<string, Schema>>;
     operations: readonly Operation[];
+    /** the bundled documents to keep beside the index, by name */
+    documents: Readonly<Record<string, string>>;
 }
 export interface OpenIndex {
     dir: string;

package/dist/schema/files.js

@@ -1,32 +1,44 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
 import { MultiDirectedGraph } from 'graphology';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { CliError, EXIT } from '@zenera/cli/lib';
 // ---------------------------------------------------------------------------
 // What an index is, on disk
 //
-// Four things in a directory, and the order they are written in is the whole
-// crash story: the manifest goes last, so a half-built index has no manifest
-// and reads as "not indexed" rather than as a store that quietly lost half its
-// operations.
+// A directory, and the order it is written in is the whole crash story: the
+// manifest goes last, so a half-built index has no manifest and reads as "not
+// indexed" rather than as a store that quietly lost half its operations.
 //
 // `graph.json` is deliberately thin — names, directions, edges — because it is
 // parsed whole on every search. The schemas are the bulk of the bytes and are
 // wanted only when something is being printed, so they live apart and are read
 // on first use.
+//
+// `sources/` holds the documents themselves, bundled, so the index is one
+// portable thing: nothing in it names a path outside itself, it can be moved or
+// shipped whole, and the graph can be rebuilt — or re-embedded with another
+// model — without going looking for the files it was made from.
 // ---------------------------------------------------------------------------
-export const INDEX_VERSION = 1;
+export const INDEX_VERSION = 3;
 export const MANIFEST_FILE = 'manifest.json';
 export const GRAPH_FILE = 'graph.json';
 export const SCHEMAS_FILE = 'schemas.json';
 export const OPERATIONS_FILE = 'operations.json';
 export const LANCE_DIR = 'lance';
+export const SOURCES_DIR = 'sources';
 export const lancePath = (dir) => join(dir, LANCE_DIR);
 export async function writeIndex(dir, index) {
     await mkdir(dir, { recursive: true });
     await writeFile(join(dir, GRAPH_FILE), JSON.stringify(index.graph.export()));
     await writeFile(join(dir, SCHEMAS_FILE), JSON.stringify(index.types));
     await writeFile(join(dir, OPERATIONS_FILE), JSON.stringify(index.operations));
+    const documents = Object.entries(index.documents);
+    if (documents.length > 0) {
+        await mkdir(join(dir, SOURCES_DIR), { recursive: true });
+        for (const [name, text] of documents) {
+            await writeFile(join(dir, SOURCES_DIR, `${name}.json`), text);
+        }
+    }
     await writeFile(join(dir, MANIFEST_FILE), `${JSON.stringify(index.manifest, null, 4)}\n`);
 }
 export async function openIndex(dir) {
@@ -58,7 +70,7 @@ export async function readManifest(dir) {
     }
     const manifest = JSON.parse(text);
     if (manifest.version !== INDEX_VERSION) {
-        throw new CliError(`${dir} was written by another version of this indexer`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
+        throw new CliError(`${dir} is a version ${manifest.version} index, and this indexer reads version ${INDEX_VERSION}`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
     }
     return manifest;
 }

package/dist/schema/progress.d.ts

@@ -0,0 +1,26 @@
+import type { Counts, Manifest } from './files.ts';
+export declare const LOCK_FILE = ".lock";
+export declare const README_FILE = "README.md";
+export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
+export interface BuildPlan {
+    /** where the index goes */
+    dir: string;
+    files: readonly string[];
+    /** the embedding reference as it was typed */
+    embedding: string;
+    indexer: string;
+}
+export interface Journal {
+    phase(name: Phase): void;
+    /** what the documents turned out to hold, once they have been read */
+    read(counts: Counts): void;
+    progress(done: number, total: number): void;
+    finish(manifest: Manifest): void;
+    fail(reason: unknown): void;
+}
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export declare function beginBuild(plan: BuildPlan): Journal;
+//# sourceMappingURL=progress.d.ts.map

package/dist/schema/progress.js

@@ -0,0 +1,316 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { basename, join } from 'node:path';
+// ---------------------------------------------------------------------------
+// A build that says what it is doing
+//
+// Indexing a large document is minutes of silence with a directory slowly
+// filling up, and the directory is the only thing a second person — or a second
+// process, or an agent reading the tree — ever sees. So the build writes two
+// files into it and keeps them true:
+//
+//   .lock       who is building this, right now. Gone when nothing is.
+//   README.md   a progress report while it runs, and a description of what the
+//               index holds once it does not.
+//
+// Nothing in either file is an absolute path. A project's `assets/` directory
+// is mounted at `/assets` inside an agent's sandbox, so an index built there is
+// read under a name this process never sees; a host path would be a lie there.
+// ---------------------------------------------------------------------------
+export const LOCK_FILE = '.lock';
+export const README_FILE = 'README.md';
+/** The floor on how often README.md is rewritten. */
+const INTERVAL_MS = 5000;
+const PHASES = {
+    reading: 'reading the documents',
+    graph: 'building the graph',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export function beginBuild(plan) {
+    const documents = plan.files.map((file) => basename(file));
+    const started = Date.now();
+    const lock = {
+        pid: process.pid,
+        host: hostname(),
+        startedAt: new Date(started).toISOString(),
+        indexer: plan.indexer,
+        embedding: plan.embedding,
+        documents,
+    };
+    mkdirSync(plan.dir, { recursive: true });
+    claim(join(plan.dir, LOCK_FILE), lock);
+    let phase = 'reading';
+    let counts;
+    let done = 0;
+    let total = 0;
+    let wroteAt = 0;
+    let closed = false;
+    const write = (body) => {
+        // Through a temp name, so a reader never catches half a file.
+        const target = join(plan.dir, README_FILE);
+        const temp = `${target}.tmp`;
+        writeFileSync(temp, body);
+        renameSync(temp, target);
+        wroteAt = Date.now();
+    };
+    const report = () => write(building({
+        documents,
+        embedding: plan.embedding,
+        started,
+        phase,
+        done,
+        total,
+        counts,
+    }));
+    const maybe = () => {
+        if (!closed && Date.now() - wroteAt >= INTERVAL_MS) {
+            report();
+        }
+    };
+    // The first embedding request can block for ten seconds or more, so a purely
+    // event-driven throttle would leave the elapsed line frozen through it.
+    const ticker = setInterval(maybe, INTERVAL_MS);
+    ticker.unref();
+    const close = (body) => {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        clearInterval(ticker);
+        write(body);
+        rmSync(join(plan.dir, LOCK_FILE), { force: true });
+    };
+    report();
+    return {
+        phase(name) {
+            phase = name;
+            maybe();
+        },
+        read(seen) {
+            counts = seen;
+            total = seen.entities;
+            maybe();
+        },
+        progress(at, of) {
+            done = at;
+            total = of;
+            maybe();
+        },
+        finish(manifest) {
+            close(complete(plan.dir, manifest, Date.now() - started));
+        },
+        fail(reason) {
+            close(failed({ documents, phase, reason, started }));
+        },
+    };
+}
+/**
+ * `wx` makes the create and the check one operation, so two builds racing for
+ * the same directory cannot both win. A lock whose process is gone is stale by
+ * definition and is taken rather than respected — a crashed build must not make
+ * a directory permanently unbuildable.
+ */
+function claim(path, lock) {
+    const body = `${JSON.stringify(lock, null, 4)}\n`;
+    try {
+        writeFileSync(path, body, { flag: 'wx' });
+        return;
+    }
+    catch (err) {
+        if (err.code !== 'EEXIST') {
+            throw err;
+        }
+    }
+    const held = readLock(path);
+    if (held && held.host === hostname() && alive(held.pid)) {
+        throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
+    }
+    writeFileSync(path, body);
+}
+function readLock(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
+ * EPERM means it exists and belongs to someone else, which still counts.
+ */
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+function building(state) {
+    const now = Date.now();
+    const rows = [
+        ['documents', state.documents.join(', ')],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(now - state.started)} ago)`,
+        ],
+        ['step', PHASES[state.phase]],
+    ];
+    if (state.counts) {
+        rows.push(['found', entities(state.counts)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(now).toISOString()]);
+    return [
+        '# Schema index — being built',
+        '',
+        'A searchable index of the API documents named below, written by `zen rag schema index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(dir, manifest, ms) {
+    const titles = manifest.sources.map((s) => s.title).filter(Boolean);
+    const what = titles.length > 0 ? titles.join(', ') : basename(dir);
+    return [
+        `# Schema index — ${what}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(ms)}.`,
+        'Ask it for the operations and types behind a question and it answers with a subgraph:',
+        'the endpoints that match, the schemas they carry, and the fields inside those — printed',
+        'as text, Mermaid, TypeScript or OpenAPI.',
+        '',
+        '## What it covers',
+        '',
+        ...sourceTable(manifest.sources),
+        '',
+        `${entities(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['graph.json', 'the nodes and edges: operations, types, fields'],
+            ['schemas.json', 'the JSON Schema of every type'],
+            ['operations.json', 'every operation, with its parameters and responses'],
+            ...(manifest.sources.some((s) => s.path)
+                ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
+                : []),
+            ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag schema search --dir . --all "how do I cancel a subscription"',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Schema index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag schema index`.',
+        '',
+        ...fields([
+            ['documents', state.documents.join(', ')],
+            ['step', PHASES[state.phase]],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+// Small renderings
+// ---------------------------------------------------------------------------
+/** An indented block, which markdown renders verbatim and an agent reads as a table. */
+function fields(rows) {
+    const width = Math.max(...rows.map((r) => (r[0] ?? '').length));
+    return rows.map(([name, value]) => `    ${(name ?? '').padEnd(width)}  ${value ?? ''}`);
+}
+const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
+function sourceTable(sources) {
+    const rows = sources.map((s) => [
+        s.file,
+        s.dialect,
+        String(s.paths),
+        String(s.methods),
+        String(s.types),
+        String(s.properties),
+    ]);
+    const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
+    const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' | ')} |`;
+    return [
+        line(HEADERS),
+        `| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`,
+        ...rows.map(line),
+    ];
+}
+function entities(counts) {
+    return (`${plural(counts.entities, 'entity', 'entities')}: ` +
+        `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
+}
+function searched(indexes) {
+    if (!indexes.fts && !indexes.vector) {
+        return 'scanned flat: neither index was built';
+    }
+    if (!indexes.vector) {
+        // Below a couple of thousand rows an IVF index has nothing to train on.
+        return 'searched by full text and by vector, the latter as a flat scan';
+    }
+    return indexes.fts
+        ? 'searched by full text and by vector'
+        : 'searched by vector, with no full-text index';
+}
+function plural(n, one, many = `${one}s`) {
+    return `${n} ${n === 1 ? one : many}`;
+}
+function duration(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 1) {
+        return 'under a second';
+    }
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    return minutes < 60
+        ? `${minutes}m${seconds % 60}s`
+        : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
+}
+function message(reason) {
+    const text = reason instanceof Error ? reason.message : String(reason);
+    return text.split('\n')[0]?.trim() || 'no reason given';
+}
+//# sourceMappingURL=progress.js.map

package/dist/schema/spec.d.ts

@@ -36,7 +36,10 @@ export interface Operation {
     responses: ResponseSpec[];
 }
 export interface ApiDoc {
+    /** the document's name within this index, unique in the corpus */
     source: string;
+    /** what the file was called on the machine that read it */
+    file: string;
     sha256: string;
     dialect: Dialect;
     title: string;
@@ -49,6 +52,8 @@ export interface Corpus {
     types: Record<string, Schema>;
     /** which document each type id came from */
     typeSource: Record<string, string>;
+    /** the bundled document behind each `source`, as JSON text */
+    documents: Record<string, string>;
 }
 /** A `CliError` so an unreadable document exits 3 wherever it is raised. */
 export declare class SpecError extends CliError {

package/dist/schema/spec.js

@@ -1,7 +1,7 @@
 import SwaggerParser from '@apidevtools/swagger-parser';
+import { CliError, EXIT } from '@zenera/cli/lib';
 import { createHash } from 'node:crypto';
 import { basename, extname } from 'node:path';
-import { CliError, EXIT } from '@zenera/cli/lib';
 import { docOf, isObject, normalize } from "./schema.js";
 // ---------------------------------------------------------------------------
 // Documents, flattened — but not dereferenced
@@ -38,12 +38,26 @@ export async function loadSpecs(files) {
         throw new SpecError('no document given', 'name at least one openapi/swagger file');
     }
     const loaded = [];
+    const taken = new Set();
     for (const file of files) {
-        loaded.push(await loadSpec(file));
+        loaded.push(await loadSpec(file, distinct(slugOf(file), taken)));
     }
     return settle(loaded);
 }
-async function loadSpec(file) {
+/**
+ * The slug is the document's identity everywhere below this file — the node
+ * attribute, the store column, the name its copy is written under — so two
+ * `api.yaml`s in different directories must not answer to the same word.
+ */
+function distinct(slug, taken) {
+    let candidate = slug;
+    for (let n = 2; taken.has(candidate); n++) {
+        candidate = `${slug}_${n}`;
+    }
+    taken.add(candidate);
+    return candidate;
+}
+async function loadSpec(file, slug) {
     let raw;
     try {
         raw = (await SwaggerParser.bundle(file));
@@ -51,6 +65,8 @@ async function loadSpec(file) {
     catch (err) {
         throw new SpecError(`${file}: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`, 'the document must be a readable OpenAPI 3.x or Swagger 2.0 file');
     }
+    // `bundle` has resolved every external `$ref`, so this text stands alone.
+    const document = JSON.stringify(raw, null, 2);
     const dialect = dialectOf(raw, file);
     const source = raw.components?.schemas ?? raw.definitions ?? {};
     const types = new Map();
@@ -59,15 +75,17 @@ async function loadSpec(file) {
     }
     return {
         doc: {
-            source: file,
-            sha256: createHash('sha256').update(JSON.stringify(raw)).digest('hex'),
+            source: slug,
+            file: basename(file),
+            sha256: createHash('sha256').update(document).digest('hex'),
             dialect,
             title: raw.info?.title?.trim() || basename(file),
             version: raw.info?.version?.trim() || '',
         },
-        slug: slugOf(file),
-        operations: operationsOf(raw, dialect, file),
+        slug,
+        operations: operationsOf(raw, dialect, slug),
         types,
+        document,
     };
 }
 function dialectOf(doc, file) {
@@ -89,7 +107,7 @@ function slugOf(file) {
 // ---------------------------------------------------------------------------
 // Operations
 // ---------------------------------------------------------------------------
-function operationsOf(raw, dialect, file) {
+function operationsOf(raw, dialect, source) {
     const prefix = dialect === 'swagger-2.0' ? (raw.basePath ?? '') : '';
     const out = [];
     for (const [template, item] of Object.entries(raw.paths ?? {})) {
@@ -107,7 +125,7 @@ function operationsOf(raw, dialect, file) {
             const path = join(prefix, template);
             const own = paramsOf(op.parameters, dialect);
             out.push({
-                source: file,
+                source,
                 method,
                 path,
                 operationId: text(op.operationId) || synthesizeId(method, path),
@@ -275,7 +293,13 @@ function settle(loaded) {
             });
         }
     });
-    return { docs: loaded.map((one) => one.doc), operations, types, typeSource };
+    return {
+        docs: loaded.map((one) => one.doc),
+        operations,
+        types,
+        typeSource,
+        documents: Object.fromEntries(loaded.map((one) => [one.slug, one.document])),
+    };
 }
 function unique(id, slug, seen) {
     let candidate = seen.has(id) ? `${slug}.${id}` : id;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.2",
+    "version": "1.1.4",
     "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
     "keywords": [
         "agents",
@@ -53,7 +53,7 @@
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
         "graphology": "^0.26.0",
-        "@zenera/cli": "^1.1.2",
-        "@zenera/neo": "^1.1.2"
+        "@zenera/cli": "^1.1.4",
+        "@zenera/neo": "^1.1.4"
     }
 }