@zenera/rag 1.1.9 → 1.1.11

package/dist/docs/tools.js

@@ -0,0 +1,300 @@
+import { tool } from '@zenera/neo';
+import { PatternError } from "../common/match.js";
+import { assemble } from "./assemble.js";
+import { CHUNK_KINDS } from "./chunk.js";
+import { grepLines, listFiles, listSections, listTables, readRange, readSection, } from "./lookup.js";
+import { renderAssembly } from "./render.js";
+import { SEARCH_MODES } from "./search.js";
+// ---------------------------------------------------------------------------
+// The same index, given to an agent
+//
+// Four tools over one engine, and only one of them ranks anything. `search_docs`
+// is the way in when the question is vague; the other three are exact, because
+// a model told "no results" by a vector search has learned nothing — a ranking
+// returns the top of a list, so an empty answer and an absent thing look
+// identical.
+//
+// `search_docs` is shaped for the second call rather than the first. The first
+// is always a sentence and always returns too much of the wrong tree; the
+// second is the same sentence with `files: ["nsx_4.2*/api/**"]`, or
+// `section: "Rate limits"`, or `kind: ["table"]` because the answer is a table
+// and not the prose around it. Those are parameters and not separate tools, so
+// narrowing costs one call and not three.
+//
+// Every answer carries line numbers, and `read_docs` takes them. That is the
+// loop the whole thing exists for: find the passage, read around it, then edit
+// the file the passage came from — and a passage that cannot be pointed at is a
+// passage nothing can be done with.
+// ---------------------------------------------------------------------------
+const GROUP = 'docs';
+/** Kept small on purpose: a tool result is prompt, and the model asked for one thing. */
+const DEFAULT_LIMIT = 5;
+const MAX_LIMIT = 25;
+/** A listing is lines rather than passages, so it can afford more of them. */
+const DEFAULT_ROWS = 50;
+const MAX_ROWS = 200;
+/** A ceiling on what one answer may quote, so one long section cannot eat it. */
+const DEFAULT_MAX_LINES = 200;
+const MAX_LINES = 800;
+export function docsTools(index, options = {}) {
+    const limitOf = (asked) => clamp(asked ?? options.limit ?? DEFAULT_LIMIT, 1, MAX_LIMIT);
+    const linesOf = (asked) => clamp(asked ?? options.maxLines ?? DEFAULT_MAX_LINES, 20, MAX_LINES);
+    const searchDocs = tool({
+        name: 'search_docs',
+        group: GROUP,
+        description: 'Searches the documents and answers with the passages that matched, quoted ' +
+            'verbatim with their line numbers and with a marker wherever something between ' +
+            'two passages was left out. When the first answer is from the wrong part of the ' +
+            'corpus, ask again with the same query and a narrowing: `files` for a path ' +
+            'pattern, `section` for a heading, `kind` for tables or code only. Pass the ids ' +
+            'from an earlier answer in exclude_ids to be shown something new instead.',
+        parameters: {
+            type: 'object',
+            properties: {
+                query: {
+                    type: 'string',
+                    description: 'What is wanted, as a sentence rather than keywords.',
+                },
+                files: strings('Only documents whose name matches. A glob if it has * or ?, e.g. ' +
+                    '"guides/**" or "nsx_4.2*/api/**"; otherwise a substring.'),
+                exclude_files: strings('Documents to leave out, matched the same way.'),
+                section: strings('Only under these headings, and whatever nests inside them. A heading ' +
+                    'title, or a structure path from an earlier answer.'),
+                kind: {
+                    type: 'array',
+                    items: { type: 'string', enum: [...CHUNK_KINDS] },
+                    description: 'Only these kinds of block. Use ["table"] or ["table_row"] when the ' +
+                        'answer is tabular, ["code"] for examples.',
+                },
+                mode: {
+                    type: 'string',
+                    enum: [...SEARCH_MODES],
+                    description: 'hybrid blends meaning and wording; text is exact wording only, for ' +
+                        'an error string or an identifier.',
+                },
+                exclude_ids: strings('Passage ids already seen, as printed in an earlier answer.'),
+                limit: {
+                    type: 'integer',
+                    description: `Passages kept. Default ${DEFAULT_LIMIT}, at most ${MAX_LIMIT}.`,
+                },
+                before: { type: 'integer', description: 'Extra lines quoted before each passage.' },
+                after: { type: 'integer', description: 'Extra lines quoted after each passage.' },
+                max_lines: {
+                    type: 'integer',
+                    description: `A ceiling on the whole answer. Default ${DEFAULT_MAX_LINES}.`,
+                },
+            },
+            required: ['query'],
+            additionalProperties: false,
+        },
+        execute: async (args) => {
+            const query = {
+                query: args.query,
+                files: args.files,
+                exclude_files: args.exclude_files,
+                section: args.section,
+                kinds: args.kind,
+                mode: args.mode,
+                exclude_ids: args.exclude_ids,
+                limit: limitOf(args.limit),
+            };
+            const result = await guard(() => index.search(query));
+            if ('error' in result) {
+                return result;
+            }
+            if (result.files.length === 0) {
+                return {
+                    found: 0,
+                    hint: 'no document matched `files` — call list_docs to see their names',
+                };
+            }
+            if (result.matches.length === 0) {
+                return {
+                    found: 0,
+                    scope: { documents: result.files.length, sections: result.sections.length },
+                    hint: args.section?.length
+                        ? 'nothing under that section — drop `section` and search the whole document'
+                        : 'try fewer words, or mode "text" if it is an exact string',
+                };
+            }
+            const excerpt = await assemble(index, result.matches, {
+                before: args.before,
+                after: args.after,
+                maxLines: linesOf(args.max_lines),
+            });
+            return {
+                found: result.matches.length,
+                ids: result.matches.map((m) => m.id),
+                documents: excerpt.files.map((f) => f.path),
+                truncated: excerpt.truncated,
+                passages: renderAssembly(excerpt, { colour: false }),
+            };
+        },
+    });
+    const listDocs = tool({
+        name: 'list_docs',
+        group: GROUP,
+        description: 'Lists what is in the index — the documents, their headings, or their tables — ' +
+            'without searching or ranking anything. Call it first to learn the document ' +
+            "names that `files` patterns are matched against, or to see a document's " +
+            'structure before asking about one part of it.',
+        parameters: {
+            type: 'object',
+            properties: {
+                what: {
+                    type: 'string',
+                    enum: ['files', 'sections', 'tables'],
+                    description: 'Default files.',
+                },
+                files: strings('Only these documents, matched by glob or substring.'),
+                section: strings('Only under these headings.'),
+                depth: {
+                    type: 'integer',
+                    description: 'sections only: the deepest heading level to report.',
+                },
+                limit: { type: 'integer', description: `Rows kept. Default ${DEFAULT_ROWS}.` },
+            },
+            additionalProperties: false,
+        },
+        execute: async (args) => {
+            const options = {
+                files: args.files,
+                section: args.section,
+                depth: args.depth,
+                limit: clamp(args.limit ?? DEFAULT_ROWS, 1, MAX_ROWS),
+            };
+            const what = args.what ?? 'files';
+            const result = await guard(() => what === 'sections'
+                ? listSections(index, options)
+                : what === 'tables'
+                    ? listTables(index, options)
+                    : listFiles(index, options));
+            if ('error' in result) {
+                return result;
+            }
+            return {
+                found: result.found,
+                truncated: result.truncated,
+                [what]: result.rows,
+                ...(result.found === 0
+                    ? { hint: 'nothing matched — widen `files`, or call it with no arguments' }
+                    : {}),
+            };
+        },
+    });
+    const grepDocs = tool({
+        name: 'grep_docs',
+        group: GROUP,
+        description: 'Every line matching a pattern, with the document and line number, and the ' +
+            'section it sits in. Exact, not ranked, and it reports the true total even when ' +
+            'the rows are cut — so unlike search_docs it can answer "does this string appear ' +
+            'anywhere". Reach for it with an identifier, an error message, or a flag name.',
+        parameters: {
+            type: 'object',
+            properties: {
+                pattern: {
+                    type: 'string',
+                    description: 'A substring, a glob if it has * or ?, or a regex with regex.',
+                },
+                files: strings('Only these documents.'),
+                section: strings('Only under these headings.'),
+                regex: { type: 'boolean', description: 'Read the pattern as a regex.' },
+                case_sensitive: { type: 'boolean', description: 'Match the capitals too.' },
+                limit: { type: 'integer', description: `Lines kept. Default ${DEFAULT_ROWS}.` },
+            },
+            required: ['pattern'],
+            additionalProperties: false,
+        },
+        execute: async (args) => {
+            const result = await guard(() => grepLines(index, args.pattern, {
+                files: args.files,
+                section: args.section,
+                regex: args.regex,
+                caseSensitive: args.case_sensitive,
+                limit: clamp(args.limit ?? DEFAULT_ROWS, 1, MAX_ROWS),
+            }));
+            if ('error' in result) {
+                return result;
+            }
+            return {
+                found: result.found,
+                truncated: result.truncated,
+                lines: result.rows,
+                ...(result.found === 0 ? { hint: 'nothing matched anywhere in scope' } : {}),
+            };
+        },
+    });
+    const readDocs = tool({
+        name: 'read_docs',
+        group: GROUP,
+        description: 'Reads a document verbatim: a whole named section, or a line range as printed by ' +
+            'search_docs or grep_docs. Use it when a passage was found and the lines around ' +
+            'it are needed in full, with nothing omitted and nothing summarised.',
+        parameters: {
+            type: 'object',
+            properties: {
+                file: {
+                    type: 'string',
+                    description: 'A document name, exactly as list_docs prints it.',
+                },
+                section: {
+                    type: 'string',
+                    description: 'A heading title or structure path. Overrides from/to.',
+                },
+                from: { type: 'integer', description: 'First line, 1-based. Default 1.' },
+                to: { type: 'integer', description: 'Last line. Default the end of the document.' },
+            },
+            required: ['file'],
+            additionalProperties: false,
+        },
+        execute: async (args) => {
+            const file = index.resolveFiles([args.file])[0];
+            if (!file) {
+                return {
+                    error: `no document called ${args.file}`,
+                    hint: 'call list_docs for the names',
+                };
+            }
+            const result = await guard(async () => args.section
+                ? await readSection(index, file, args.section)
+                : await readRange(index, file, args.from ?? 1, args.to ?? Infinity));
+            if ('error' in result) {
+                return result;
+            }
+            // A whole large document is prompt spent on lines nobody asked for.
+            const kept = result.lines.slice(0, MAX_LINES);
+            return {
+                file: result.file,
+                start: result.start,
+                end: result.start + kept.length - 1,
+                total: result.total,
+                truncated: kept.length < result.lines.length,
+                text: kept.map((line, at) => `${result.start + at} | ${line}`).join('\n'),
+            };
+        },
+    });
+    return [searchDocs, listDocs, grepDocs, readDocs];
+}
+// ---------------------------------------------------------------------------
+const strings = (description) => ({
+    type: 'array',
+    items: { type: 'string' },
+    description,
+});
+const clamp = (value, low, high) => Math.min(Math.max(Math.trunc(value), low), high);
+/**
+ * A bad pattern is a bad argument, not a failure. A model handed a thrown
+ * exception retries the same call; one handed a sentence fixes it.
+ */
+async function guard(run) {
+    try {
+        return await run();
+    }
+    catch (err) {
+        if (err instanceof PatternError) {
+            return { error: err.message };
+        }
+        throw err;
+    }
+}
+//# sourceMappingURL=tools.js.map

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export * from './common/locate.ts';
 export * from './common/manifest.ts';
 export * from './common/match.ts';
+export * as docs from './docs/index.ts';
 export * from './schema/build.ts';
 export * from './schema/entities.ts';
 export * from './schema/files.ts';

package/dist/index.js

@@ -1,6 +1,9 @@
 export * from "./common/locate.js";
 export * from "./common/manifest.js";
 export * from "./common/match.js";
+// A namespace rather than a re-export: both subjects have a `Manifest`, a
+// `writeIndex` and a `search`, and flattening them would collide on every one.
+export * as docs from "./docs/index.js";
 export * from "./schema/build.js";
 export * from "./schema/entities.js";
 export * from "./schema/files.js";

package/dist/schema/build.d.ts

@@ -1,4 +1,5 @@
 import type { Embedder } from '@zenera/neo';
+import { type PhaseTiming } from '../common/progress.ts';
 import { type EntityRecord } from './entities.ts';
 import { type Counts, type Manifest, type SourceRecord } from './files.ts';
 export interface BuildOptions {
@@ -9,10 +10,19 @@ export interface BuildOptions {
     embeddingRef?: string;
     /** told the manifest, so a store can say what wrote it */
     indexer: string;
-    /** texts sent to the embedder at once */
-    batch?: number;
     /** keep a bundled copy of each document in the index. On by default. */
     sources?: boolean;
+    /**
+     * The width asked of the embedder, when one was asked for. Part of the cache
+     * key, because a truncated vector is a different vector; left undefined when
+     * nobody asked, because that is a different key again from asking for the
+     * number the model would have chosen anyway.
+     */
+    dimensions?: number;
+    /** reuse vectors this machine already has; on by default */
+    cache?: boolean;
+    /** keep them somewhere other than the shared store */
+    cacheDir?: string;
     signal?: AbortSignal;
     /** what the documents turned out to hold, before a vector has been paid for */
     onRead?: (summary: BuildSummary) => void;
@@ -25,6 +35,10 @@ export interface BuildSummary {
 export interface BuildResult {
     manifest: Manifest;
     entities: EntityRecord[];
+    /** what each phase cost, so a slow build can say which part was slow */
+    timings: readonly PhaseTiming[];
+    /** vectors that came out of the shared cache instead of being paid for again */
+    reused: number;
 }
 export declare function buildIndex(options: BuildOptions): Promise<BuildResult>;
 //# sourceMappingURL=build.d.ts.map

package/dist/schema/build.js

@@ -1,11 +1,11 @@
+import { embedStream, NO_CACHE, openCache } from "../common/cache.js";
 import { beginBuild } from "../common/progress.js";
 import { toEntities } from "./entities.js";
 import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
 import { buildGraph } from "./graph.js";
 import { PHASES, SCHEMA_REPORT } from "./readme.js";
 import { loadSpecs } from "./spec.js";
-import { writeStore } from "./store.js";
-const DEFAULT_BATCH = 96;
+import { openStore } from "./store.js";
 export async function buildIndex(options) {
     const journal = beginBuild({
         dir: options.out,
@@ -15,6 +15,15 @@ export async function buildIndex(options) {
         phases: PHASES,
         report: SCHEMA_REPORT,
     });
+    const ref = options.embeddingRef ?? options.embedder.id;
+    const cache = options.cache === false
+        ? NO_CACHE
+        : openCache(options.embedder, {
+            ref,
+            dir: options.cacheDir,
+            dimensions: options.dimensions,
+        });
+    let writer;
     try {
         const corpus = await loadSpecs(options.files);
         journal.phase('graph');
@@ -33,18 +42,20 @@ export async function buildIndex(options) {
         journal.read(summary.counts, summary.counts.entities);
         options.onRead?.(summary);
         journal.phase('embedding');
-        const vectors = await embedAll(entities, options, journal);
+        writer = await openStore(options.out);
+        const dimensions = await embedAll(entities, options, journal, cache, writer);
         journal.phase('writing');
-        const written = await writeStore(options.out, entities, vectors);
+        const written = await writer.finish();
         const manifest = {
             version: INDEX_VERSION,
             kind: 'schema',
             createdAt: new Date().toISOString(),
             indexer: options.indexer,
             embedding: {
-                ref: options.embeddingRef ?? options.embedder.id,
+                ref,
                 id: options.embedder.id,
-                dimensions: vectors[0]?.length ?? 0,
+                dimensions,
+                requested: options.dimensions,
             },
             sources: summary.sources,
             counts: summary.counts,
@@ -57,32 +68,33 @@ export async function buildIndex(options) {
             operations: corpus.operations,
             documents: keep ? corpus.documents : {},
         });
+        cache.commit();
         journal.finish(manifest);
-        return { manifest, entities };
+        return { manifest, entities, timings: journal.timings, reused: cache.hits };
     }
     catch (err) {
+        writer?.close();
+        cache.abandon();
         journal.fail(err);
         throw err;
     }
 }
-async function embedAll(entities, options, journal) {
-    const size = options.batch ?? DEFAULT_BATCH;
-    const out = [];
-    for (let at = 0; at < entities.length; at += size) {
-        const slice = entities.slice(at, at + size);
-        const response = await options.embedder.embed({
-            input: slice.map((e) => e.text),
-            taskType: 'document',
-            signal: options.signal,
-        });
-        if (response.vectors.length !== slice.length) {
-            throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
-        }
-        out.push(...response.vectors.map((v) => Float32Array.from(v)));
-        journal.progress(out.length, entities.length);
-        options.onProgress?.(out.length, entities.length);
-    }
-    return out;
+async function embedAll(entities, options, journal, cache, writer) {
+    // A window at a time. How many texts fit in a request, and how many
+    // requests may be in flight, are still the embedder's to answer — it knows
+    // the model's caps and it is the one that sees the 429s.
+    return embedStream({
+        embedder: options.embedder,
+        cache,
+        records: entities,
+        textOf: (entity) => entity.text,
+        signal: options.signal,
+        onProgress: (done, total) => {
+            journal.progress(done, total);
+            options.onProgress?.(done, total);
+        },
+        onWindow: (window, vectors) => writer.add(window, vectors),
+    });
 }
 /**
  * Counted off the entities rather than the corpus, so a schema the document

package/dist/schema/command.js

@@ -1,9 +1,10 @@
-import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, table, usageError, write, } from '@zenera/cli/lib';
+import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
 import { relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
 import { assertSameEmbedding } from "../common/manifest.js";
 import { isGlob, loose, matcher, PatternError, wildcard } from "../common/match.js";
+import { breakdown } from "../common/prose.js";
 import { buildIndex } from "./build.js";
 import { openIndex, readManifest, readSource, SCHEMA_INDEX } from "./files.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
@@ -68,11 +69,17 @@ export const command = {
                 '  -o, --out <dir>',
                 dim(`Where the index goes. Default ${DEFAULT_DIR}, or ${DIR_ENV}.`),
             ],
+            ['  --batch <n>', dim("Texts per embedding request. Default: the model's own cap.")],
             [
-                '  --batch <n>',
-                dim('Texts per embedding request, and how often progress prints. Default 96.'),
+                '  --dimensions <n>',
+                dim("Narrower vectors, if the model allows it. Default: the model's own width."),
             ],
             ['  --no-sources', dim('Do not keep a copy of each document in the index.')],
+            [
+                '  --no-cache',
+                dim('Embed everything again, ignoring vectors this machine already has.'),
+            ],
+            ['  --cache-dir <dir>', dim('Keep the vectors somewhere other than the shared cache.')],
         ]),
         '',
         'Search terms (repeatable)',
@@ -99,6 +106,7 @@ export const command = {
             ['  --exclude-method <name>', dim('Drop an operation by name.')],
             ['  --exclude-type <name>', dim('Drop a schema by name.')],
             ['  --exclude-property <name>', dim('Drop a field by name.')],
+            ['  --source <name>', dim('Only this document, as `stats` names it. Repeatable.')],
             ['  --limit <n>', dim('Seeds kept per term. Default 5.')],
             ['  --max-hops <n>', dim('How far apart two hits may be. Default 3.')],
             ['  --max-nodes <n>', dim('Nodes per result. Default 200.')],
@@ -188,34 +196,48 @@ async function index(args, ctx) {
         out: { type: 'string', short: 'o' },
         embedding: { type: 'string' },
         batch: { type: 'string' },
+        dimensions: { type: 'string' },
         'no-sources': { type: 'boolean' },
+        'no-cache': { type: 'boolean' },
+        'cache-dir': { type: 'string' },
         quiet: { type: 'boolean' },
     }, INDEX_USAGE);
     if (positionals.length === 0) {
         throw usageError('no document given', INDEX_USAGE);
     }
     const out = outputDir(ctx.cwd, values.out, SCHEMA_INDEX);
+    const cacheDir = values['cache-dir'] ? resolve(ctx.cwd, values['cache-dir']) : paths.cache();
     const loud = !values.quiet && !ctx.json;
-    const chosen = await resolveEmbedder(values.embedding);
+    // Undefined when unasked, all the way to the cache key: a width nobody
+    // named is not the same key as the width the model happens to default to,
+    // and resolving it here would miss every vector already paid for.
+    const dimensions = values.dimensions ? count(values.dimensions, '--dimensions') : undefined;
+    const chosen = await resolveEmbedder(values.embedding, {
+        maxBatch: values.batch ? count(values.batch, '--batch') : undefined,
+        dimensions,
+    });
     const started = Date.now();
-    const { manifest } = await buildIndex({
+    const { manifest, timings, reused } = await buildIndex({
         files: positionals.map((file) => resolve(ctx.cwd, file)),
         out,
         embedder: chosen,
         embeddingRef: values.embedding,
         indexer: 'zenera-rag',
-        batch: values.batch ? count(values.batch, '--batch') : undefined,
         sources: !values['no-sources'],
+        dimensions,
+        cache: !values['no-cache'],
+        cacheDir: values['cache-dir'] ? cacheDir : undefined,
         onRead: loud
             ? (summary) => {
                 printSources(summary.sources);
-                // The first batch can take a while and says nothing while it
-                // does; this is the line that makes that a wait, not a hang.
+                // Embedding is one call now, and a long one; this is the line
+                // that makes the wait before the first progress report a wait
+                // rather than a hang.
                 note(dim(`  embedding ${summary.counts.entities} entities with ${chosen.id} …`));
             }
             : undefined,
         onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
+            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
             : undefined,
     });
     if (ctx.json) {
@@ -228,6 +250,9 @@ async function index(args, ctx) {
     write(out);
     note(`  wrote ${bold(String(manifest.counts.entities))} entities to ${bold(out)}, ` +
         `embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
+    note(dim(`  ${breakdown(timings)}`));
+    note(dim(`  reused ${reused}/${manifest.counts.entities} vectors` +
+        `${values['no-cache'] ? ' (--no-cache)' : ` from ${cacheDir}`}`));
     const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
     note(dim(`  search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
 }
@@ -282,6 +307,7 @@ const SEARCH_OPTIONS = {
     'exclude-method': MANY,
     'exclude-type': MANY,
     'exclude-property': MANY,
+    source: MANY,
     limit: { type: 'string' },
     'max-hops': { type: 'string' },
     'max-nodes': { type: 'string' },
@@ -313,7 +339,9 @@ async function search(args, ctx) {
     const manifest = await readManifest(dir);
     const ref = values.embedding ?? manifest.embedding.ref;
     assertSameEmbedding(manifest, ref);
-    const index = await SchemaIndex.open(dir, await resolveEmbedder(ref));
+    // A query has to be asked at the width the entities were written at, and
+    // the ref alone does not say what that was.
+    const index = await SchemaIndex.open(dir, await resolveEmbedder(ref, { dimensions: manifest.embedding.requested }));
     try {
         if (values.interactive) {
             await repl(index, query, { format, ...options });
@@ -362,6 +390,7 @@ function fromFlags(values, positionals = []) {
     put('exclude_methods', values['exclude-method']);
     put('exclude_types', values['exclude-type']);
     put('exclude_properties', values['exclude-property']);
+    put('sources', values.source);
     put('direction', values.direction);
     put('method_type', values['method-type']);
     put('limit', values.limit && count(values.limit, '--limit'));

package/dist/schema/query.js

@@ -19,6 +19,7 @@ const LISTS = [
     'exclude_methods',
     'exclude_types',
     'exclude_properties',
+    'sources',
 ];
 const NUMBERS = ['limit', 'max_hops', 'max_nodes'];
 const DIRECTIONS = ['input', 'output', 'any'];

package/dist/schema/readme.js

@@ -1,6 +1,6 @@
 import { basename } from 'node:path';
 import { INTERVAL_MS } from "../common/progress.js";
-import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+import { breakdown, duration, fields, grid, message, plural, searched } from "../common/prose.js";
 export const PHASES = {
     reading: 'reading the documents',
     graph: 'building the graph',
@@ -22,9 +22,10 @@ function building(state) {
         rows.push(['found', entities(state.summary)]);
     }
     if (state.total > 0) {
-        const percent = Math.round((state.done / state.total) * 100);
+        const percent = Math.floor((state.done / state.total) * 100);
         rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
     }
+    rows.push(['timing', breakdown(state.timings)]);
     rows.push(['updated', new Date(state.now).toISOString()]);
     return [
         '# Schema index — being built',
@@ -60,6 +61,8 @@ function complete(state) {
         `${entities(manifest.counts)},`,
         `${searched(manifest.indexes)}.`,
         '',
+        `Time: ${breakdown(state.timings)}.`,
+        '',
         '## Files',
         '',
         ...fields([
@@ -97,6 +100,7 @@ function failed(state) {
             ['step', state.step],
             ['reason', message(state.reason)],
             ['started', new Date(state.started).toISOString()],
+            ['timing', breakdown(state.timings)],
             [
                 'failed',
                 `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,

package/dist/schema/search.d.ts

@@ -24,6 +24,8 @@ export interface SchemaQuery {
     exclude_methods?: readonly string[];
     exclude_types?: readonly string[];
     exclude_properties?: readonly string[];
+    /** document names, as `stats` prints them; any one of them is enough */
+    sources?: readonly string[];
     /** seeds kept per query string */
     limit?: number;
     max_hops?: number;