@zenera/rag 1.1.3 → 1.1.5

package/dist/schema/progress.js

@@ -0,0 +1,316 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { basename, join } from 'node:path';
+// ---------------------------------------------------------------------------
+// A build that says what it is doing
+//
+// Indexing a large document is minutes of silence with a directory slowly
+// filling up, and the directory is the only thing a second person — or a second
+// process, or an agent reading the tree — ever sees. So the build writes two
+// files into it and keeps them true:
+//
+//   .lock       who is building this, right now. Gone when nothing is.
+//   README.md   a progress report while it runs, and a description of what the
+//               index holds once it does not.
+//
+// Nothing in either file is an absolute path. A project's `assets/` directory
+// is mounted at `/assets` inside an agent's sandbox, so an index built there is
+// read under a name this process never sees; a host path would be a lie there.
+// ---------------------------------------------------------------------------
+export const LOCK_FILE = '.lock';
+export const README_FILE = 'README.md';
+/** The floor on how often README.md is rewritten. */
+const INTERVAL_MS = 5000;
+const PHASES = {
+    reading: 'reading the documents',
+    graph: 'building the graph',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export function beginBuild(plan) {
+    const documents = plan.files.map((file) => basename(file));
+    const started = Date.now();
+    const lock = {
+        pid: process.pid,
+        host: hostname(),
+        startedAt: new Date(started).toISOString(),
+        indexer: plan.indexer,
+        embedding: plan.embedding,
+        documents,
+    };
+    mkdirSync(plan.dir, { recursive: true });
+    claim(join(plan.dir, LOCK_FILE), lock);
+    let phase = 'reading';
+    let counts;
+    let done = 0;
+    let total = 0;
+    let wroteAt = 0;
+    let closed = false;
+    const write = (body) => {
+        // Through a temp name, so a reader never catches half a file.
+        const target = join(plan.dir, README_FILE);
+        const temp = `${target}.tmp`;
+        writeFileSync(temp, body);
+        renameSync(temp, target);
+        wroteAt = Date.now();
+    };
+    const report = () => write(building({
+        documents,
+        embedding: plan.embedding,
+        started,
+        phase,
+        done,
+        total,
+        counts,
+    }));
+    const maybe = () => {
+        if (!closed && Date.now() - wroteAt >= INTERVAL_MS) {
+            report();
+        }
+    };
+    // The first embedding request can block for ten seconds or more, so a purely
+    // event-driven throttle would leave the elapsed line frozen through it.
+    const ticker = setInterval(maybe, INTERVAL_MS);
+    ticker.unref();
+    const close = (body) => {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        clearInterval(ticker);
+        write(body);
+        rmSync(join(plan.dir, LOCK_FILE), { force: true });
+    };
+    report();
+    return {
+        phase(name) {
+            phase = name;
+            maybe();
+        },
+        read(seen) {
+            counts = seen;
+            total = seen.entities;
+            maybe();
+        },
+        progress(at, of) {
+            done = at;
+            total = of;
+            maybe();
+        },
+        finish(manifest) {
+            close(complete(plan.dir, manifest, Date.now() - started));
+        },
+        fail(reason) {
+            close(failed({ documents, phase, reason, started }));
+        },
+    };
+}
+/**
+ * `wx` makes the create and the check one operation, so two builds racing for
+ * the same directory cannot both win. A lock whose process is gone is stale by
+ * definition and is taken rather than respected — a crashed build must not make
+ * a directory permanently unbuildable.
+ */
+function claim(path, lock) {
+    const body = `${JSON.stringify(lock, null, 4)}\n`;
+    try {
+        writeFileSync(path, body, { flag: 'wx' });
+        return;
+    }
+    catch (err) {
+        if (err.code !== 'EEXIST') {
+            throw err;
+        }
+    }
+    const held = readLock(path);
+    if (held && held.host === hostname() && alive(held.pid)) {
+        throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
+    }
+    writeFileSync(path, body);
+}
+function readLock(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
+ * EPERM means it exists and belongs to someone else, which still counts.
+ */
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+function building(state) {
+    const now = Date.now();
+    const rows = [
+        ['documents', state.documents.join(', ')],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(now - state.started)} ago)`,
+        ],
+        ['step', PHASES[state.phase]],
+    ];
+    if (state.counts) {
+        rows.push(['found', entities(state.counts)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(now).toISOString()]);
+    return [
+        '# Schema index — being built',
+        '',
+        'A searchable index of the API documents named below, written by `zen rag schema index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(dir, manifest, ms) {
+    const titles = manifest.sources.map((s) => s.title).filter(Boolean);
+    const what = titles.length > 0 ? titles.join(', ') : basename(dir);
+    return [
+        `# Schema index — ${what}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(ms)}.`,
+        'Ask it for the operations and types behind a question and it answers with a subgraph:',
+        'the endpoints that match, the schemas they carry, and the fields inside those — printed',
+        'as text, Mermaid, TypeScript or OpenAPI.',
+        '',
+        '## What it covers',
+        '',
+        ...sourceTable(manifest.sources),
+        '',
+        `${entities(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['graph.json', 'the nodes and edges: operations, types, fields'],
+            ['schemas.json', 'the JSON Schema of every type'],
+            ['operations.json', 'every operation, with its parameters and responses'],
+            ...(manifest.sources.some((s) => s.path)
+                ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
+                : []),
+            ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag schema search --dir . --all "how do I cancel a subscription"',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Schema index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag schema index`.',
+        '',
+        ...fields([
+            ['documents', state.documents.join(', ')],
+            ['step', PHASES[state.phase]],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+// Small renderings
+// ---------------------------------------------------------------------------
+/** An indented block, which markdown renders verbatim and an agent reads as a table. */
+function fields(rows) {
+    const width = Math.max(...rows.map((r) => (r[0] ?? '').length));
+    return rows.map(([name, value]) => `    ${(name ?? '').padEnd(width)}  ${value ?? ''}`);
+}
+const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
+function sourceTable(sources) {
+    const rows = sources.map((s) => [
+        s.file,
+        s.dialect,
+        String(s.paths),
+        String(s.methods),
+        String(s.types),
+        String(s.properties),
+    ]);
+    const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
+    const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' | ')} |`;
+    return [
+        line(HEADERS),
+        `| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`,
+        ...rows.map(line),
+    ];
+}
+function entities(counts) {
+    return (`${plural(counts.entities, 'entity', 'entities')}: ` +
+        `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
+}
+function searched(indexes) {
+    if (!indexes.fts && !indexes.vector) {
+        return 'scanned flat: neither index was built';
+    }
+    if (!indexes.vector) {
+        // Below a couple of thousand rows an IVF index has nothing to train on.
+        return 'searched by full text and by vector, the latter as a flat scan';
+    }
+    return indexes.fts
+        ? 'searched by full text and by vector'
+        : 'searched by vector, with no full-text index';
+}
+function plural(n, one, many = `${one}s`) {
+    return `${n} ${n === 1 ? one : many}`;
+}
+function duration(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 1) {
+        return 'under a second';
+    }
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    return minutes < 60
+        ? `${minutes}m${seconds % 60}s`
+        : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
+}
+function message(reason) {
+    const text = reason instanceof Error ? reason.message : String(reason);
+    return text.split('\n')[0]?.trim() || 'no reason given';
+}
+//# sourceMappingURL=progress.js.map

package/dist/schema/spec.d.ts

@@ -36,7 +36,10 @@ export interface Operation {
     responses: ResponseSpec[];
 }
 export interface ApiDoc {
+    /** the document's name within this index, unique in the corpus */
     source: string;
+    /** what the file was called on the machine that read it */
+    file: string;
     sha256: string;
     dialect: Dialect;
     title: string;
@@ -49,6 +52,8 @@ export interface Corpus {
     types: Record<string, Schema>;
     /** which document each type id came from */
     typeSource: Record<string, string>;
+    /** the bundled document behind each `source`, as JSON text */
+    documents: Record<string, string>;
 }
 /** A `CliError` so an unreadable document exits 3 wherever it is raised. */
 export declare class SpecError extends CliError {

package/dist/schema/spec.js

@@ -1,7 +1,7 @@
 import SwaggerParser from '@apidevtools/swagger-parser';
+import { CliError, EXIT } from '@zenera/cli/lib';
 import { createHash } from 'node:crypto';
 import { basename, extname } from 'node:path';
-import { CliError, EXIT } from '@zenera/cli/lib';
 import { docOf, isObject, normalize } from "./schema.js";
 // ---------------------------------------------------------------------------
 // Documents, flattened — but not dereferenced
@@ -38,12 +38,26 @@ export async function loadSpecs(files) {
         throw new SpecError('no document given', 'name at least one openapi/swagger file');
     }
     const loaded = [];
+    const taken = new Set();
     for (const file of files) {
-        loaded.push(await loadSpec(file));
+        loaded.push(await loadSpec(file, distinct(slugOf(file), taken)));
     }
     return settle(loaded);
 }
-async function loadSpec(file) {
+/**
+ * The slug is the document's identity everywhere below this file — the node
+ * attribute, the store column, the name its copy is written under — so two
+ * `api.yaml`s in different directories must not answer to the same word.
+ */
+function distinct(slug, taken) {
+    let candidate = slug;
+    for (let n = 2; taken.has(candidate); n++) {
+        candidate = `${slug}_${n}`;
+    }
+    taken.add(candidate);
+    return candidate;
+}
+async function loadSpec(file, slug) {
     let raw;
     try {
         raw = (await SwaggerParser.bundle(file));
@@ -51,6 +65,8 @@ async function loadSpec(file) {
     catch (err) {
         throw new SpecError(`${file}: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`, 'the document must be a readable OpenAPI 3.x or Swagger 2.0 file');
     }
+    // `bundle` has resolved every external `$ref`, so this text stands alone.
+    const document = JSON.stringify(raw, null, 2);
     const dialect = dialectOf(raw, file);
     const source = raw.components?.schemas ?? raw.definitions ?? {};
     const types = new Map();
@@ -59,15 +75,17 @@ async function loadSpec(file) {
     }
     return {
         doc: {
-            source: file,
-            sha256: createHash('sha256').update(JSON.stringify(raw)).digest('hex'),
+            source: slug,
+            file: basename(file),
+            sha256: createHash('sha256').update(document).digest('hex'),
             dialect,
             title: raw.info?.title?.trim() || basename(file),
             version: raw.info?.version?.trim() || '',
         },
-        slug: slugOf(file),
-        operations: operationsOf(raw, dialect, file),
+        slug,
+        operations: operationsOf(raw, dialect, slug),
         types,
+        document,
     };
 }
 function dialectOf(doc, file) {
@@ -89,7 +107,7 @@ function slugOf(file) {
 // ---------------------------------------------------------------------------
 // Operations
 // ---------------------------------------------------------------------------
-function operationsOf(raw, dialect, file) {
+function operationsOf(raw, dialect, source) {
     const prefix = dialect === 'swagger-2.0' ? (raw.basePath ?? '') : '';
     const out = [];
     for (const [template, item] of Object.entries(raw.paths ?? {})) {
@@ -107,7 +125,7 @@ function operationsOf(raw, dialect, file) {
             const path = join(prefix, template);
             const own = paramsOf(op.parameters, dialect);
             out.push({
-                source: file,
+                source,
                 method,
                 path,
                 operationId: text(op.operationId) || synthesizeId(method, path),
@@ -275,7 +293,13 @@ function settle(loaded) {
             });
         }
     });
-    return { docs: loaded.map((one) => one.doc), operations, types, typeSource };
+    return {
+        docs: loaded.map((one) => one.doc),
+        operations,
+        types,
+        typeSource,
+        documents: Object.fromEntries(loaded.map((one) => [one.slug, one.document])),
+    };
 }
 function unique(id, slug, seen) {
     let candidate = seen.has(id) ? `${slug}.${id}` : id;

package/dist/schema/subgraph.d.ts

@@ -37,6 +37,12 @@ export interface StitchOptions {
 export declare const DEFAULT_MAX_HOPS = 3;
 export declare const DEFAULT_MAX_NODES = 200;
 export declare function stitch(graph: ApiGraph, seeds: readonly Seed[], options?: StitchOptions): Subgraph[];
+/**
+ * Exactly the nodes named, and only the edges between them. The counterpart to
+ * `stitch`: no neighbours are gathered, because the caller is not asking what
+ * this is connected to — they already know what they want and want it whole.
+ */
+export declare function select(graph: ApiGraph, ids: readonly string[]): Subgraph;
 /** Breadth-first, ignoring edge direction, giving up past `maxHops`. */
 export declare function path(graph: ApiGraph, from: string, to: string, maxHops: number): string[] | undefined;
 //# sourceMappingURL=subgraph.d.ts.map

package/dist/schema/subgraph.js

@@ -150,6 +150,33 @@ function connect(graph, seeds, maxHops) {
     }
     return out;
 }
+/**
+ * Exactly the nodes named, and only the edges between them. The counterpart to
+ * `stitch`: no neighbours are gathered, because the caller is not asking what
+ * this is connected to — they already know what they want and want it whole.
+ */
+export function select(graph, ids) {
+    const kept = new Set(ids.filter((id) => graph.hasNode(id)));
+    const nodes = [...kept].map((id) => ({
+        id,
+        kind: graph.getNodeAttribute(id, 'kind'),
+        attributes: graph.getNodeAttributes(id),
+        hit: true,
+        score: 1,
+    }));
+    const edges = [];
+    for (const id of kept) {
+        for (const edge of graph.outEdges(id)) {
+            const target = graph.target(edge);
+            if (!kept.has(target)) {
+                continue;
+            }
+            const a = graph.getEdgeAttributes(edge);
+            edges.push({ source: id, target, relation: a.relation, status: a.status, in: a.in });
+        }
+    }
+    return { nodes, edges, hits: [...kept], score: kept.size, truncated: false };
+}
 /** Breadth-first, ignoring edge direction, giving up past `maxHops`. */
 export function path(graph, from, to, maxHops) {
     if (from === to) {