@zenera/rag 1.1.24 → 1.1.26

package/dist/schema/match.js

@@ -0,0 +1,92 @@
+// ---------------------------------------------------------------------------
+// Matching, with nothing learned in between
+//
+// Everything here is exact. A glob matches the characters it names and a
+// substring is a substring, which is the whole point of the surfaces built on
+// it: a vector index answers "what is near this", and near is a ranking, so it
+// can only ever return the top of a list. When the question is "does the word
+// `password` appear anywhere at all", a ranking is the wrong instrument and no
+// amount of tuning makes it the right one.
+//
+// A pattern may arrive from a model, so a regex is a bounded promise: the
+// length is capped here and the scan that uses it keeps a deadline.
+// ---------------------------------------------------------------------------
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export const MAX_PATTERN = 200;
+export class PatternError extends Error {
+}
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export function matcher(pattern, options = {}) {
+    guard(pattern);
+    if (!options.regex) {
+        if (options.caseSensitive) {
+            return (text) => text.includes(pattern);
+        }
+        const needle = pattern.toLowerCase();
+        return (text) => text.toLowerCase().includes(needle);
+    }
+    const expression = compile(pattern, options.caseSensitive ? '' : 'i');
+    // `lastIndex` is not carried between calls: the flags never include `g`.
+    return (text) => expression.test(text);
+}
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export function wildcard(pattern, options = {}) {
+    guard(pattern);
+    const source = [...pattern]
+        .map((char) => (char === '*' ? '.*' : char === '?' ? '.' : escape(char)))
+        .join('');
+    const expression = compile(`^${source}$`, options.caseSensitive ? '' : 'i');
+    return (text) => expression.test(text);
+}
+/** Whether a pattern is asking to be read as a glob at all. */
+export const isGlob = (pattern) => /[*?]/.test(pattern);
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ *
+ * `regex` settles it outright, and has to: a star is punctuation in both
+ * languages, so `^/(users|teams)/.*` read as a glob would match nothing and
+ * never say why.
+ */
+export function loose(pattern, options = {}) {
+    if (options.regex) {
+        return matcher(pattern, options);
+    }
+    return isGlob(pattern) ? wildcard(pattern, options) : matcher(pattern, options);
+}
+/** True when any of the patterns matches; no patterns means no opinion. */
+export function anyOf(matchers) {
+    if (matchers.length === 0) {
+        return undefined;
+    }
+    return (text) => matchers.some((match) => match(text));
+}
+// ---------------------------------------------------------------------------
+function guard(pattern) {
+    if (pattern.length === 0) {
+        throw new PatternError('the pattern is empty');
+    }
+    if (pattern.length > MAX_PATTERN) {
+        throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
+    }
+}
+function compile(source, flags) {
+    try {
+        return new RegExp(source, flags);
+    }
+    catch (err) {
+        throw new PatternError(`invalid pattern: ${err.message}`);
+    }
+}
+const escape = (char) => char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+//# sourceMappingURL=match.js.map

package/dist/schema/progress.d.ts

@@ -0,0 +1,26 @@
+import type { Counts, Manifest } from './files.ts';
+export declare const LOCK_FILE = ".lock";
+export declare const README_FILE = "README.md";
+export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
+export interface BuildPlan {
+    /** where the index goes */
+    dir: string;
+    files: readonly string[];
+    /** the embedding reference as it was typed */
+    embedding: string;
+    indexer: string;
+}
+export interface Journal {
+    phase(name: Phase): void;
+    /** what the documents turned out to hold, once they have been read */
+    read(counts: Counts): void;
+    progress(done: number, total: number): void;
+    finish(manifest: Manifest): void;
+    fail(reason: unknown): void;
+}
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export declare function beginBuild(plan: BuildPlan): Journal;
+//# sourceMappingURL=progress.d.ts.map

package/dist/schema/progress.js

@@ -0,0 +1,316 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { basename, join } from 'node:path';
+// ---------------------------------------------------------------------------
+// A build that says what it is doing
+//
+// Indexing a large document is minutes of silence with a directory slowly
+// filling up, and the directory is the only thing a second person — or a second
+// process, or an agent reading the tree — ever sees. So the build writes two
+// files into it and keeps them true:
+//
+//   .lock       who is building this, right now. Gone when nothing is.
+//   README.md   a progress report while it runs, and a description of what the
+//               index holds once it does not.
+//
+// Nothing in either file is an absolute path. A project's `assets/` directory
+// is mounted at `/assets` inside an agent's sandbox, so an index built there is
+// read under a name this process never sees; a host path would be a lie there.
+// ---------------------------------------------------------------------------
+export const LOCK_FILE = '.lock';
+export const README_FILE = 'README.md';
+/** The floor on how often README.md is rewritten. */
+const INTERVAL_MS = 5000;
+const PHASES = {
+    reading: 'reading the documents',
+    graph: 'building the graph',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export function beginBuild(plan) {
+    const documents = plan.files.map((file) => basename(file));
+    const started = Date.now();
+    const lock = {
+        pid: process.pid,
+        host: hostname(),
+        startedAt: new Date(started).toISOString(),
+        indexer: plan.indexer,
+        embedding: plan.embedding,
+        documents,
+    };
+    mkdirSync(plan.dir, { recursive: true });
+    claim(join(plan.dir, LOCK_FILE), lock);
+    let phase = 'reading';
+    let counts;
+    let done = 0;
+    let total = 0;
+    let wroteAt = 0;
+    let closed = false;
+    const write = (body) => {
+        // Through a temp name, so a reader never catches half a file.
+        const target = join(plan.dir, README_FILE);
+        const temp = `${target}.tmp`;
+        writeFileSync(temp, body);
+        renameSync(temp, target);
+        wroteAt = Date.now();
+    };
+    const report = () => write(building({
+        documents,
+        embedding: plan.embedding,
+        started,
+        phase,
+        done,
+        total,
+        counts,
+    }));
+    const maybe = () => {
+        if (!closed && Date.now() - wroteAt >= INTERVAL_MS) {
+            report();
+        }
+    };
+    // The first embedding request can block for ten seconds or more, so a purely
+    // event-driven throttle would leave the elapsed line frozen through it.
+    const ticker = setInterval(maybe, INTERVAL_MS);
+    ticker.unref();
+    const close = (body) => {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        clearInterval(ticker);
+        write(body);
+        rmSync(join(plan.dir, LOCK_FILE), { force: true });
+    };
+    report();
+    return {
+        phase(name) {
+            phase = name;
+            maybe();
+        },
+        read(seen) {
+            counts = seen;
+            total = seen.entities;
+            maybe();
+        },
+        progress(at, of) {
+            done = at;
+            total = of;
+            maybe();
+        },
+        finish(manifest) {
+            close(complete(plan.dir, manifest, Date.now() - started));
+        },
+        fail(reason) {
+            close(failed({ documents, phase, reason, started }));
+        },
+    };
+}
+/**
+ * `wx` makes the create and the check one operation, so two builds racing for
+ * the same directory cannot both win. A lock whose process is gone is stale by
+ * definition and is taken rather than respected — a crashed build must not make
+ * a directory permanently unbuildable.
+ */
+function claim(path, lock) {
+    const body = `${JSON.stringify(lock, null, 4)}\n`;
+    try {
+        writeFileSync(path, body, { flag: 'wx' });
+        return;
+    }
+    catch (err) {
+        if (err.code !== 'EEXIST') {
+            throw err;
+        }
+    }
+    const held = readLock(path);
+    if (held && held.host === hostname() && alive(held.pid)) {
+        throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
+    }
+    writeFileSync(path, body);
+}
+function readLock(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
+ * EPERM means it exists and belongs to someone else, which still counts.
+ */
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+function building(state) {
+    const now = Date.now();
+    const rows = [
+        ['documents', state.documents.join(', ')],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(now - state.started)} ago)`,
+        ],
+        ['step', PHASES[state.phase]],
+    ];
+    if (state.counts) {
+        rows.push(['found', entities(state.counts)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(now).toISOString()]);
+    return [
+        '# Schema index — being built',
+        '',
+        'A searchable index of the API documents named below, written by `zen rag schema index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(dir, manifest, ms) {
+    const titles = manifest.sources.map((s) => s.title).filter(Boolean);
+    const what = titles.length > 0 ? titles.join(', ') : basename(dir);
+    return [
+        `# Schema index — ${what}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(ms)}.`,
+        'Ask it for the operations and types behind a question and it answers with a subgraph:',
+        'the endpoints that match, the schemas they carry, and the fields inside those — printed',
+        'as text, Mermaid, TypeScript or OpenAPI.',
+        '',
+        '## What it covers',
+        '',
+        ...sourceTable(manifest.sources),
+        '',
+        `${entities(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['graph.json', 'the nodes and edges: operations, types, fields'],
+            ['schemas.json', 'the JSON Schema of every type'],
+            ['operations.json', 'every operation, with its parameters and responses'],
+            ...(manifest.sources.some((s) => s.path)
+                ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
+                : []),
+            ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag schema search --dir . --all "how do I cancel a subscription"',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Schema index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag schema index`.',
+        '',
+        ...fields([
+            ['documents', state.documents.join(', ')],
+            ['step', PHASES[state.phase]],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+// Small renderings
+// ---------------------------------------------------------------------------
+/** An indented block, which markdown renders verbatim and an agent reads as a table. */
+function fields(rows) {
+    const width = Math.max(...rows.map((r) => (r[0] ?? '').length));
+    return rows.map(([name, value]) => `    ${(name ?? '').padEnd(width)}  ${value ?? ''}`);
+}
+const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
+function sourceTable(sources) {
+    const rows = sources.map((s) => [
+        s.file,
+        s.dialect,
+        String(s.paths),
+        String(s.methods),
+        String(s.types),
+        String(s.properties),
+    ]);
+    const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
+    const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' | ')} |`;
+    return [
+        line(HEADERS),
+        `| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`,
+        ...rows.map(line),
+    ];
+}
+function entities(counts) {
+    return (`${plural(counts.entities, 'entity', 'entities')}: ` +
+        `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
+}
+function searched(indexes) {
+    if (!indexes.fts && !indexes.vector) {
+        return 'scanned flat: neither index was built';
+    }
+    if (!indexes.vector) {
+        // Below a couple of thousand rows an IVF index has nothing to train on.
+        return 'searched by full text and by vector, the latter as a flat scan';
+    }
+    return indexes.fts
+        ? 'searched by full text and by vector'
+        : 'searched by vector, with no full-text index';
+}
+function plural(n, one, many = `${one}s`) {
+    return `${n} ${n === 1 ? one : many}`;
+}
+function duration(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 1) {
+        return 'under a second';
+    }
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    return minutes < 60
+        ? `${minutes}m${seconds % 60}s`
+        : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
+}
+function message(reason) {
+    const text = reason instanceof Error ? reason.message : String(reason);
+    return text.split('\n')[0]?.trim() || 'no reason given';
+}
+//# sourceMappingURL=progress.js.map

package/dist/schema/tools.js

@@ -1,5 +1,4 @@
-import { tool } from '@zenera/neo';
-import { loose, matcher, PatternError } from "../common/match.js";
+import { loose, matcher, PatternError, tool } from '@zenera/neo';
 import { toTypeScript } from "./hydrate.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
 import { FORMATS, isFormat, present } from "./present.js";

package/dist/schema/trace.d.ts

@@ -1,4 +1,4 @@
-import type { Matcher } from '../common/match.ts';
+import type { Matcher } from '@zenera/neo';
 import type { ApiGraph, NodeAttrs, NodeKind } from './graph.ts';
 export type Side = 'input' | 'output' | 'both';
 export interface Route {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.24",
+    "version": "1.1.26",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.24",
-        "@zenera/neo": "^1.1.24",
+        "@zenera/cli": "^1.1.26",
+        "@zenera/neo": "^1.1.26",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",