@zenera/rag 1.1.25 → 1.1.26

package/dist/common/match.d.ts

@@ -0,0 +1,40 @@
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export declare const MAX_PATTERN = 200;
+export declare class PatternError extends Error {
+}
+export interface MatchOptions {
+    /** read the pattern as a regular expression rather than as a literal */
+    regex?: boolean;
+    caseSensitive?: boolean;
+}
+export type Matcher = (text: string) => boolean;
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export declare function wildcard(pattern: string, options?: {
+    caseSensitive?: boolean;
+}): Matcher;
+/** Whether a pattern is asking to be read as a glob at all. */
+export declare const isGlob: (pattern: string) => boolean;
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ *
+ * `regex` settles it outright, and has to: a star is punctuation in both
+ * languages, so `^/(users|teams)/.*` read as a glob would match nothing and
+ * never say why.
+ */
+export declare function loose(pattern: string, options?: MatchOptions): Matcher;
+/** True when any of the patterns matches; no patterns means no opinion. */
+export declare function anyOf(matchers: readonly Matcher[]): Matcher | undefined;
+//# sourceMappingURL=match.d.ts.map

package/dist/common/match.js

@@ -0,0 +1,92 @@
+// ---------------------------------------------------------------------------
+// Matching, with nothing learned in between
+//
+// Everything here is exact. A glob matches the characters it names and a
+// substring is a substring, which is the whole point of the surfaces built on
+// it: a vector index answers "what is near this", and near is a ranking, so it
+// can only ever return the top of a list. When the question is "does the word
+// `password` appear anywhere at all", a ranking is the wrong instrument and no
+// amount of tuning makes it the right one.
+//
+// A pattern may arrive from a model, so a regex is a bounded promise: the
+// length is capped here and the scan that uses it keeps a deadline.
+// ---------------------------------------------------------------------------
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export const MAX_PATTERN = 200;
+export class PatternError extends Error {
+}
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export function matcher(pattern, options = {}) {
+    guard(pattern);
+    if (!options.regex) {
+        if (options.caseSensitive) {
+            return (text) => text.includes(pattern);
+        }
+        const needle = pattern.toLowerCase();
+        return (text) => text.toLowerCase().includes(needle);
+    }
+    const expression = compile(pattern, options.caseSensitive ? '' : 'i');
+    // `lastIndex` is not carried between calls: the flags never include `g`.
+    return (text) => expression.test(text);
+}
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export function wildcard(pattern, options = {}) {
+    guard(pattern);
+    const source = [...pattern]
+        .map((char) => (char === '*' ? '.*' : char === '?' ? '.' : escape(char)))
+        .join('');
+    const expression = compile(`^${source}$`, options.caseSensitive ? '' : 'i');
+    return (text) => expression.test(text);
+}
+/** Whether a pattern is asking to be read as a glob at all. */
+export const isGlob = (pattern) => /[*?]/.test(pattern);
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ *
+ * `regex` settles it outright, and has to: a star is punctuation in both
+ * languages, so `^/(users|teams)/.*` read as a glob would match nothing and
+ * never say why.
+ */
+export function loose(pattern, options = {}) {
+    if (options.regex) {
+        return matcher(pattern, options);
+    }
+    return isGlob(pattern) ? wildcard(pattern, options) : matcher(pattern, options);
+}
+/** True when any of the patterns matches; no patterns means no opinion. */
+export function anyOf(matchers) {
+    if (matchers.length === 0) {
+        return undefined;
+    }
+    return (text) => matchers.some((match) => match(text));
+}
+// ---------------------------------------------------------------------------
+function guard(pattern) {
+    if (pattern.length === 0) {
+        throw new PatternError('the pattern is empty');
+    }
+    if (pattern.length > MAX_PATTERN) {
+        throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
+    }
+}
+function compile(source, flags) {
+    try {
+        return new RegExp(source, flags);
+    }
+    catch (err) {
+        throw new PatternError(`invalid pattern: ${err.message}`);
+    }
+}
+const escape = (char) => char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+//# sourceMappingURL=match.js.map

package/dist/present.d.ts

@@ -0,0 +1,17 @@
+import { type HydrateOptions } from './schema/hydrate.ts';
+import { type RenderOptions } from './schema/render.ts';
+import type { Schema } from './schema/schema.ts';
+import type { Operation } from './schema/spec.ts';
+import type { Subgraph } from './schema/subgraph.ts';
+export declare const FORMATS: readonly ["text", "mermaid", "mermaid-flowchart", "ts", "openapi"];
+export type Format = (typeof FORMATS)[number];
+export declare function isFormat(value: string): value is Format;
+export interface OutputOptions extends RenderOptions, HydrateOptions {
+}
+/** Two of the four formats need what is on disk; an open index is enough. */
+export interface SchemaSource {
+    schemas(): Promise<Record<string, Schema>>;
+    operations(): Promise<Operation[]>;
+}
+export declare function present(index: SchemaSource, subgraphs: readonly Subgraph[], format: Format, options?: OutputOptions): Promise<string>;
+//# sourceMappingURL=present.d.ts.map

package/dist/present.js

@@ -0,0 +1,40 @@
+import { toOpenApi, toTypeScript } from "./schema/hydrate.js";
+import { render } from "./schema/render.js";
+// ---------------------------------------------------------------------------
+// One subgraph, in whichever of the four shapes was asked for
+//
+// The command, the prompt loop and the agent tool all print through here, so
+// `--format ts` from a shell and `format: "ts"` from a model are the same
+// bytes. Two of the four need the schemas off disk and two do not, which is
+// the only reason this is async.
+// ---------------------------------------------------------------------------
+export const FORMATS = ['text', 'mermaid', 'mermaid-flowchart', 'ts', 'openapi'];
+export function isFormat(value) {
+    return FORMATS.includes(value);
+}
+export async function present(index, subgraphs, format, options = {}) {
+    if (subgraphs.length === 0) {
+        return format === 'openapi' ? '{}' : '';
+    }
+    if (format === 'ts') {
+        const schemas = await index.schemas();
+        return subgraphs.map((s) => toTypeScript(s, schemas, options)).join('\n');
+    }
+    if (format === 'openapi') {
+        const [schemas, operations] = await Promise.all([index.schemas(), index.operations()]);
+        // One document for the whole answer: several would each have to repeat
+        // the components the others also touched.
+        const merged = subgraphs.reduce((all, one) => ({
+            nodes: [...all.nodes, ...one.nodes],
+            edges: [...all.edges, ...one.edges],
+            hits: [...all.hits, ...one.hits],
+            score: all.score + one.score,
+            truncated: all.truncated || one.truncated,
+        }), { nodes: [], edges: [], hits: [], score: 0, truncated: false });
+        return JSON.stringify(toOpenApi(merged, schemas, operations), null, 2);
+    }
+    return subgraphs
+        .map((s, at) => `${subgraphs.length > 1 ? `# result ${at + 1}\n` : ''}${render(s, format, options)}`)
+        .join('\n\n');
+}
+//# sourceMappingURL=present.js.map

package/dist/query.d.ts

@@ -0,0 +1,7 @@
+import type { SchemaQuery } from './schema/search.ts';
+export declare class QueryError extends Error {
+}
+export declare function parseQuery(value: unknown): SchemaQuery;
+/** Whether anything was actually asked, as opposed to only filtered. */
+export declare function isEmpty(query: SchemaQuery): boolean;
+//# sourceMappingURL=query.d.ts.map

package/dist/query.js

@@ -0,0 +1,88 @@
+// ---------------------------------------------------------------------------
+// A query, from somewhere untrusted
+//
+// `--query` and the tool both hand over a whole object, which means the shape
+// has to be checked rather than assumed. The rule is the same either way: a
+// key that is not known is an error, because a silently ignored
+// `output_propertys` looks exactly like a search that found nothing.
+// ---------------------------------------------------------------------------
+const LISTS = [
+    'all',
+    'methods',
+    'types',
+    'input_types',
+    'output_types',
+    'properties',
+    'input_properties',
+    'output_properties',
+    'exclude_ids',
+    'exclude_methods',
+    'exclude_types',
+    'exclude_properties',
+];
+const NUMBERS = ['limit', 'max_hops', 'max_nodes'];
+const DIRECTIONS = ['input', 'output', 'any'];
+const METHOD_TYPES = ['read_only', 'read_write', 'any'];
+const KNOWN = new Set([...LISTS, ...NUMBERS, 'direction', 'method_type']);
+export class QueryError extends Error {
+}
+export function parseQuery(value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        throw new QueryError('a query must be an object');
+    }
+    const input = value;
+    for (const key of Object.keys(input)) {
+        if (!KNOWN.has(key)) {
+            throw new QueryError(`unknown query field "${key}" — expected ${[...KNOWN].join(', ')}`);
+        }
+    }
+    const out = {};
+    for (const key of LISTS) {
+        const list = input[key];
+        if (list === undefined) {
+            continue;
+        }
+        if (!Array.isArray(list) || list.some((v) => typeof v !== 'string')) {
+            throw new QueryError(`${key} must be an array of strings`);
+        }
+        out[key] = list;
+    }
+    for (const key of NUMBERS) {
+        const number = input[key];
+        if (number === undefined) {
+            continue;
+        }
+        if (typeof number !== 'number' || !Number.isInteger(number) || number < 1) {
+            throw new QueryError(`${key} must be a whole number of at least 1`);
+        }
+        out[key] = number;
+    }
+    if (input.direction !== undefined) {
+        out.direction = oneOf('direction', input.direction, DIRECTIONS);
+    }
+    if (input.method_type !== undefined) {
+        out.method_type = oneOf('method_type', input.method_type, METHOD_TYPES);
+    }
+    return out;
+}
+/** Whether anything was actually asked, as opposed to only filtered. */
+export function isEmpty(query) {
+    const terms = [
+        query.all,
+        query.methods,
+        query.types,
+        query.input_types,
+        query.output_types,
+        query.properties,
+        query.input_properties,
+        query.output_properties,
+    ];
+    return terms.every((list) => !list || list.length === 0);
+}
+function oneOf(key, value, allowed) {
+    if (typeof value !== 'string' || !allowed.includes(value)) {
+        throw new QueryError(`${key} must be one of ${allowed.join(', ')}`);
+    }
+    return value;
+}
+//# sourceMappingURL=query.js.map

package/dist/repl.d.ts

@@ -0,0 +1,8 @@
+import { type Format, type OutputOptions } from './present.ts';
+import type { SchemaIndex, SchemaQuery } from './schema/search.ts';
+interface Settings extends OutputOptions {
+    format: Format;
+}
+export declare function repl(index: SchemaIndex, initial: SchemaQuery, settings: Settings): Promise<void>;
+export {};
+//# sourceMappingURL=repl.d.ts.map

package/dist/repl.js

@@ -0,0 +1,119 @@
+import { createInterface } from 'node:readline/promises';
+import { bold, cyan, dim, note, write } from '@zenera/cli/lib';
+import { isFormat, present } from "./present.js";
+import { isEmpty, parseQuery, QueryError } from "./query.js";
+const FIELDS = {
+    all: 'all',
+    method: 'methods',
+    type: 'types',
+    'input-type': 'input_types',
+    'output-type': 'output_types',
+    property: 'properties',
+    'input-property': 'input_properties',
+    'output-property': 'output_properties',
+};
+export async function repl(index, initial, settings) {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    let query = { ...initial };
+    let format = settings.format;
+    const options = { docs: settings.docs, onlyHits: settings.onlyHits };
+    note(bold(`${index.manifest.sources.map((s) => s.title).join(', ')}`));
+    note(dim('  a bare line searches every kind; `help` lists the rest.'));
+    try {
+        if (!isEmpty(query)) {
+            query = await run(index, query, format, options);
+        }
+        for (;;) {
+            const line = (await rl.question(cyan('rag> '))).trim();
+            if (line === '') {
+                continue;
+            }
+            if (line === 'quit' || line === 'exit') {
+                return;
+            }
+            if (line === 'help') {
+                help();
+                continue;
+            }
+            if (line === 'reset') {
+                query = {};
+                note(dim('  cleared, exclusions and all'));
+                continue;
+            }
+            if (line === 'show') {
+                note(dim(`  ${JSON.stringify(query)}`));
+                continue;
+            }
+            const [head, ...rest] = line.split(' ');
+            const tail = rest.join(' ').trim();
+            if (head === 'format') {
+                if (isFormat(tail)) {
+                    format = tail;
+                    note(dim(`  format is ${tail}`));
+                }
+                else {
+                    note(dim('  expected text, mermaid, mermaid-flowchart, ts or openapi'));
+                }
+                continue;
+            }
+            if (head === 'direction' || head === 'method-type') {
+                query = patch(query, head === 'direction' ? 'direction' : 'method_type', tail);
+                continue;
+            }
+            if (head && head in FIELDS && tail) {
+                query = { ...query, [FIELDS[head]]: [tail] };
+                query = await run(index, query, format, options);
+                continue;
+            }
+            query = await run(index, { ...query, all: [line] }, format, options);
+        }
+    }
+    finally {
+        rl.close();
+    }
+}
+/**
+ * Runs the search, prints it, and hands back the query with everything just
+ * shown added to the exclusions — so asking the same thing twice moves on
+ * instead of repeating itself.
+ */
+async function run(index, query, format, options) {
+    const result = await index.search(query);
+    const text = await present(index, result.subgraphs, format, options);
+    if (text) {
+        write(text);
+    }
+    note(dim(`  ${result.seeds.length} seed(s) · ${result.subgraphs.length} result(s)` +
+        (result.empty.length > 0 ? ` · nothing for: ${result.empty.join(', ')}` : '')));
+    if (result.seeds.length === 0) {
+        return query;
+    }
+    const seen = new Set([...(query.exclude_ids ?? []), ...result.seeds.map((s) => s.id)]);
+    note(dim(`  ${seen.size} node(s) now excluded — \`reset\` to see them again`));
+    return { ...query, exclude_ids: [...seen] };
+}
+function patch(query, key, value) {
+    try {
+        return { ...query, ...parseQuery({ [key]: value }) };
+    }
+    catch (err) {
+        note(dim(`  ${err instanceof QueryError ? err.message : String(err)}`));
+        return query;
+    }
+}
+function help() {
+    for (const line of [
+        '  <text>                  search everything',
+        '  all|method|type <text>  search one field',
+        '  input-property <text>   also: output-property, property, input-type, output-type',
+        '  direction <d>           input | output | any',
+        '  method-type <t>         read_only | read_write | any',
+        '  format <f>              text | mermaid | mermaid-flowchart | ts | openapi',
+        '  show                    the query as it stands',
+        '  reset                   forget it, exclusions included',
+        '  quit',
+    ]) {
+        note(dim(line));
+    }
+}
+//# sourceMappingURL=repl.js.map

package/dist/schema/locate.d.ts

@@ -0,0 +1,20 @@
+/** The name a new index is given. Nothing searches for it; only `index --out` writes it. */
+export declare const DEFAULT_DIR = "./schema-db";
+export declare const DIR_ENV = "ZEN_SCHEMA_DB";
+/** How the directory was arrived at, which is what decides whether to say so. */
+export type DirSource = 'flag' | 'env' | 'found' | 'default';
+export interface Located {
+    dir: string;
+    from: DirSource;
+}
+export interface LocateOptions {
+    env?: NodeJS.ProcessEnv;
+    /** do not climb above this; the home directory, or the root, by default */
+    ceiling?: string;
+}
+export declare function locateIndex(cwd: string, flag?: string, options?: LocateOptions): Located;
+/** Where a new index goes: the same environment variable, minus the search. */
+export declare function outputDir(cwd: string, flag?: string, env?: NodeJS.ProcessEnv): string;
+/** An index is a directory with a manifest in it; nothing else is asserted here. */
+export declare function isIndex(dir: string): boolean;
+//# sourceMappingURL=locate.d.ts.map

package/dist/schema/locate.js

@@ -0,0 +1,141 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { readdirSync, statSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, isAbsolute, join, parse, relative, resolve } from 'node:path';
+import { MANIFEST_FILE } from "./files.js";
+// ---------------------------------------------------------------------------
+// Which index, when nobody said
+//
+// `--dir` and `$ZEN_SCHEMA_DB` are taken as written, missing or not: naming a
+// directory that turns out not to hold an index has to fail saying so, because
+// quietly using a different one would be a worse answer than an error.
+//
+// With neither, the directory is looked for. There is no list of blessed names
+// here and there should not be — an index is self-describing, so what is being
+// looked for is a `manifest.json`, not a directory called `schema-db`.
+// `schema-db` is only the name a *new* index is given, and nothing reads it
+// back. The search is nearest-first: this directory, then what is under it,
+// then up a level and again, so `/assets/…/whatever` is reachable from
+// `/workspace` because the two meet at a shared root on the way up.
+//
+// Three things bound it, and each is bounding a different kind of mistake.
+// Depth and a visit budget bound the cost. The ceiling — the home directory,
+// or the filesystem root when the search began outside it — bounds the
+// blast radius, because an index in someone else's tree is not yours. And two
+// indexes at the same distance is an ambiguity rather than a tie to break:
+// choosing one silently is the one failure worth ruling out entirely, since
+// the wrong index does not error, it answers confidently about another API.
+// ---------------------------------------------------------------------------
+/** The name a new index is given. Nothing searches for it; only `index --out` writes it. */
+export const DEFAULT_DIR = './schema-db';
+export const DIR_ENV = 'ZEN_SCHEMA_DB';
+/** Far enough to climb out of a package into its workspace, not far enough to roam. */
+const MAX_LEVELS = 6;
+/** How far below a directory an index may sit and still count as being in it. */
+const MAX_DEPTH = 3;
+/** A directory with more entries than this is a data store, not a place to keep an index. */
+const MAX_ENTRIES = 128;
+/** Directories the whole search may read, however it is shaped. */
+const MAX_VISITS = 400;
+export function locateIndex(cwd, flag, options = {}) {
+    const env = options.env ?? process.env;
+    if (flag) {
+        return { dir: resolve(cwd, flag), from: 'flag' };
+    }
+    const named = env[DIR_ENV]?.trim();
+    if (named) {
+        return { dir: resolve(cwd, named), from: 'env' };
+    }
+    const found = search(resolve(cwd), options.ceiling ?? ceilingFor(cwd));
+    // Nothing found still answers with the default, so the error names the
+    // directory everyone expects rather than the last place that was searched.
+    return found
+        ? { dir: found, from: 'found' }
+        : { dir: resolve(cwd, DEFAULT_DIR), from: 'default' };
+}
+/** Where a new index goes: the same environment variable, minus the search. */
+export function outputDir(cwd, flag, env = process.env) {
+    return resolve(cwd, flag ?? env[DIR_ENV]?.trim() ?? DEFAULT_DIR);
+}
+/** An index is a directory with a manifest in it; nothing else is asserted here. */
+export function isIndex(dir) {
+    try {
+        return statSync(join(dir, MANIFEST_FILE)).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function search(cwd, ceiling) {
+    const budget = { left: MAX_VISITS, seen: new Set() };
+    let dir = cwd;
+    for (let level = 0; level < MAX_LEVELS; level++) {
+        const found = nearest(dir, budget);
+        if (found.length === 1) {
+            return found[0];
+        }
+        if (found.length > 1) {
+            throw new CliError(`more than one index is equally close to here: ${found.join(', ')}`, EXIT.usage, `say which with --dir, or set ${DIR_ENV}`);
+        }
+        budget.seen.add(dir);
+        const up = dirname(dir);
+        if (up === dir || dir === ceiling) {
+            break;
+        }
+        dir = up;
+    }
+    return undefined;
+}
+/** Every index at the shallowest depth that has any, so a tie can be reported as one. */
+function nearest(root, budget) {
+    let frontier = [root];
+    for (let depth = 0; depth <= MAX_DEPTH && frontier.length > 0; depth++) {
+        const found = frontier.filter(isIndex);
+        if (found.length > 0) {
+            return found;
+        }
+        const next = [];
+        for (const dir of frontier) {
+            if (budget.left <= 0) {
+                return [];
+            }
+            budget.left--;
+            next.push(...children(dir).filter((child) => !budget.seen.has(child)));
+        }
+        frontier = next;
+    }
+    return [];
+}
+/**
+ * The subdirectories of one directory. Hidden directories and `node_modules`
+ * are skipped: an index kept out of sight is not one anybody meant to be found
+ * by looking.
+ */
+function children(dir) {
+    let entries;
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    if (entries.length > MAX_ENTRIES) {
+        return [];
+    }
+    return entries
+        .filter((e) => e.isDirectory() && !e.name.startsWith('.') && e.name !== 'node_modules')
+        .map((e) => join(dir, e.name));
+}
+/**
+ * Home is the ceiling for anyone working inside it. Starting outside it —
+ * a container whose workspace is `/workspace`, a CI checkout — there is no
+ * home to stay within, so the root is the only stop.
+ */
+function ceilingFor(cwd) {
+    const home = homedir();
+    const below = relative(home, resolve(cwd));
+    return below === '' || (!below.startsWith('..') && !isAbsolute(below))
+        ? home
+        : parse(resolve(cwd)).root;
+}
+//# sourceMappingURL=locate.js.map

package/dist/schema/match.d.ts

@@ -0,0 +1,40 @@
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export declare const MAX_PATTERN = 200;
+export declare class PatternError extends Error {
+}
+export interface MatchOptions {
+    /** read the pattern as a regular expression rather than as a literal */
+    regex?: boolean;
+    caseSensitive?: boolean;
+}
+export type Matcher = (text: string) => boolean;
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export declare function wildcard(pattern: string, options?: {
+    caseSensitive?: boolean;
+}): Matcher;
+/** Whether a pattern is asking to be read as a glob at all. */
+export declare const isGlob: (pattern: string) => boolean;
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ *
+ * `regex` settles it outright, and has to: a star is punctuation in both
+ * languages, so `^/(users|teams)/.*` read as a glob would match nothing and
+ * never say why.
+ */
+export declare function loose(pattern: string, options?: MatchOptions): Matcher;
+/** True when any of the patterns matches; no patterns means no opinion. */
+export declare function anyOf(matchers: readonly Matcher[]): Matcher | undefined;
+//# sourceMappingURL=match.d.ts.map

package/dist/schema/match.js

@@ -0,0 +1,92 @@
+// ---------------------------------------------------------------------------
+// Matching, with nothing learned in between
+//
+// Everything here is exact. A glob matches the characters it names and a
+// substring is a substring, which is the whole point of the surfaces built on
+// it: a vector index answers "what is near this", and near is a ranking, so it
+// can only ever return the top of a list. When the question is "does the word
+// `password` appear anywhere at all", a ranking is the wrong instrument and no
+// amount of tuning makes it the right one.
+//
+// A pattern may arrive from a model, so a regex is a bounded promise: the
+// length is capped here and the scan that uses it keeps a deadline.
+// ---------------------------------------------------------------------------
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export const MAX_PATTERN = 200;
+export class PatternError extends Error {
+}
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export function matcher(pattern, options = {}) {
+    guard(pattern);
+    if (!options.regex) {
+        if (options.caseSensitive) {
+            return (text) => text.includes(pattern);
+        }
+        const needle = pattern.toLowerCase();
+        return (text) => text.toLowerCase().includes(needle);
+    }
+    const expression = compile(pattern, options.caseSensitive ? '' : 'i');
+    // `lastIndex` is not carried between calls: the flags never include `g`.
+    return (text) => expression.test(text);
+}
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export function wildcard(pattern, options = {}) {
+    guard(pattern);
+    const source = [...pattern]
+        .map((char) => (char === '*' ? '.*' : char === '?' ? '.' : escape(char)))
+        .join('');
+    const expression = compile(`^${source}$`, options.caseSensitive ? '' : 'i');
+    return (text) => expression.test(text);
+}
+/** Whether a pattern is asking to be read as a glob at all. */
+export const isGlob = (pattern) => /[*?]/.test(pattern);
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ *
+ * `regex` settles it outright, and has to: a star is punctuation in both
+ * languages, so `^/(users|teams)/.*` read as a glob would match nothing and
+ * never say why.
+ */
+export function loose(pattern, options = {}) {
+    if (options.regex) {
+        return matcher(pattern, options);
+    }
+    return isGlob(pattern) ? wildcard(pattern, options) : matcher(pattern, options);
+}
+/** True when any of the patterns matches; no patterns means no opinion. */
+export function anyOf(matchers) {
+    if (matchers.length === 0) {
+        return undefined;
+    }
+    return (text) => matchers.some((match) => match(text));
+}
+// ---------------------------------------------------------------------------
+function guard(pattern) {
+    if (pattern.length === 0) {
+        throw new PatternError('the pattern is empty');
+    }
+    if (pattern.length > MAX_PATTERN) {
+        throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
+    }
+}
+function compile(source, flags) {
+    try {
+        return new RegExp(source, flags);
+    }
+    catch (err) {
+        throw new PatternError(`invalid pattern: ${err.message}`);
+    }
+}
+const escape = (char) => char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+//# sourceMappingURL=match.js.map

package/dist/schema/progress.d.ts

@@ -0,0 +1,26 @@
+import type { Counts, Manifest } from './files.ts';
+export declare const LOCK_FILE = ".lock";
+export declare const README_FILE = "README.md";
+export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
+export interface BuildPlan {
+    /** where the index goes */
+    dir: string;
+    files: readonly string[];
+    /** the embedding reference as it was typed */
+    embedding: string;
+    indexer: string;
+}
+export interface Journal {
+    phase(name: Phase): void;
+    /** what the documents turned out to hold, once they have been read */
+    read(counts: Counts): void;
+    progress(done: number, total: number): void;
+    finish(manifest: Manifest): void;
+    fail(reason: unknown): void;
+}
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export declare function beginBuild(plan: BuildPlan): Journal;
+//# sourceMappingURL=progress.d.ts.map

package/dist/schema/progress.js

@@ -0,0 +1,316 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { basename, join } from 'node:path';
+// ---------------------------------------------------------------------------
+// A build that says what it is doing
+//
+// Indexing a large document is minutes of silence with a directory slowly
+// filling up, and the directory is the only thing a second person — or a second
+// process, or an agent reading the tree — ever sees. So the build writes two
+// files into it and keeps them true:
+//
+//   .lock       who is building this, right now. Gone when nothing is.
+//   README.md   a progress report while it runs, and a description of what the
+//               index holds once it does not.
+//
+// Nothing in either file is an absolute path. A project's `assets/` directory
+// is mounted at `/assets` inside an agent's sandbox, so an index built there is
+// read under a name this process never sees; a host path would be a lie there.
+// ---------------------------------------------------------------------------
+export const LOCK_FILE = '.lock';
+export const README_FILE = 'README.md';
+/** The floor on how often README.md is rewritten. */
+const INTERVAL_MS = 5000;
+const PHASES = {
+    reading: 'reading the documents',
+    graph: 'building the graph',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export function beginBuild(plan) {
+    const documents = plan.files.map((file) => basename(file));
+    const started = Date.now();
+    const lock = {
+        pid: process.pid,
+        host: hostname(),
+        startedAt: new Date(started).toISOString(),
+        indexer: plan.indexer,
+        embedding: plan.embedding,
+        documents,
+    };
+    mkdirSync(plan.dir, { recursive: true });
+    claim(join(plan.dir, LOCK_FILE), lock);
+    let phase = 'reading';
+    let counts;
+    let done = 0;
+    let total = 0;
+    let wroteAt = 0;
+    let closed = false;
+    const write = (body) => {
+        // Through a temp name, so a reader never catches half a file.
+        const target = join(plan.dir, README_FILE);
+        const temp = `${target}.tmp`;
+        writeFileSync(temp, body);
+        renameSync(temp, target);
+        wroteAt = Date.now();
+    };
+    const report = () => write(building({
+        documents,
+        embedding: plan.embedding,
+        started,
+        phase,
+        done,
+        total,
+        counts,
+    }));
+    const maybe = () => {
+        if (!closed && Date.now() - wroteAt >= INTERVAL_MS) {
+            report();
+        }
+    };
+    // The first embedding request can block for ten seconds or more, so a purely
+    // event-driven throttle would leave the elapsed line frozen through it.
+    const ticker = setInterval(maybe, INTERVAL_MS);
+    ticker.unref();
+    const close = (body) => {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        clearInterval(ticker);
+        write(body);
+        rmSync(join(plan.dir, LOCK_FILE), { force: true });
+    };
+    report();
+    return {
+        phase(name) {
+            phase = name;
+            maybe();
+        },
+        read(seen) {
+            counts = seen;
+            total = seen.entities;
+            maybe();
+        },
+        progress(at, of) {
+            done = at;
+            total = of;
+            maybe();
+        },
+        finish(manifest) {
+            close(complete(plan.dir, manifest, Date.now() - started));
+        },
+        fail(reason) {
+            close(failed({ documents, phase, reason, started }));
+        },
+    };
+}
+/**
+ * `wx` makes the create and the check one operation, so two builds racing for
+ * the same directory cannot both win. A lock whose process is gone is stale by
+ * definition and is taken rather than respected — a crashed build must not make
+ * a directory permanently unbuildable.
+ */
+function claim(path, lock) {
+    const body = `${JSON.stringify(lock, null, 4)}\n`;
+    try {
+        writeFileSync(path, body, { flag: 'wx' });
+        return;
+    }
+    catch (err) {
+        if (err.code !== 'EEXIST') {
+            throw err;
+        }
+    }
+    const held = readLock(path);
+    if (held && held.host === hostname() && alive(held.pid)) {
+        throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
+    }
+    writeFileSync(path, body);
+}
+function readLock(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
+ * EPERM means it exists and belongs to someone else, which still counts.
+ */
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+function building(state) {
+    const now = Date.now();
+    const rows = [
+        ['documents', state.documents.join(', ')],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(now - state.started)} ago)`,
+        ],
+        ['step', PHASES[state.phase]],
+    ];
+    if (state.counts) {
+        rows.push(['found', entities(state.counts)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(now).toISOString()]);
+    return [
+        '# Schema index — being built',
+        '',
+        'A searchable index of the API documents named below, written by `zen rag schema index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(dir, manifest, ms) {
+    const titles = manifest.sources.map((s) => s.title).filter(Boolean);
+    const what = titles.length > 0 ? titles.join(', ') : basename(dir);
+    return [
+        `# Schema index — ${what}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(ms)}.`,
+        'Ask it for the operations and types behind a question and it answers with a subgraph:',
+        'the endpoints that match, the schemas they carry, and the fields inside those — printed',
+        'as text, Mermaid, TypeScript or OpenAPI.',
+        '',
+        '## What it covers',
+        '',
+        ...sourceTable(manifest.sources),
+        '',
+        `${entities(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['graph.json', 'the nodes and edges: operations, types, fields'],
+            ['schemas.json', 'the JSON Schema of every type'],
+            ['operations.json', 'every operation, with its parameters and responses'],
+            ...(manifest.sources.some((s) => s.path)
+                ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
+                : []),
+            ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag schema search --dir . --all "how do I cancel a subscription"',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Schema index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag schema index`.',
+        '',
+        ...fields([
+            ['documents', state.documents.join(', ')],
+            ['step', PHASES[state.phase]],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+// Small renderings
+// ---------------------------------------------------------------------------
+/** An indented block, which markdown renders verbatim and an agent reads as a table. */
+function fields(rows) {
+    const width = Math.max(...rows.map((r) => (r[0] ?? '').length));
+    return rows.map(([name, value]) => `    ${(name ?? '').padEnd(width)}  ${value ?? ''}`);
+}
+const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
+function sourceTable(sources) {
+    const rows = sources.map((s) => [
+        s.file,
+        s.dialect,
+        String(s.paths),
+        String(s.methods),
+        String(s.types),
+        String(s.properties),
+    ]);
+    const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
+    const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' | ')} |`;
+    return [
+        line(HEADERS),
+        `| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`,
+        ...rows.map(line),
+    ];
+}
+function entities(counts) {
+    return (`${plural(counts.entities, 'entity', 'entities')}: ` +
+        `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
+}
+function searched(indexes) {
+    if (!indexes.fts && !indexes.vector) {
+        return 'scanned flat: neither index was built';
+    }
+    if (!indexes.vector) {
+        // Below a couple of thousand rows an IVF index has nothing to train on.
+        return 'searched by full text and by vector, the latter as a flat scan';
+    }
+    return indexes.fts
+        ? 'searched by full text and by vector'
+        : 'searched by vector, with no full-text index';
+}
+function plural(n, one, many = `${one}s`) {
+    return `${n} ${n === 1 ? one : many}`;
+}
+function duration(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 1) {
+        return 'under a second';
+    }
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    return minutes < 60
+        ? `${minutes}m${seconds % 60}s`
+        : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
+}
+function message(reason) {
+    const text = reason instanceof Error ? reason.message : String(reason);
+    return text.split('\n')[0]?.trim() || 'no reason given';
+}
+//# sourceMappingURL=progress.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.25",
+    "version": "1.1.26",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.25",
-        "@zenera/neo": "^1.1.25",
+        "@zenera/cli": "^1.1.26",
+        "@zenera/neo": "^1.1.26",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",