@zenera/rag 1.1.3 → 1.1.5

package/dist/schema/build.js

@@ -1,42 +1,67 @@
 import { toEntities } from "./entities.js";
-import { INDEX_VERSION, writeIndex, } from "./files.js";
+import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
 import { buildGraph } from "./graph.js";
+import { beginBuild } from "./progress.js";
 import { loadSpecs } from "./spec.js";
 import { writeStore } from "./store.js";
 const DEFAULT_BATCH = 96;
 export async function buildIndex(options) {
-    const corpus = await loadSpecs(options.files);
-    const { graph, types } = buildGraph(corpus);
-    const entities = toEntities(graph);
-    const summary = {
-        sources: sourcesOf(corpus, entities, options.files),
-        counts: {
-            methods: entities.filter((e) => e.kind === 'method').length,
-            types: entities.filter((e) => e.kind === 'type').length,
-            properties: entities.filter((e) => e.kind === 'property').length,
-            entities: entities.length,
-        },
-    };
-    options.onRead?.(summary);
-    const vectors = await embedAll(entities, options);
-    const written = await writeStore(options.out, entities, vectors);
-    const manifest = {
-        version: INDEX_VERSION,
-        createdAt: new Date().toISOString(),
+    const journal = beginBuild({
+        dir: options.out,
+        files: options.files,
+        embedding: options.embeddingRef ?? options.embedder.id,
         indexer: options.indexer,
-        embedding: {
-            ref: options.embeddingRef ?? options.embedder.id,
-            id: options.embedder.id,
-            dimensions: vectors[0]?.length ?? 0,
-        },
-        sources: summary.sources,
-        counts: summary.counts,
-        indexes: { fts: written.fts, vector: written.vector },
-    };
-    await writeIndex(options.out, { manifest, graph, types, operations: corpus.operations });
-    return { manifest, entities };
+    });
+    try {
+        const corpus = await loadSpecs(options.files);
+        journal.phase('graph');
+        const { graph, types } = buildGraph(corpus);
+        const entities = toEntities(graph);
+        const keep = options.sources !== false;
+        const summary = {
+            sources: sourcesOf(corpus, entities, keep),
+            counts: {
+                methods: entities.filter((e) => e.kind === 'method').length,
+                types: entities.filter((e) => e.kind === 'type').length,
+                properties: entities.filter((e) => e.kind === 'property').length,
+                entities: entities.length,
+            },
+        };
+        journal.read(summary.counts);
+        options.onRead?.(summary);
+        journal.phase('embedding');
+        const vectors = await embedAll(entities, options, journal);
+        journal.phase('writing');
+        const written = await writeStore(options.out, entities, vectors);
+        const manifest = {
+            version: INDEX_VERSION,
+            createdAt: new Date().toISOString(),
+            indexer: options.indexer,
+            embedding: {
+                ref: options.embeddingRef ?? options.embedder.id,
+                id: options.embedder.id,
+                dimensions: vectors[0]?.length ?? 0,
+            },
+            sources: summary.sources,
+            counts: summary.counts,
+            indexes: { fts: written.fts, vector: written.vector },
+        };
+        await writeIndex(options.out, {
+            manifest,
+            graph,
+            types,
+            operations: corpus.operations,
+            documents: keep ? corpus.documents : {},
+        });
+        journal.finish(manifest);
+        return { manifest, entities };
+    }
+    catch (err) {
+        journal.fail(err);
+        throw err;
+    }
 }
-async function embedAll(entities, options) {
+async function embedAll(entities, options, journal) {
     const size = options.batch ?? DEFAULT_BATCH;
     const out = [];
     for (let at = 0; at < entities.length; at += size) {
@@ -50,6 +75,7 @@ async function embedAll(entities, options) {
             throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
         }
         out.push(...response.vectors.map((v) => Float32Array.from(v)));
+        journal.progress(out.length, entities.length);
         options.onProgress?.(out.length, entities.length);
     }
     return out;
@@ -59,12 +85,14 @@ async function embedAll(entities, options) {
  * never named — an inline request body, say — is credited to the file it came
  * from instead of going missing.
  */
-function sourcesOf(corpus, entities, files) {
-    return corpus.docs.map((doc, index) => {
+function sourcesOf(corpus, entities, keep) {
+    return corpus.docs.map((doc) => {
         const mine = entities.filter((e) => e.source === doc.source);
         const operations = corpus.operations.filter((op) => op.source === doc.source);
         return {
-            path: files[index] ?? doc.source,
+            name: doc.source,
+            file: doc.file,
+            path: keep ? `${SOURCES_DIR}/${doc.source}.json` : undefined,
             sha256: doc.sha256,
             dialect: doc.dialect,
             title: doc.title,

package/dist/schema/entities.d.ts

@@ -15,4 +15,6 @@ export interface EntityRecord {
     text: string;
 }
 export declare function toEntities(graph: ApiGraph): EntityRecord[];
+/** Exported so a literal search reads the same string the index was built from. */
+export declare function textOf(graph: ApiGraph, id: string): string;
 //# sourceMappingURL=entities.d.ts.map

package/dist/schema/entities.js

@@ -22,7 +22,8 @@ export function toEntities(graph) {
     }
     return out.sort((a, b) => a.id.localeCompare(b.id));
 }
-function textOf(graph, id) {
+/** Exported so a literal search reads the same string the index was built from. */
+export function textOf(graph, id) {
     const a = graph.getNodeAttributes(id);
     const parts = [];
     if (a.kind === 'method') {

package/dist/schema/files.d.ts

@@ -1,14 +1,20 @@
 import type { ApiGraph } from './graph.ts';
 import type { Schema } from './schema.ts';
 import type { Operation } from './spec.ts';
-export declare const INDEX_VERSION = 1;
+export declare const INDEX_VERSION = 3;
 export declare const MANIFEST_FILE = "manifest.json";
 export declare const GRAPH_FILE = "graph.json";
 export declare const SCHEMAS_FILE = "schemas.json";
 export declare const OPERATIONS_FILE = "operations.json";
 export declare const LANCE_DIR = "lance";
+export declare const SOURCES_DIR = "sources";
 export interface SourceRecord {
-    path: string;
+    /** the document's name within the index: what every entity's `source` says */
+    name: string;
+    /** what the file was called on the machine that built this */
+    file: string;
+    /** the bundled copy, relative to the index; absent when none was kept */
+    path?: string;
     sha256: string;
     dialect: string;
     title: string;
@@ -47,6 +53,8 @@ export interface WrittenIndex {
     graph: ApiGraph;
     types: Readonly<Record<string, Schema>>;
     operations: readonly Operation[];
+    /** the bundled documents to keep beside the index, by name */
+    documents: Readonly<Record<string, string>>;
 }
 export interface OpenIndex {
     dir: string;
@@ -60,6 +68,13 @@ export interface OpenIndex {
 export declare const lancePath: (dir: string) => string;
 export declare function writeIndex(dir: string, index: WrittenIndex): Promise<void>;
 export declare function openIndex(dir: string): Promise<OpenIndex>;
+/**
+ * The bundled document as it was indexed. Kept only when the index was built
+ * with sources, which is why the manifest is asked first: the difference
+ * between "no such document" and "this index did not keep them" is the whole
+ * of what the caller can do next.
+ */
+export declare function readSource(dir: string, name: string): Promise<string | undefined>;
 export declare function readManifest(dir: string): Promise<Manifest>;
 /**
  * A store answers with the neighbours of a vector, and a vector means nothing

package/dist/schema/files.js

@@ -1,32 +1,44 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
 import { MultiDirectedGraph } from 'graphology';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { CliError, EXIT } from '@zenera/cli/lib';
 // ---------------------------------------------------------------------------
 // What an index is, on disk
 //
-// Four things in a directory, and the order they are written in is the whole
-// crash story: the manifest goes last, so a half-built index has no manifest
-// and reads as "not indexed" rather than as a store that quietly lost half its
-// operations.
+// A directory, and the order it is written in is the whole crash story: the
+// manifest goes last, so a half-built index has no manifest and reads as "not
+// indexed" rather than as a store that quietly lost half its operations.
 //
 // `graph.json` is deliberately thin — names, directions, edges — because it is
 // parsed whole on every search. The schemas are the bulk of the bytes and are
 // wanted only when something is being printed, so they live apart and are read
 // on first use.
+//
+// `sources/` holds the documents themselves, bundled, so the index is one
+// portable thing: nothing in it names a path outside itself, it can be moved or
+// shipped whole, and the graph can be rebuilt — or re-embedded with another
+// model — without going looking for the files it was made from.
 // ---------------------------------------------------------------------------
-export const INDEX_VERSION = 1;
+export const INDEX_VERSION = 3;
 export const MANIFEST_FILE = 'manifest.json';
 export const GRAPH_FILE = 'graph.json';
 export const SCHEMAS_FILE = 'schemas.json';
 export const OPERATIONS_FILE = 'operations.json';
 export const LANCE_DIR = 'lance';
+export const SOURCES_DIR = 'sources';
 export const lancePath = (dir) => join(dir, LANCE_DIR);
 export async function writeIndex(dir, index) {
     await mkdir(dir, { recursive: true });
     await writeFile(join(dir, GRAPH_FILE), JSON.stringify(index.graph.export()));
     await writeFile(join(dir, SCHEMAS_FILE), JSON.stringify(index.types));
     await writeFile(join(dir, OPERATIONS_FILE), JSON.stringify(index.operations));
+    const documents = Object.entries(index.documents);
+    if (documents.length > 0) {
+        await mkdir(join(dir, SOURCES_DIR), { recursive: true });
+        for (const [name, text] of documents) {
+            await writeFile(join(dir, SOURCES_DIR, `${name}.json`), text);
+        }
+    }
     await writeFile(join(dir, MANIFEST_FILE), `${JSON.stringify(index.manifest, null, 4)}\n`);
 }
 export async function openIndex(dir) {
@@ -48,6 +60,23 @@ function once(load) {
     let pending;
     return () => (pending ??= load());
 }
+/**
+ * The bundled document as it was indexed. Kept only when the index was built
+ * with sources, which is why the manifest is asked first: the difference
+ * between "no such document" and "this index did not keep them" is the whole
+ * of what the caller can do next.
+ */
+export async function readSource(dir, name) {
+    const manifest = await readManifest(dir);
+    const record = manifest.sources.find((s) => s.name === name);
+    if (!record) {
+        throw new CliError(`${dir} holds no document called ${name}`, EXIT.failed, `it has: ${manifest.sources.map((s) => s.name).join(', ')}`);
+    }
+    if (!record.path) {
+        return undefined;
+    }
+    return await readFile(join(dir, record.path), 'utf8');
+}
 export async function readManifest(dir) {
     let text;
     try {
@@ -58,7 +87,7 @@ export async function readManifest(dir) {
     }
     const manifest = JSON.parse(text);
     if (manifest.version !== INDEX_VERSION) {
-        throw new CliError(`${dir} was written by another version of this indexer`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
+        throw new CliError(`${dir} is a version ${manifest.version} index, and this indexer reads version ${INDEX_VERSION}`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
     }
     return manifest;
 }

package/dist/schema/lookup.d.ts

@@ -0,0 +1,44 @@
+import type { ApiGraph, NodeAttrs, NodeKind } from './graph.ts';
+import { type Matcher } from './match.ts';
+export interface Row extends NodeAttrs {
+    id: string;
+}
+export interface ListFilter {
+    kind: NodeKind;
+    /** any one matching is enough; none means every name passes */
+    name?: readonly Matcher[];
+    path?: readonly Matcher[];
+    source?: string;
+    methodType?: string;
+    direction?: string;
+    limit?: number;
+}
+export interface Listing {
+    /** how many matched, whatever was kept */
+    found: number;
+    rows: Row[];
+    truncated: boolean;
+}
+export interface Match {
+    id: string;
+    attributes: NodeAttrs;
+    /** the indexed text this matched against */
+    text: string;
+}
+export interface GrepFilter {
+    kinds?: readonly string[];
+    source?: string;
+    limit?: number;
+}
+export interface Grep {
+    found: number;
+    matches: Match[];
+    truncated: boolean;
+}
+export declare function listNodes(graph: ApiGraph, filter: ListFilter): Listing;
+export declare function grepNodes(graph: ApiGraph, match: Matcher, filter?: GrepFilter): Grep;
+/** How many fields a type carries, which is most of what a listing wants to say. */
+export declare function propertyCount(graph: ApiGraph, id: string): number;
+/** That count, said properly, in the one phrasing the CLI and the tools share. */
+export declare const fields: (n: number) => string;
+//# sourceMappingURL=lookup.d.ts.map

package/dist/schema/lookup.js

@@ -0,0 +1,83 @@
+import { textOf } from "./entities.js";
+import { PatternError } from "./match.js";
+/** A scan is bounded, because a pattern may have come from a model. */
+const DEADLINE_MS = 2000;
+const DEADLINE_EVERY = 500;
+export function listNodes(graph, filter) {
+    const rows = [];
+    graph.forEachNode((id, a) => {
+        if (a.kind !== filter.kind || !passes(a, filter)) {
+            return;
+        }
+        if (filter.name && !filter.name.some((match) => match(a.name))) {
+            return;
+        }
+        if (filter.path && !filter.path.some((match) => match(a.path))) {
+            return;
+        }
+        rows.push({ ...a, id });
+    });
+    rows.sort(order(filter.kind));
+    return cut(rows, filter.limit);
+}
+export function grepNodes(graph, match, filter = {}) {
+    const kinds = filter.kinds?.length ? new Set(filter.kinds) : undefined;
+    const matches = [];
+    const until = Date.now() + DEADLINE_MS;
+    let seen = 0;
+    for (const id of graph.nodes()) {
+        if (++seen % DEADLINE_EVERY === 0 && Date.now() > until) {
+            throw new PatternError(`the pattern is still running after ${DEADLINE_MS / 1000}s — it is too expensive to be useful`);
+        }
+        const a = graph.getNodeAttributes(id);
+        if (kinds && !kinds.has(a.kind)) {
+            continue;
+        }
+        if (filter.source && a.source !== filter.source) {
+            continue;
+        }
+        const text = textOf(graph, id);
+        if (match(text)) {
+            matches.push({ id, attributes: a, text });
+        }
+    }
+    matches.sort((a, b) => a.id.localeCompare(b.id));
+    const kept = cut(matches, filter.limit);
+    return { found: kept.found, matches: kept.rows, truncated: kept.truncated };
+}
+/** How many fields a type carries, which is most of what a listing wants to say. */
+export function propertyCount(graph, id) {
+    return graph
+        .outEdges(id)
+        .filter((e) => graph.getEdgeAttribute(e, 'relation') === 'HAS_PROPERTY').length;
+}
+/** That count, said properly, in the one phrasing the CLI and the tools share. */
+export const fields = (n) => `${n} ${n === 1 ? 'field' : 'fields'}`;
+// ---------------------------------------------------------------------------
+function passes(a, filter) {
+    if (filter.source && a.source !== filter.source) {
+        return false;
+    }
+    if (filter.methodType && a.methodType !== filter.methodType) {
+        return false;
+    }
+    if (filter.direction && a.direction !== filter.direction && a.direction !== 'both') {
+        return false;
+    }
+    return true;
+}
+/** Operations read as a table of routes; everything else reads as a list of names. */
+function order(kind) {
+    if (kind !== 'method') {
+        return (a, b) => a.id.localeCompare(b.id);
+    }
+    return (a, b) => a.path.localeCompare(b.path) || a.httpMethod.localeCompare(b.httpMethod);
+}
+function cut(rows, limit) {
+    const found = rows.length;
+    if (!limit || limit >= found) {
+        return { found, rows, truncated: false };
+    }
+    return { found, rows: rows.slice(0, limit), truncated: true };
+}
+//# sourceMappingURL=lookup.js.map

package/dist/schema/match.d.ts

@@ -0,0 +1,36 @@
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export declare const MAX_PATTERN = 200;
+export declare class PatternError extends Error {
+}
+export interface MatchOptions {
+    /** read the pattern as a regular expression rather than as a literal */
+    regex?: boolean;
+    caseSensitive?: boolean;
+}
+export type Matcher = (text: string) => boolean;
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export declare function wildcard(pattern: string, options?: {
+    caseSensitive?: boolean;
+}): Matcher;
+/** Whether a pattern is asking to be read as a glob at all. */
+export declare const isGlob: (pattern: string) => boolean;
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ */
+export declare function loose(pattern: string, options?: MatchOptions): Matcher;
+/** True when any of the patterns matches; no patterns means no opinion. */
+export declare function anyOf(matchers: readonly Matcher[]): Matcher | undefined;
+//# sourceMappingURL=match.d.ts.map

package/dist/schema/match.js

@@ -0,0 +1,85 @@
+// ---------------------------------------------------------------------------
+// Matching, with nothing learned in between
+//
+// Everything here is exact. A glob matches the characters it names and a
+// substring is a substring, which is the whole point of the surfaces built on
+// it: a vector index answers "what is near this", and near is a ranking, so it
+// can only ever return the top of a list. When the question is "does the word
+// `password` appear anywhere at all", a ranking is the wrong instrument and no
+// amount of tuning makes it the right one.
+//
+// A pattern may arrive from a model, so a regex is a bounded promise: the
+// length is capped here and the scan that uses it keeps a deadline.
+// ---------------------------------------------------------------------------
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export const MAX_PATTERN = 200;
+export class PatternError extends Error {
+}
+/**
+ * A predicate over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export function matcher(pattern, options = {}) {
+    guard(pattern);
+    if (!options.regex) {
+        if (options.caseSensitive) {
+            return (text) => text.includes(pattern);
+        }
+        const needle = pattern.toLowerCase();
+        return (text) => text.toLowerCase().includes(needle);
+    }
+    const expression = compile(pattern, options.caseSensitive ? '' : 'i');
+    // `lastIndex` is not carried between calls: the flags never include `g`.
+    return (text) => expression.test(text);
+}
+/**
+ * A glob, matched against the whole string. Globs rather than regexes because
+ * these are for naming things — a path, a schema — and a star is what everyone
+ * reaches for first.
+ */
+export function wildcard(pattern, options = {}) {
+    guard(pattern);
+    const source = [...pattern]
+        .map((char) => (char === '*' ? '.*' : char === '?' ? '.' : escape(char)))
+        .join('');
+    const expression = compile(`^${source}$`, options.caseSensitive ? '' : 'i');
+    return (text) => expression.test(text);
+}
+/** Whether a pattern is asking to be read as a glob at all. */
+export const isGlob = (pattern) => /[*?]/.test(pattern);
+/**
+ * What someone means when they type a name into a filter. With a star in it,
+ * a glob; without one, a substring — because `password` typed into `--name` is
+ * a search for the word, and a whole-string match would answer nothing and
+ * look like the field does not exist.
+ */
+export function loose(pattern, options = {}) {
+    return isGlob(pattern) ? wildcard(pattern, options) : matcher(pattern, options);
+}
+/** True when any of the patterns matches; no patterns means no opinion. */
+export function anyOf(matchers) {
+    if (matchers.length === 0) {
+        return undefined;
+    }
+    return (text) => matchers.some((match) => match(text));
+}
+// ---------------------------------------------------------------------------
+function guard(pattern) {
+    if (pattern.length === 0) {
+        throw new PatternError('the pattern is empty');
+    }
+    if (pattern.length > MAX_PATTERN) {
+        throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
+    }
+}
+function compile(source, flags) {
+    try {
+        return new RegExp(source, flags);
+    }
+    catch (err) {
+        throw new PatternError(`invalid pattern: ${err.message}`);
+    }
+}
+const escape = (char) => char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+//# sourceMappingURL=match.js.map

package/dist/schema/progress.d.ts

@@ -0,0 +1,26 @@
+import type { Counts, Manifest } from './files.ts';
+export declare const LOCK_FILE = ".lock";
+export declare const README_FILE = "README.md";
+export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
+export interface BuildPlan {
+    /** where the index goes */
+    dir: string;
+    files: readonly string[];
+    /** the embedding reference as it was typed */
+    embedding: string;
+    indexer: string;
+}
+export interface Journal {
+    phase(name: Phase): void;
+    /** what the documents turned out to hold, once they have been read */
+    read(counts: Counts): void;
+    progress(done: number, total: number): void;
+    finish(manifest: Manifest): void;
+    fail(reason: unknown): void;
+}
+/**
+ * Takes the directory, or refuses it. Two builds writing one index would
+ * interleave their LanceDB writes and leave a store neither of them describes.
+ */
+export declare function beginBuild(plan: BuildPlan): Journal;
+//# sourceMappingURL=progress.d.ts.map