@zenera/rag 1.1.5 → 1.1.8

package/dist/schema/files.d.ts

@@ -1,8 +1,10 @@
+import { type IndexHead, type IndexSpec } from '../common/manifest.ts';
 import type { ApiGraph } from './graph.ts';
 import type { Schema } from './schema.ts';
 import type { Operation } from './spec.ts';
 export declare const INDEX_VERSION = 3;
-export declare const MANIFEST_FILE = "manifest.json";
+/** How a schema index is found, read, and refused. */
+export declare const SCHEMA_INDEX: IndexSpec;
 export declare const GRAPH_FILE = "graph.json";
 export declare const SCHEMAS_FILE = "schemas.json";
 export declare const OPERATIONS_FILE = "operations.json";
@@ -30,23 +32,9 @@ export interface Counts {
     properties: number;
     entities: number;
 }
-export interface Manifest {
-    version: number;
-    createdAt: string;
-    indexer: string;
-    /** `ref` as it was typed, `id` as the embedder answers to it */
-    embedding: {
-        ref: string;
-        id: string;
-        dimensions: number;
-    };
+export interface Manifest extends IndexHead {
     sources: SourceRecord[];
     counts: Counts;
-    /** whether the table carries an fts index, and whether it carries a vector one */
-    indexes: {
-        fts: boolean;
-        vector: boolean;
-    };
 }
 export interface WrittenIndex {
     manifest: Manifest;
@@ -75,15 +63,5 @@ export declare function openIndex(dir: string): Promise<OpenIndex>;
  * of what the caller can do next.
  */
 export declare function readSource(dir: string, name: string): Promise<string | undefined>;
-export declare function readManifest(dir: string): Promise<Manifest>;
-/**
- * A store answers with the neighbours of a vector, and a vector means nothing
- * without the model that produced it. Asking one model's index a question
- * embedded by another returns rows, in an order that is noise.
- *
- * Either spelling is accepted, because `openai:text-embedding-3-small` and
- * `text-embedding-3-small` are one model and which of them was typed is not
- * something anyone should have to remember.
- */
-export declare function assertSameEmbedding(manifest: Manifest, ref: string): void;
+export declare const readManifest: (dir: string) => Promise<Manifest>;
 //# sourceMappingURL=files.d.ts.map

package/dist/schema/files.js

@@ -2,6 +2,7 @@ import { CliError, EXIT } from '@zenera/cli/lib';
 import { MultiDirectedGraph } from 'graphology';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { MANIFEST_FILE, readHead } from "../common/manifest.js";
 // ---------------------------------------------------------------------------
 // What an index is, on disk
 //
@@ -20,7 +21,13 @@ import { join } from 'node:path';
 // model — without going looking for the files it was made from.
 // ---------------------------------------------------------------------------
 export const INDEX_VERSION = 3;
-export const MANIFEST_FILE = 'manifest.json';
+/** How a schema index is found, read, and refused. */
+export const SCHEMA_INDEX = {
+    kind: 'schema',
+    version: INDEX_VERSION,
+    defaultDir: './schema-db',
+    envName: 'ZEN_SCHEMA_DB',
+};
 export const GRAPH_FILE = 'graph.json';
 export const SCHEMAS_FILE = 'schemas.json';
 export const OPERATIONS_FILE = 'operations.json';
@@ -77,32 +84,5 @@ export async function readSource(dir, name) {
     }
     return await readFile(join(dir, record.path), 'utf8');
 }
-export async function readManifest(dir) {
-    let text;
-    try {
-        text = await readFile(join(dir, MANIFEST_FILE), 'utf8');
-    }
-    catch {
-        throw new CliError(`${dir} does not hold an index`, EXIT.invalid, 'build one first with `zen rag schema index`');
-    }
-    const manifest = JSON.parse(text);
-    if (manifest.version !== INDEX_VERSION) {
-        throw new CliError(`${dir} is a version ${manifest.version} index, and this indexer reads version ${INDEX_VERSION}`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
-    }
-    return manifest;
-}
-/**
- * A store answers with the neighbours of a vector, and a vector means nothing
- * without the model that produced it. Asking one model's index a question
- * embedded by another returns rows, in an order that is noise.
- *
- * Either spelling is accepted, because `openai:text-embedding-3-small` and
- * `text-embedding-3-small` are one model and which of them was typed is not
- * something anyone should have to remember.
- */
-export function assertSameEmbedding(manifest, ref) {
-    if (ref !== manifest.embedding.ref && ref !== manifest.embedding.id) {
-        throw new CliError(`this index was built with ${manifest.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${manifest.embedding.ref}, or rebuild it`);
-    }
-}
+export const readManifest = (dir) => readHead(dir, SCHEMA_INDEX);
 //# sourceMappingURL=files.js.map

package/dist/schema/lookup.d.ts

@@ -1,5 +1,5 @@
-import type { ApiGraph, NodeAttrs, NodeKind } from './graph.ts';
-import { type Matcher } from './match.ts';
+import { type Matcher } from '../common/match.ts';
+import { type ApiGraph, type NodeAttrs, type NodeKind } from './graph.ts';
 export interface Row extends NodeAttrs {
     id: string;
 }
@@ -28,6 +28,9 @@ export interface Match {
 export interface GrepFilter {
     kinds?: readonly string[];
     source?: string;
+    /** the same two constraints `list` takes, so one question has one spelling */
+    name?: readonly Matcher[];
+    path?: readonly Matcher[];
     limit?: number;
 }
 export interface Grep {
@@ -41,4 +44,11 @@ export declare function grepNodes(graph: ApiGraph, match: Matcher, filter?: Grep
 export declare function propertyCount(graph: ApiGraph, id: string): number;
 /** That count, said properly, in the one phrasing the CLI and the tools share. */
 export declare const fields: (n: number) => string;
+/**
+ * The route a node sits on. An operation carries its own; a parameter carries
+ * the operation's name instead, so it is looked up. A schema has no route at
+ * all and never will — the same DTO is returned by half the API — which is
+ * why a `--path` filter is a filter on the operations and what hangs off them.
+ */
+export declare function routeOf(graph: ApiGraph, id: string, a: NodeAttrs): string;
 //# sourceMappingURL=lookup.d.ts.map

package/dist/schema/lookup.js

@@ -1,5 +1,6 @@
+import { PatternError } from "../common/match.js";
 import { textOf } from "./entities.js";
-import { PatternError } from "./match.js";
+import { methodId } from "./graph.js";
 /** A scan is bounded, because a pattern may have come from a model. */
 const DEADLINE_MS = 2000;
 const DEADLINE_EVERY = 500;
@@ -12,7 +13,7 @@ export function listNodes(graph, filter) {
         if (filter.name && !filter.name.some((match) => match(a.name))) {
             return;
         }
-        if (filter.path && !filter.path.some((match) => match(a.path))) {
+        if (filter.path && !matchesRoute(graph, id, a, filter.path)) {
             return;
         }
         rows.push({ ...a, id });
@@ -36,6 +37,12 @@ export function grepNodes(graph, match, filter = {}) {
         if (filter.source && a.source !== filter.source) {
             continue;
         }
+        if (filter.name && !filter.name.some((match) => match(a.name))) {
+            continue;
+        }
+        if (filter.path && !matchesRoute(graph, id, a, filter.path)) {
+            continue;
+        }
         const text = textOf(graph, id);
         if (match(text)) {
             matches.push({ id, attributes: a, text });
@@ -53,7 +60,27 @@ export function propertyCount(graph, id) {
 }
 /** That count, said properly, in the one phrasing the CLI and the tools share. */
 export const fields = (n) => `${n} ${n === 1 ? 'field' : 'fields'}`;
+/**
+ * The route a node sits on. An operation carries its own; a parameter carries
+ * the operation's name instead, so it is looked up. A schema has no route at
+ * all and never will — the same DTO is returned by half the API — which is
+ * why a `--path` filter is a filter on the operations and what hangs off them.
+ */
+export function routeOf(graph, id, a) {
+    if (a.path) {
+        return a.path;
+    }
+    if (a.kind !== 'property' || !a.parent) {
+        return '';
+    }
+    const owner = methodId(a.parent);
+    return graph.hasNode(owner) ? graph.getNodeAttribute(owner, 'path') : '';
+}
 // ---------------------------------------------------------------------------
+function matchesRoute(graph, id, a, patterns) {
+    const route = routeOf(graph, id, a);
+    return route !== '' && patterns.some((match) => match(route));
+}
 function passes(a, filter) {
     if (filter.source && a.source !== filter.source) {
         return false;

package/dist/{present.d.ts → schema/present.d.ts}

@@ -1,8 +1,8 @@
-import { type HydrateOptions } from './schema/hydrate.ts';
-import { type RenderOptions } from './schema/render.ts';
-import type { Schema } from './schema/schema.ts';
-import type { Operation } from './schema/spec.ts';
-import type { Subgraph } from './schema/subgraph.ts';
+import { type HydrateOptions } from './hydrate.ts';
+import { type RenderOptions } from './render.ts';
+import type { Schema } from './schema.ts';
+import type { Operation } from './spec.ts';
+import type { Subgraph } from './subgraph.ts';
 export declare const FORMATS: readonly ["text", "mermaid", "mermaid-flowchart", "ts", "openapi"];
 export type Format = (typeof FORMATS)[number];
 export declare function isFormat(value: string): value is Format;

package/dist/{present.js → schema/present.js}

@@ -1,5 +1,5 @@
-import { toOpenApi, toTypeScript } from "./schema/hydrate.js";
-import { render } from "./schema/render.js";
+import { toOpenApi, toTypeScript } from "./hydrate.js";
+import { render } from "./render.js";
 // ---------------------------------------------------------------------------
 // One subgraph, in whichever of the four shapes was asked for
 //

package/dist/{query.d.ts → schema/query.d.ts}

@@ -1,4 +1,4 @@
-import type { SchemaQuery } from './schema/search.ts';
+import type { SchemaQuery } from './search.ts';
 export declare class QueryError extends Error {
 }
 export declare function parseQuery(value: unknown): SchemaQuery;

package/dist/schema/readme.d.ts

@@ -0,0 +1,6 @@
+import type { Report } from '../common/progress.ts';
+import type { Counts, Manifest } from './files.ts';
+export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
+export declare const PHASES: Record<Phase, string>;
+export declare const SCHEMA_REPORT: Report<Counts, Manifest>;
+//# sourceMappingURL=readme.d.ts.map

package/dist/schema/readme.js

@@ -0,0 +1,122 @@
+import { basename } from 'node:path';
+import { INTERVAL_MS } from "../common/progress.js";
+import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+export const PHASES = {
+    reading: 'reading the documents',
+    graph: 'building the graph',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+export const SCHEMA_REPORT = { building, complete, failed };
+function building(state) {
+    const rows = [
+        ['documents', state.documents.join(', ')],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(state.now - state.started)} ago)`,
+        ],
+        ['step', state.step],
+    ];
+    if (state.summary) {
+        rows.push(['found', entities(state.summary)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(state.now).toISOString()]);
+    return [
+        '# Schema index — being built',
+        '',
+        'A searchable index of the API documents named below, written by `zen rag schema index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(state) {
+    const { manifest } = state;
+    const titles = manifest.sources.map((s) => s.title).filter(Boolean);
+    const what = titles.length > 0 ? titles.join(', ') : basename(state.dir);
+    return [
+        `# Schema index — ${what}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(state.ms)}.`,
+        'Ask it for the operations and types behind a question and it answers with a subgraph:',
+        'the endpoints that match, the schemas they carry, and the fields inside those — printed',
+        'as text, Mermaid, TypeScript or OpenAPI.',
+        '',
+        '## What it covers',
+        '',
+        ...sourceTable(manifest.sources),
+        '',
+        `${entities(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['graph.json', 'the nodes and edges: operations, types, fields'],
+            ['schemas.json', 'the JSON Schema of every type'],
+            ['operations.json', 'every operation, with its parameters and responses'],
+            ...(manifest.sources.some((s) => s.path)
+                ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
+                : []),
+            ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag schema search --dir . --all "how do I cancel a subscription"',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Schema index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag schema index`.',
+        '',
+        ...fields([
+            ['documents', state.documents.join(', ')],
+            ['step', state.step],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
+const sourceTable = (sources) => grid(HEADERS, sources.map((s) => [
+    s.file,
+    s.dialect,
+    String(s.paths),
+    String(s.methods),
+    String(s.types),
+    String(s.properties),
+]));
+function entities(counts) {
+    return (`${plural(counts.entities, 'entity', 'entities')}: ` +
+        `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
+}
+//# sourceMappingURL=readme.js.map

package/dist/schema/render.d.ts

@@ -3,7 +3,15 @@ export type RenderFormat = 'text' | 'mermaid' | 'mermaid-flowchart';
 export interface RenderOptions {
     docs?: boolean;
     maxDoc?: number;
+    /** name the document each operation and schema came from */
+    source?: boolean;
 }
+/**
+ * Which document something came from, spelled one way everywhere. An index
+ * over four revisions of one API has four `GET /infra/tier-0s`, and a listing
+ * that does not say which is which is a listing you have to go and check.
+ */
+export declare const sourceTag: (source: string) => string;
 export declare function render(sub: Subgraph, format: RenderFormat, options?: RenderOptions): string;
 export declare function toText(sub: Subgraph, options?: RenderOptions): string;
 export declare function toMermaid(sub: Subgraph, options?: RenderOptions): string;

package/dist/schema/render.js

@@ -1,4 +1,10 @@
 const HIT = '»';
+/**
+ * Which document something came from, spelled one way everywhere. An index
+ * over four revisions of one API has four `GET /infra/tier-0s`, and a listing
+ * that does not say which is which is a listing you have to go and check.
+ */
+export const sourceTag = (source) => (source ? `[source: ${source}]` : '');
 export function render(sub, format, options = {}) {
     switch (format) {
         case 'mermaid':
@@ -44,7 +50,7 @@ export function toText(sub, options = {}) {
 function methodLines(view, method, options) {
     const a = method.attributes;
     const out = [
-        `  ${mark(method)}${a.httpMethod} ${a.path}  ${a.name}${doc(method, options, ' —')}`,
+        `  ${mark(method)}${a.httpMethod} ${a.path}  ${a.name}${from(method, options)}${doc(method, options, ' —')}`,
     ];
     for (const edge of view.out(method.id, 'HAS_PARAM')) {
         const node = view.node(edge.target);
@@ -62,7 +68,7 @@ function methodLines(view, method, options) {
 }
 function typeLines(view, type, options) {
     const out = [
-        `  ${mark(type)}${type.attributes.name}${side(type.attributes)}${doc(type, options, ' —')}`,
+        `  ${mark(type)}${type.attributes.name}${side(type.attributes)}${from(type, options)}${doc(type, options, ' —')}`,
     ];
     const composes = view.out(type.id, 'COMPOSES').map((e) => view.name(e.target));
     if (composes.length > 0) {
@@ -96,6 +102,10 @@ function doc(node, options, lead) {
     }
     return `${lead} ${clip(node.attributes.doc, options.maxDoc ?? 120)}`;
 }
+function from(node, options) {
+    const tag = options.source ? sourceTag(node.attributes.source) : '';
+    return tag ? `  ${tag}` : '';
+}
 // ---------------------------------------------------------------------------
 // Mermaid
 // ---------------------------------------------------------------------------

package/dist/{repl.d.ts → schema/repl.d.ts}

@@ -1,5 +1,5 @@
 import { type Format, type OutputOptions } from './present.ts';
-import type { SchemaIndex, SchemaQuery } from './schema/search.ts';
+import type { SchemaIndex, SchemaQuery } from './search.ts';
 interface Settings extends OutputOptions {
     format: Format;
 }

package/dist/schema/search.js

@@ -1,4 +1,5 @@
-import { assertSameEmbedding, openIndex } from "./files.js";
+import { assertSameEmbedding } from "../common/manifest.js";
+import { openIndex } from "./files.js";
 import { EntityStore } from "./store.js";
 import { DEFAULT_MAX_HOPS, DEFAULT_MAX_NODES, stitch, } from "./subgraph.js";
 export const DEFAULT_LIMIT = 5;

package/dist/schema/tools.d.ts

@@ -1,10 +1,12 @@
 import { type AnyTool } from '@zenera/neo';
-import { type Format } from '../present.ts';
+import { type Format } from './present.ts';
 import type { SchemaIndex } from './search.ts';
 export interface SchemaToolOptions {
     /** what `format` defaults to when the model does not say */
     format?: Format;
     docs?: boolean;
+    /** name the document each answer came from; on by default past one document */
+    source?: boolean;
 }
 export declare function schemaTools<TCtx = unknown>(index: SchemaIndex, options?: SchemaToolOptions): AnyTool<TCtx>[];
 //# sourceMappingURL=tools.d.ts.map