@zenera/rag 1.1.0

package/dist/schema/spec.d.ts

@@ -0,0 +1,58 @@
+import { CliError } from '@zenera/cli/lib';
+import { type Dialect, type Schema } from './schema.ts';
+export declare const METHODS: readonly ["get", "put", "post", "delete", "patch", "head", "options"];
+export type Method = (typeof METHODS)[number];
+export type ParamIn = 'path' | 'query' | 'header' | 'cookie' | 'formData';
+export type MethodType = 'read_only' | 'read_write';
+/** GET/HEAD/OPTIONS answer a question; everything else changes something. */
+export declare function methodTypeOf(method: Method): MethodType;
+export interface ParamSpec {
+    name: string;
+    in: ParamIn;
+    required: boolean;
+    doc: string;
+    schema: Schema;
+}
+export interface ResponseSpec {
+    status: number;
+    doc: string;
+    schema: Schema;
+}
+export interface Operation {
+    source: string;
+    method: Method;
+    /** the template, `/users/{user_id}` */
+    path: string;
+    operationId: string;
+    summary: string;
+    description: string;
+    tags: string[];
+    params: ParamSpec[];
+    requestBody?: {
+        required: boolean;
+        schema: Schema;
+    };
+    /** every 2xx that carries a body, in status order */
+    responses: ResponseSpec[];
+}
+export interface ApiDoc {
+    source: string;
+    sha256: string;
+    dialect: Dialect;
+    title: string;
+    version: string;
+}
+export interface Corpus {
+    docs: ApiDoc[];
+    operations: Operation[];
+    /** every named component schema in the corpus, by its settled id */
+    types: Record<string, Schema>;
+    /** which document each type id came from */
+    typeSource: Record<string, string>;
+}
+/** A `CliError` so an unreadable document exits 3 wherever it is raised. */
+export declare class SpecError extends CliError {
+    constructor(message: string, hint?: string);
+}
+export declare function loadSpecs(files: readonly string[]): Promise<Corpus>;
+//# sourceMappingURL=spec.d.ts.map

package/dist/schema/spec.js

@@ -0,0 +1,309 @@
+import SwaggerParser from '@apidevtools/swagger-parser';
+import { createHash } from 'node:crypto';
+import { basename, extname } from 'node:path';
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { docOf, isObject, normalize } from "./schema.js";
+// ---------------------------------------------------------------------------
+// Documents, flattened — but not dereferenced
+//
+// `bundle` rather than `dereference`: it resolves external files and leaves
+// internal `$ref`s standing. That is the opposite of what a mock server wants
+// and exactly what a graph wants, because `#/components/schemas/User` is an
+// edge and `User` is a node id.
+//
+// Two documents may both call a schema `User`. Rather than qualify every name
+// and make the single-document case ugly, names are qualified only where they
+// actually collide, and every `$ref` in the corpus is rewritten to the id that
+// was settled on. Below this file there is one flat namespace of type ids.
+// ---------------------------------------------------------------------------
+export const METHODS = ['get', 'put', 'post', 'delete', 'patch', 'head', 'options'];
+/** GET/HEAD/OPTIONS answer a question; everything else changes something. */
+export function methodTypeOf(method) {
+    return method === 'get' || method === 'head' || method === 'options'
+        ? 'read_only'
+        : 'read_write';
+}
+/** A `CliError` so an unreadable document exits 3 wherever it is raised. */
+export class SpecError extends CliError {
+    constructor(message, hint) {
+        super(message, EXIT.invalid, hint);
+        this.name = 'SpecError';
+    }
+}
+// ---------------------------------------------------------------------------
+// Loading
+// ---------------------------------------------------------------------------
+export async function loadSpecs(files) {
+    if (files.length === 0) {
+        throw new SpecError('no document given', 'name at least one openapi/swagger file');
+    }
+    const loaded = [];
+    for (const file of files) {
+        loaded.push(await loadSpec(file));
+    }
+    return settle(loaded);
+}
+async function loadSpec(file) {
+    let raw;
+    try {
+        raw = (await SwaggerParser.bundle(file));
+    }
+    catch (err) {
+        throw new SpecError(`${file}: ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`, 'the document must be a readable OpenAPI 3.x or Swagger 2.0 file');
+    }
+    const dialect = dialectOf(raw, file);
+    const source = raw.components?.schemas ?? raw.definitions ?? {};
+    const types = new Map();
+    for (const [name, schema] of Object.entries(source)) {
+        types.set(name, normalize(schema, dialect));
+    }
+    return {
+        doc: {
+            source: file,
+            sha256: createHash('sha256').update(JSON.stringify(raw)).digest('hex'),
+            dialect,
+            title: raw.info?.title?.trim() || basename(file),
+            version: raw.info?.version?.trim() || '',
+        },
+        slug: slugOf(file),
+        operations: operationsOf(raw, dialect, file),
+        types,
+    };
+}
+function dialectOf(doc, file) {
+    if (doc.swagger?.startsWith('2.')) {
+        return 'swagger-2.0';
+    }
+    if (doc.openapi?.startsWith('3.1')) {
+        return 'openapi-3.1';
+    }
+    if (doc.openapi?.startsWith('3.')) {
+        return 'openapi-3.0';
+    }
+    throw new SpecError(`${file}: neither \`swagger: 2.0\` nor \`openapi: 3.x\` is declared`);
+}
+function slugOf(file) {
+    const name = basename(file, extname(file));
+    return name.replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '') || 'api';
+}
+// ---------------------------------------------------------------------------
+// Operations
+// ---------------------------------------------------------------------------
+function operationsOf(raw, dialect, file) {
+    const prefix = dialect === 'swagger-2.0' ? (raw.basePath ?? '') : '';
+    const out = [];
+    for (const [template, item] of Object.entries(raw.paths ?? {})) {
+        if (!isObject(item)) {
+            continue;
+        }
+        // Path-level parameters apply to every method on it, and an operation
+        // may override one by repeating its name and location.
+        const shared = paramsOf(item.parameters, dialect);
+        for (const method of METHODS) {
+            const op = item[method];
+            if (!isObject(op)) {
+                continue;
+            }
+            const path = join(prefix, template);
+            const own = paramsOf(op.parameters, dialect);
+            out.push({
+                source: file,
+                method,
+                path,
+                operationId: text(op.operationId) || synthesizeId(method, path),
+                summary: text(op.summary),
+                description: text(op.description),
+                tags: Array.isArray(op.tags) ? op.tags.filter((t) => typeof t === 'string') : [],
+                params: override(shared, own).filter((p) => p.in !== 'body'),
+                requestBody: bodyOf(op, own, dialect),
+                responses: responsesOf(op, dialect),
+            });
+        }
+    }
+    return out;
+}
+function join(prefix, template) {
+    const base = prefix.replace(/\/+$/, '');
+    return `${base}${template.startsWith('/') ? '' : '/'}${template}` || '/';
+}
+function synthesizeId(method, path) {
+    const tail = path
+        .replace(/[{}]/g, '')
+        .split('/')
+        .filter(Boolean)
+        .join('_')
+        .replace(/[^A-Za-z0-9_]+/g, '_');
+    return `${method}_${tail || 'root'}`;
+}
+const text = (v) => (typeof v === 'string' ? v.trim() : '');
+function paramsOf(value, dialect) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    const out = [];
+    for (const entry of value) {
+        if (!isObject(entry) || typeof entry.name !== 'string') {
+            continue;
+        }
+        const where = entry.in;
+        if (typeof where !== 'string') {
+            continue;
+        }
+        // Swagger 2.0 puts the type keywords on the parameter itself; 3.x
+        // wraps them in `schema`, which is the shape everything below wants.
+        const schema = isObject(entry.schema) ? entry.schema : envelope(entry);
+        out.push({
+            name: entry.name,
+            in: where,
+            required: entry.required === true || where === 'path',
+            doc: docOf(entry) || docOf(schema),
+            schema: normalize(schema, dialect),
+        });
+    }
+    return out;
+}
+/** What is left of a Swagger 2.0 parameter once it stops describing itself. */
+const ENVELOPE = new Set([
+    'name',
+    'in',
+    'required',
+    'description',
+    'allowEmptyValue',
+    'collectionFormat',
+    'schema',
+]);
+function envelope(entry) {
+    return Object.fromEntries(Object.entries(entry).filter(([key]) => !ENVELOPE.has(key)));
+}
+/** An operation-level parameter replaces the path-level one it shadows. */
+function override(shared, own) {
+    const key = (p) => `${p.in}:${p.name}`;
+    const taken = new Set(own.map(key));
+    return [...shared.filter((p) => !taken.has(key(p))), ...own];
+}
+function bodyOf(op, params, dialect) {
+    const body = params.find((p) => p.in === 'body');
+    if (body) {
+        return { required: body.required, schema: body.schema };
+    }
+    const request = op.requestBody;
+    if (!isObject(request)) {
+        return undefined;
+    }
+    const schema = pickContent(request.content);
+    return schema
+        ? { required: request.required === true, schema: normalize(schema, dialect) }
+        : undefined;
+}
+function responsesOf(op, dialect) {
+    const responses = op.responses;
+    if (!isObject(responses)) {
+        return [];
+    }
+    const out = [];
+    for (const [code, value] of Object.entries(responses)) {
+        const status = Number(code);
+        if (!Number.isInteger(status) || status < 200 || status > 299 || !isObject(value)) {
+            continue;
+        }
+        // Swagger 2.0 hangs the schema straight off the response.
+        const schema = pickContent(value.content) ?? (isObject(value.schema) ? value.schema : undefined);
+        // A 204 says the call worked and nothing else; there is no node in it.
+        if (!schema) {
+            continue;
+        }
+        out.push({ status, doc: docOf(value), schema: normalize(schema, dialect) });
+    }
+    return out.sort((a, b) => a.status - b.status);
+}
+/** JSON first; anything else only if the operation speaks nothing else. */
+function pickContent(content) {
+    if (!isObject(content)) {
+        return undefined;
+    }
+    const entries = Object.entries(content).filter(([, v]) => isObject(v) && isObject(v.schema));
+    const json = entries.find(([type]) => /\bjson\b/.test(type)) ?? entries[0];
+    return json ? json[1].schema : undefined;
+}
+// ---------------------------------------------------------------------------
+// Settling names
+// ---------------------------------------------------------------------------
+/**
+ * Assigns every component schema its final id and rewrites the corpus to use
+ * it. A name owned by one document keeps it; a name two documents both claim
+ * becomes `<slug>.<Name>` on both sides, so an id never silently changes
+ * meaning depending on which file was read first.
+ */
+function settle(loaded) {
+    const owners = new Map();
+    for (const one of loaded) {
+        for (const name of one.types.keys()) {
+            owners.set(name, (owners.get(name) ?? 0) + 1);
+        }
+    }
+    const ids = loaded.map((one) => {
+        const map = new Map();
+        for (const name of one.types.keys()) {
+            map.set(name, owners.get(name) === 1 ? name : `${one.slug}.${name}`);
+        }
+        return map;
+    });
+    const types = {};
+    const typeSource = {};
+    const operations = [];
+    const seenOps = new Set();
+    loaded.forEach((one, index) => {
+        const map = ids[index];
+        for (const [name, schema] of one.types) {
+            const id = map.get(name);
+            types[id] = rewrite(schema, map);
+            typeSource[id] = one.doc.source;
+        }
+        for (const op of one.operations) {
+            operations.push({
+                ...op,
+                operationId: unique(op.operationId, one.slug, seenOps),
+                params: op.params.map((p) => ({ ...p, schema: rewrite(p.schema, map) })),
+                requestBody: op.requestBody && {
+                    ...op.requestBody,
+                    schema: rewrite(op.requestBody.schema, map),
+                },
+                responses: op.responses.map((r) => ({
+                    ...r,
+                    schema: r.schema && rewrite(r.schema, map),
+                })),
+            });
+        }
+    });
+    return { docs: loaded.map((one) => one.doc), operations, types, typeSource };
+}
+function unique(id, slug, seen) {
+    let candidate = seen.has(id) ? `${slug}.${id}` : id;
+    for (let n = 2; seen.has(candidate); n++) {
+        candidate = `${slug}.${id}_${n}`;
+    }
+    seen.add(candidate);
+    return candidate;
+}
+/** Points every internal `$ref` at the settled id, in one flat `$defs` space. */
+function rewrite(value, ids) {
+    if (Array.isArray(value)) {
+        return value.map((v) => rewrite(v, ids));
+    }
+    if (!isObject(value)) {
+        return value;
+    }
+    const out = {};
+    for (const [key, inner] of Object.entries(value)) {
+        if (key === '$ref' && typeof inner === 'string') {
+            const name = /^#\/(?:components\/schemas|definitions|\$defs)\/(.+)$/.exec(inner)?.[1];
+            const decoded = name && decodeURIComponent(name.replace(/~1/g, '/').replace(/~0/g, '~'));
+            out.$ref = decoded ? `#/$defs/${ids.get(decoded) ?? decoded}` : inner;
+        }
+        else {
+            out[key] = rewrite(inner, ids);
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=spec.js.map

package/dist/schema/store.d.ts

@@ -0,0 +1,32 @@
+import { type Connection, type Table } from '@lancedb/lancedb';
+import type { EntityRecord } from './entities.ts';
+export interface StoreFilter {
+    kinds?: readonly string[];
+    directions?: readonly string[];
+    methodTypes?: readonly string[];
+}
+export interface Hit {
+    record: EntityRecord;
+    /** 0-based position in this query's own result list */
+    rank: number;
+    /** what the store thought, where it says; 0 when it does not */
+    relevance: number;
+}
+export interface WriteResult {
+    rows: number;
+    fts: boolean;
+    vector: boolean;
+}
+export declare function writeStore(dir: string, rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+export declare class EntityStore {
+    #private;
+    constructor(db: Connection, table: Table);
+    static open(dir: string): Promise<EntityStore>;
+    /**
+     * One hybrid query: the same string goes to the full-text side and, as a
+     * vector, to the nearest-neighbour side, and LanceDB fuses the two.
+     */
+    search(text: string, vector: Float32Array, filter: StoreFilter, limit: number): Promise<Hit[]>;
+    close(): void;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/schema/store.js

@@ -0,0 +1,126 @@
+import { connect, Index } from '@lancedb/lancedb';
+import { CliError, EXIT } from '@zenera/cli/lib';
+import { lancePath } from "./files.js";
+// ---------------------------------------------------------------------------
+// The hybrid index
+//
+// LanceDB holds one table: a row per graph node, a materialized `text` column
+// carrying both the embedding and the full-text index, and the handful of
+// enum columns a query filters on.
+//
+// Those enums are the *only* thing that reaches the SQL predicate. Exclusion
+// lists — which arrive from a model, or from a shell — are applied afterwards
+// in JavaScript. Escaping them into `where()` would work right up until it did
+// not, and there is nothing here that a filter string buys.
+// ---------------------------------------------------------------------------
+const TABLE = 'entities';
+/** Below this an IVF index has nothing to train on, and a flat scan is faster. */
+const VECTOR_INDEX_MIN_ROWS = 2000;
+const KINDS = new Set(['method', 'type', 'property']);
+const DIRECTIONS = new Set(['input', 'output', 'both', 'none']);
+const METHOD_TYPES = new Set(['read_only', 'read_write', 'n/a']);
+export async function writeStore(dir, rows, vectors) {
+    if (rows.length === 0) {
+        throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+    }
+    const db = await connect(lancePath(dir));
+    // Every column is always populated — never null — so the Arrow schema is
+    // inferred from the first row without a declaration to keep in step.
+    const table = await db.createTable(TABLE, rows.map((row, i) => ({ ...row, vector: vectors[i] })), { mode: 'overwrite' });
+    await table.createIndex('text', { config: Index.fts() });
+    for (const column of ['kind', 'direction', 'methodType']) {
+        await table.createIndex(column, { config: Index.bitmap() });
+    }
+    const vector = rows.length >= VECTOR_INDEX_MIN_ROWS;
+    if (vector) {
+        await table.createIndex('vector');
+    }
+    db.close();
+    return { rows: rows.length, fts: true, vector };
+}
+export class EntityStore {
+    #db;
+    #table;
+    constructor(db, table) {
+        this.#db = db;
+        this.#table = table;
+    }
+    static async open(dir) {
+        const db = await connect(lancePath(dir));
+        try {
+            return new EntityStore(db, await db.openTable(TABLE));
+        }
+        catch {
+            db.close();
+            throw new CliError(`${dir} holds no searchable table`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
+        }
+    }
+    /**
+     * One hybrid query: the same string goes to the full-text side and, as a
+     * vector, to the nearest-neighbour side, and LanceDB fuses the two.
+     */
+    async search(text, vector, filter, limit) {
+        const predicate = where(filter);
+        let query = this.#table.query().nearestToText(text).nearestTo(vector).limit(limit);
+        if (predicate) {
+            query = query.where(predicate);
+        }
+        const rows = (await query.toArray());
+        return rows.map((row, rank) => ({
+            record: strip(row),
+            rank,
+            relevance: score(row),
+        }));
+    }
+    close() {
+        this.#db.close();
+    }
+}
+// ---------------------------------------------------------------------------
+/** Closed vocabularies only. Anything else is a bug, and is treated as one. */
+function where(filter) {
+    const clauses = [
+        clause('kind', filter.kinds, KINDS),
+        clause('direction', filter.directions, DIRECTIONS),
+        clause('methodType', filter.methodTypes, METHOD_TYPES),
+    ].filter(Boolean);
+    return clauses.join(' AND ');
+}
+function clause(column, values, allowed) {
+    if (!values || values.length === 0) {
+        return '';
+    }
+    for (const value of values) {
+        if (!allowed.has(value)) {
+            throw new Error(`${column} cannot be ${JSON.stringify(value)}`);
+        }
+    }
+    return `${column} IN (${values.map((v) => `'${v}'`).join(', ')})`;
+}
+/** A hybrid query reports relevance; a one-sided one reports a distance. */
+function score(row) {
+    const relevance = row._relevance_score ?? row._score;
+    if (typeof relevance === 'number') {
+        return relevance;
+    }
+    const distance = row._distance;
+    return typeof distance === 'number' ? 1 / (1 + distance) : 0;
+}
+function strip(row) {
+    return {
+        id: row.id,
+        kind: row.kind,
+        name: row.name,
+        parent: row.parent,
+        doc: row.doc,
+        direction: row.direction,
+        methodType: row.methodType,
+        httpMethod: row.httpMethod,
+        path: row.path,
+        source: row.source,
+        signature: row.signature,
+        required: row.required,
+        text: row.text,
+    };
+}
+//# sourceMappingURL=store.js.map

package/dist/schema/subgraph.d.ts

@@ -0,0 +1,42 @@
+import type { ApiGraph, NodeAttrs, Relation } from './graph.ts';
+export interface Seed {
+    id: string;
+    /** the query string that found it */
+    term: string;
+    /** which field of the query that string came from */
+    field: string;
+    score: number;
+}
+export interface SubgraphNode {
+    id: string;
+    kind: string;
+    attributes: NodeAttrs;
+    /** the search found this one; everything else is here to connect it */
+    hit: boolean;
+    score: number;
+}
+export interface SubgraphEdge {
+    source: string;
+    target: string;
+    relation: Relation;
+    status: number;
+    in: string;
+}
+export interface Subgraph {
+    nodes: SubgraphNode[];
+    edges: SubgraphEdge[];
+    hits: string[];
+    score: number;
+    /** nodes were dropped to stay inside the budget */
+    truncated: boolean;
+}
+export interface StitchOptions {
+    maxHops?: number;
+    maxNodes?: number;
+}
+export declare const DEFAULT_MAX_HOPS = 3;
+export declare const DEFAULT_MAX_NODES = 200;
+export declare function stitch(graph: ApiGraph, seeds: readonly Seed[], options?: StitchOptions): Subgraph[];
+/** Breadth-first, ignoring edge direction, giving up past `maxHops`. */
+export declare function path(graph: ApiGraph, from: string, to: string, maxHops: number): string[] | undefined;
+//# sourceMappingURL=subgraph.d.ts.map