@zenera/rag 1.1.0

package/dist/schema/render.js

@@ -0,0 +1,254 @@
+const HIT = '»';
+export function render(sub, format, options = {}) {
+    switch (format) {
+        case 'mermaid':
+            return toMermaid(sub, options);
+        case 'mermaid-flowchart':
+            return toFlowchart(sub, options);
+        default:
+            return toText(sub, options);
+    }
+}
+// ---------------------------------------------------------------------------
+// Text
+// ---------------------------------------------------------------------------
+export function toText(sub, options = {}) {
+    const view = new View(sub);
+    const lines = [];
+    const methods = view.of('method');
+    if (methods.length > 0) {
+        lines.push('methods');
+        for (const method of methods) {
+            lines.push(...methodLines(view, method, options));
+        }
+    }
+    const types = view.of('type');
+    if (types.length > 0) {
+        lines.push(methods.length > 0 ? '' : '', 'types');
+        for (const type of types) {
+            lines.push(...typeLines(view, type, options));
+        }
+    }
+    const loose = view.of('property').filter((p) => !view.owned.has(p.id));
+    if (loose.length > 0) {
+        lines.push('', 'properties');
+        for (const property of loose) {
+            lines.push(`  ${mark(property)}${field(property)}${doc(property, options, ' ')}`);
+        }
+    }
+    if (sub.truncated) {
+        lines.push('', '… truncated to stay inside the node budget');
+    }
+    return lines.filter((line, at) => line !== '' || at > 0).join('\n');
+}
+function methodLines(view, method, options) {
+    const a = method.attributes;
+    const out = [
+        `  ${mark(method)}${a.httpMethod} ${a.path}  ${a.name}${doc(method, options, ' —')}`,
+    ];
+    for (const edge of view.out(method.id, 'HAS_PARAM')) {
+        const node = view.node(edge.target);
+        if (node) {
+            out.push(`      ${mark(node)}${field(node)}  (${edge.in})${doc(node, options, ' —')}`);
+        }
+    }
+    for (const edge of view.out(method.id, 'TAKES_INPUT')) {
+        out.push(`      accepts ${view.name(edge.target)}`);
+    }
+    for (const edge of view.out(method.id, 'RETURNS_OUTPUT')) {
+        out.push(`      returns ${view.name(edge.target)}  (${edge.status})`);
+    }
+    return out;
+}
+function typeLines(view, type, options) {
+    const out = [
+        `  ${mark(type)}${type.attributes.name}${side(type.attributes)}${doc(type, options, ' —')}`,
+    ];
+    const composes = view.out(type.id, 'COMPOSES').map((e) => view.name(e.target));
+    if (composes.length > 0) {
+        out.push(`      composes ${composes.join(', ')}`);
+    }
+    const items = view.out(type.id, 'ITEM_OF').map((e) => view.name(e.target));
+    if (items.length > 0) {
+        out.push(`      array of ${items.join(', ')}`);
+    }
+    for (const edge of view.out(type.id, 'HAS_PROPERTY')) {
+        const node = view.node(edge.target);
+        if (node) {
+            out.push(`      ${mark(node)}${field(node)}${doc(node, options, ' —')}`);
+        }
+    }
+    return out;
+}
+function field(node) {
+    const a = node.attributes;
+    return `${a.name}${a.required ? '' : '?'}: ${a.signature || 'unknown'}`;
+}
+function side(a) {
+    return a.direction === 'none' ? '' : `  (${a.direction})`;
+}
+function mark(node) {
+    return node.hit ? `${HIT} ` : '  ';
+}
+function doc(node, options, lead) {
+    if (!options.docs || !node.attributes.doc) {
+        return '';
+    }
+    return `${lead} ${clip(node.attributes.doc, options.maxDoc ?? 120)}`;
+}
+// ---------------------------------------------------------------------------
+// Mermaid
+// ---------------------------------------------------------------------------
+export function toMermaid(sub, options = {}) {
+    const view = new View(sub);
+    const lines = ['classDiagram'];
+    for (const type of view.of('type')) {
+        const fields = view
+            .out(type.id, 'HAS_PROPERTY')
+            .map((e) => view.node(e.target))
+            .filter(Boolean);
+        lines.push(`    class ${view.id(type.id)} {`);
+        for (const property of fields) {
+            const a = property.attributes;
+            lines.push(`        ${a.required ? '+' : '-'}${safe(a.name)} : ${safe(a.signature)}`);
+        }
+        lines.push('    }');
+    }
+    for (const method of view.of('method')) {
+        const id = view.id(method.id);
+        const a = method.attributes;
+        lines.push(`    class ${id} {`, `        <<${a.httpMethod} ${safe(a.path)}>>`, '    }');
+    }
+    for (const edge of sub.edges) {
+        if (!view.node(edge.source) || !view.node(edge.target)) {
+            continue;
+        }
+        // A field is a line inside a class, not a class of its own, so the type
+        // it is of has to be joined from whatever holds the field.
+        if (edge.relation === 'OF_TYPE') {
+            const owner = view.owner(edge.source);
+            if (owner) {
+                const label = safe(view.name(edge.source));
+                lines.push(`    ${view.id(owner)} --> ${view.id(edge.target)} : ${label}`);
+            }
+            continue;
+        }
+        const arrow = ARROWS[edge.relation];
+        if (!arrow) {
+            continue;
+        }
+        const label = edge.relation === 'RETURNS_OUTPUT' ? `returns ${edge.status}` : arrow.label;
+        lines.push(`    ${view.id(edge.source)} ${arrow.line} ${view.id(edge.target)} : ${label}`);
+    }
+    for (const node of sub.nodes) {
+        if (node.hit && options.docs && node.attributes.doc && node.kind !== 'property') {
+            const text = clip(node.attributes.doc, options.maxDoc ?? 100).replace(/"/g, "'");
+            lines.push(`    note for ${view.id(node.id)} "${text}"`);
+        }
+    }
+    return lines.join('\n');
+}
+const ARROWS = {
+    TAKES_INPUT: { line: '..>', label: 'accepts' },
+    RETURNS_OUTPUT: { line: '-->', label: 'returns' },
+    COMPOSES: { line: '--|>', label: 'composes' },
+    ITEM_OF: { line: 'o--', label: 'array of' },
+};
+export function toFlowchart(sub, options = {}) {
+    const view = new View(sub);
+    const lines = ['flowchart LR'];
+    for (const node of sub.nodes) {
+        const a = node.attributes;
+        const id = view.id(node.id);
+        const label = node.kind === 'method'
+            ? `${a.httpMethod} ${a.path}`
+            : node.kind === 'property'
+                ? `${a.name}: ${a.signature || 'unknown'}`
+                : a.name;
+        const [open, close] = node.kind === 'method' ? ['([', '])'] : ['[', ']'];
+        lines.push(`    ${id}${open}"${safe(node.hit ? `${HIT} ${label}` : label)}"${close}`);
+    }
+    for (const edge of sub.edges) {
+        const label = edge.relation === 'RETURNS_OUTPUT' ? `returns ${edge.status}` : LABELS[edge.relation];
+        lines.push(`    ${view.id(edge.source)} -->|${label}| ${view.id(edge.target)}`);
+    }
+    void options;
+    return lines.join('\n');
+}
+const LABELS = {
+    TAKES_INPUT: 'accepts',
+    HAS_PARAM: 'param',
+    RETURNS_OUTPUT: 'returns',
+    HAS_PROPERTY: 'has',
+    OF_TYPE: 'of type',
+    COMPOSES: 'composes',
+    ITEM_OF: 'array of',
+};
+// ---------------------------------------------------------------------------
+/** Indexed access to one subgraph, so a renderer is a loop and nothing else. */
+class View {
+    owned = new Set();
+    #nodes = new Map();
+    #out = new Map();
+    #owner = new Map();
+    #ids = new Map();
+    constructor(sub) {
+        for (const node of sub.nodes) {
+            this.#nodes.set(node.id, node);
+        }
+        for (const edge of sub.edges) {
+            const list = this.#out.get(edge.source);
+            if (list) {
+                list.push(edge);
+            }
+            else {
+                this.#out.set(edge.source, [edge]);
+            }
+            if (edge.relation === 'HAS_PROPERTY' || edge.relation === 'HAS_PARAM') {
+                this.owned.add(edge.target);
+                this.#owner.set(edge.target, edge.source);
+            }
+        }
+        const taken = new Set();
+        for (const node of sub.nodes) {
+            let id = safeId(node.attributes.name || node.id);
+            for (let n = 2; taken.has(id); n++) {
+                id = `${safeId(node.attributes.name || node.id)}_${n}`;
+            }
+            taken.add(id);
+            this.#ids.set(node.id, id);
+        }
+    }
+    node(id) {
+        return this.#nodes.get(id);
+    }
+    name(id) {
+        return this.#nodes.get(id)?.attributes.name ?? id;
+    }
+    /** The type or method a property hangs off, for renderers that draw it inline. */
+    owner(id) {
+        return this.#owner.get(id);
+    }
+    id(id) {
+        return this.#ids.get(id) ?? safeId(id);
+    }
+    of(kind) {
+        return [...this.#nodes.values()].filter((n) => n.kind === kind);
+    }
+    out(id, relation) {
+        return (this.#out.get(id) ?? []).filter((e) => e.relation === relation);
+    }
+}
+function clip(text, max) {
+    const one = text.replace(/\s+/g, ' ').trim();
+    return one.length > max ? `${one.slice(0, max - 1)}…` : one;
+}
+/** Mermaid takes the label apart on quotes, brackets and newlines. */
+function safe(text) {
+    return text.replace(/["\n]/g, ' ').replace(/[[\]{}<>|]/g, '');
+}
+function safeId(text) {
+    const cleaned = text.replace(/[^A-Za-z0-9_]+/g, '_').replace(/^_+|_+$/g, '');
+    return /^[A-Za-z_]/.test(cleaned) ? cleaned : `n_${cleaned}`;
+}
+//# sourceMappingURL=render.js.map

package/dist/schema/schema.d.ts

@@ -0,0 +1,11 @@
+export type Schema = Record<string, unknown>;
+export type Dialect = 'swagger-2.0' | 'openapi-3.0' | 'openapi-3.1';
+export declare const isObject: (v: unknown) => v is Schema;
+/** Converts one schema, and everything under it, into 2020-12. */
+export declare function normalize(root: unknown, dialect: Dialect): Schema;
+/** The component name a `$ref` points at, or undefined for anything foreign. */
+export declare function refName(value: unknown): string | undefined;
+/** Every component name reachable from a schema, one level of `$ref` deep. */
+export declare function refsIn(schema: unknown): string[];
+export declare function docOf(schema: unknown): string;
+//# sourceMappingURL=schema.d.ts.map

package/dist/schema/schema.js

@@ -0,0 +1,189 @@
+// ---------------------------------------------------------------------------
+// Schemas, made uniform — without losing their names
+//
+// Three dialects arrive here and one leaves, the same conversion the faker
+// does. What is deliberately *not* done here is dereferencing: a `$ref` is
+// left as the string it is, because a component's name is this package's
+// primary key. `#/components/schemas/ResetPasswordPayload` is the edge that
+// makes the graph a graph, and the word the model is going to say back.
+//
+// Nothing here recurses through a `$ref`, so nothing here can meet a cycle.
+// ---------------------------------------------------------------------------
+/** Subschema positions, by the shape of what sits in them. */
+const ONE = [
+    'not',
+    'if',
+    'then',
+    'else',
+    'contains',
+    'propertyNames',
+    'additionalProperties',
+    'unevaluatedProperties',
+    'additionalItems',
+    'unevaluatedItems',
+];
+const MAP = ['properties', 'patternProperties', '$defs', 'definitions'];
+const LIST = ['allOf', 'anyOf', 'oneOf', 'prefixItems'];
+/**
+ * Annotations that cost tokens and change no meaning. `discriminator` is
+ * conspicuously absent — the faker drops it, and here it is the whole reason a
+ * `oneOf` can be printed as a TypeScript tagged union rather than a bare one.
+ */
+const DROP = new Set(['externalDocs', 'xml', 'x-internal']);
+export const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+/** Converts one schema, and everything under it, into 2020-12. */
+export function normalize(root, dialect) {
+    if (!isObject(root)) {
+        return {};
+    }
+    return convert(root, dialect);
+}
+function walk(value, dialect) {
+    return isObject(value) ? convert(value, dialect) : value;
+}
+function convert(node, dialect) {
+    const out = {};
+    for (const [key, value] of Object.entries(node)) {
+        if (DROP.has(key) || value === undefined) {
+            continue;
+        }
+        if (ONE.includes(key)) {
+            out[key] = typeof value === 'boolean' ? value : walk(value, dialect);
+        }
+        else if (MAP.includes(key) && isObject(value)) {
+            const target = key === 'definitions' ? '$defs' : key;
+            out[target] = Object.fromEntries(Object.entries(value).map(([k, v]) => [
+                k,
+                typeof v === 'boolean' ? v : walk(v, dialect),
+            ]));
+        }
+        else if (LIST.includes(key) && Array.isArray(value)) {
+            out[key] = value.map((v) => walk(v, dialect));
+        }
+        else if (key === 'items') {
+            // Draft-4 and Swagger 2.0 spell tuples as an array here; 2020-12
+            // spells them `prefixItems` and keeps `items` for the tail.
+            out[Array.isArray(value) ? 'prefixItems' : 'items'] = Array.isArray(value)
+                ? value.map((v) => walk(v, dialect))
+                : walk(value, dialect);
+        }
+        else {
+            out[key] = value;
+        }
+    }
+    if (dialect !== 'openapi-3.1') {
+        widenNullable(node, out);
+        fixExclusive(out);
+    }
+    fixPattern(out);
+    // `type: file` is Swagger 2.0's way of saying "bytes".
+    if (out.type === 'file') {
+        out.type = 'string';
+    }
+    if (Array.isArray(out.required) && out.required.length === 0) {
+        delete out.required;
+    }
+    return out;
+}
+/**
+ * `pattern: /^[_a-z0-9-]+$/` — a JavaScript regex *literal*, delimiters and all
+ * — is written in real documents, and with the slashes left on it matches
+ * nothing. They come off, and an expression that still will not compile is
+ * dropped rather than shown to anyone.
+ */
+function fixPattern(out) {
+    const raw = out.pattern;
+    if (typeof raw !== 'string') {
+        return;
+    }
+    const literal = /^\/(.+)\/[dgimsuvy]*$/s.exec(raw);
+    const body = literal ? literal[1] : raw;
+    try {
+        new RegExp(body);
+        out.pattern = body;
+    }
+    catch {
+        delete out.pattern;
+    }
+}
+/** `nullable: true` is not a 2020-12 keyword; a union type is. */
+function widenNullable(node, out) {
+    if (node.nullable !== true) {
+        return;
+    }
+    delete out.nullable;
+    const type = out.type;
+    if (typeof type === 'string') {
+        out.type = [type, 'null'];
+    }
+    else if (Array.isArray(type)) {
+        out.type = type.includes('null') ? type : [...type, 'null'];
+    }
+    else if (Array.isArray(out.enum) && !out.enum.includes(null)) {
+        out.enum = [...out.enum, null];
+    }
+}
+/**
+ * Draft-4 spells an exclusive bound as a boolean flag on the inclusive one, so
+ * `exclusiveMinimum: true` left alone reads as "the minimum is 1".
+ */
+function fixExclusive(out) {
+    for (const [flag, bound] of [
+        ['exclusiveMinimum', 'minimum'],
+        ['exclusiveMaximum', 'maximum'],
+    ]) {
+        if (typeof out[flag] !== 'boolean') {
+            continue;
+        }
+        const value = out[bound];
+        if (out[flag] === true && typeof value === 'number') {
+            out[flag] = value;
+            delete out[bound];
+        }
+        else {
+            delete out[flag];
+        }
+    }
+}
+/** The component name a `$ref` points at, or undefined for anything foreign. */
+export function refName(value) {
+    if (!isObject(value) || typeof value.$ref !== 'string') {
+        return undefined;
+    }
+    const match = /^#\/(?:components\/schemas|definitions|\$defs)\/(.+)$/.exec(value.$ref);
+    return match
+        ? decodeURIComponent(match[1].replace(/~1/g, '/').replace(/~0/g, '~'))
+        : undefined;
+}
+/** Every component name reachable from a schema, one level of `$ref` deep. */
+export function refsIn(schema) {
+    const out = new Set();
+    const stack = [schema];
+    while (stack.length > 0) {
+        const node = stack.pop();
+        if (Array.isArray(node)) {
+            stack.push(...node);
+            continue;
+        }
+        if (!isObject(node)) {
+            continue;
+        }
+        const name = refName(node);
+        if (name !== undefined) {
+            out.add(name);
+            // A `$ref` may carry siblings in 2020-12, but none of them are the
+            // pointed-at schema, so there is nothing further down this branch.
+            continue;
+        }
+        stack.push(...Object.values(node));
+    }
+    return [...out];
+}
+export function docOf(schema) {
+    if (!isObject(schema)) {
+        return '';
+    }
+    const description = schema.description ?? schema.title;
+    return typeof description === 'string' ? description.trim() : '';
+}
+//# sourceMappingURL=schema.js.map

package/dist/schema/search.d.ts

@@ -0,0 +1,50 @@
+import type { Embedder } from '@zenera/neo';
+import { type Manifest, type OpenIndex } from './files.ts';
+import type { ApiGraph } from './graph.ts';
+import type { Schema } from './schema.ts';
+import type { Operation } from './spec.ts';
+import { EntityStore } from './store.ts';
+import { type Seed, type Subgraph } from './subgraph.ts';
+export type DirectionFilter = 'input' | 'output' | 'any';
+export type MethodTypeFilter = 'read_only' | 'read_write' | 'any';
+export interface SchemaQuery {
+    /** searched against everything, unfiltered */
+    all?: readonly string[];
+    methods?: readonly string[];
+    method_type?: MethodTypeFilter;
+    types?: readonly string[];
+    input_types?: readonly string[];
+    output_types?: readonly string[];
+    properties?: readonly string[];
+    input_properties?: readonly string[];
+    output_properties?: readonly string[];
+    /** the side `types` and `properties` are read on; the explicit fields override it */
+    direction?: DirectionFilter;
+    exclude_ids?: readonly string[];
+    exclude_methods?: readonly string[];
+    exclude_types?: readonly string[];
+    exclude_properties?: readonly string[];
+    /** seeds kept per query string */
+    limit?: number;
+    max_hops?: number;
+    max_nodes?: number;
+}
+export interface SearchResult {
+    seeds: Seed[];
+    subgraphs: Subgraph[];
+    /** query strings that matched nothing once exclusions were applied */
+    empty: string[];
+}
+export declare const DEFAULT_LIMIT = 5;
+export declare class SchemaIndex {
+    #private;
+    readonly manifest: Manifest;
+    readonly graph: ApiGraph;
+    constructor(index: OpenIndex, store: EntityStore, embedder: Embedder);
+    static open(dir: string, embedder: Embedder): Promise<SchemaIndex>;
+    schemas(): Promise<Record<string, Schema>>;
+    operations(): Promise<Operation[]>;
+    close(): void;
+    search(query: SchemaQuery, signal?: AbortSignal): Promise<SearchResult>;
+}
+//# sourceMappingURL=search.d.ts.map

package/dist/schema/search.js

@@ -0,0 +1,142 @@
+import { assertSameEmbedding, openIndex } from "./files.js";
+import { EntityStore } from "./store.js";
+import { DEFAULT_MAX_HOPS, DEFAULT_MAX_NODES, stitch, } from "./subgraph.js";
+export const DEFAULT_LIMIT = 5;
+/** Reciprocal rank fusion; the constant is the usual one and damps the top. */
+const RRF_K = 60;
+export class SchemaIndex {
+    manifest;
+    graph;
+    #index;
+    #store;
+    #embedder;
+    constructor(index, store, embedder) {
+        this.manifest = index.manifest;
+        this.graph = index.graph;
+        this.#index = index;
+        this.#store = store;
+        this.#embedder = embedder;
+    }
+    static async open(dir, embedder) {
+        const index = await openIndex(dir);
+        assertSameEmbedding(index.manifest, embedder.id);
+        return new SchemaIndex(index, await EntityStore.open(dir), embedder);
+    }
+    schemas() {
+        return this.#index.schemas();
+    }
+    operations() {
+        return this.#index.operations();
+    }
+    close() {
+        this.#store.close();
+    }
+    async search(query, signal) {
+        const terms = termsOf(query);
+        if (terms.length === 0) {
+            return { seeds: [], subgraphs: [], empty: [] };
+        }
+        const limit = query.limit ?? DEFAULT_LIMIT;
+        const excluded = exclusion(query);
+        const response = await this.#embedder.embed({
+            input: terms.map((t) => t.text),
+            taskType: 'query',
+            signal,
+        });
+        const seeds = [];
+        const empty = [];
+        for (const [at, term] of terms.entries()) {
+            const vector = Float32Array.from(response.vectors[at]);
+            // Over-fetch by what may be thrown away, so an exclusion list
+            // shortens the answer instead of emptying it.
+            const hits = await this.#store.search(term.text, vector, term.filter, limit + Math.min(excluded.size, limit * 4));
+            const kept = hits.filter((hit) => !excluded.has(hit.record)).slice(0, limit);
+            if (kept.length === 0) {
+                empty.push(term.text);
+            }
+            for (const hit of kept) {
+                seeds.push({
+                    id: hit.record.id,
+                    term: term.text,
+                    field: term.field,
+                    score: 1 / (RRF_K + hit.rank),
+                });
+            }
+        }
+        const subgraphs = stitch(this.graph, seeds, {
+            maxHops: query.max_hops ?? DEFAULT_MAX_HOPS,
+            maxNodes: query.max_nodes ?? DEFAULT_MAX_NODES,
+        });
+        return { seeds, subgraphs, empty };
+    }
+}
+// ---------------------------------------------------------------------------
+function termsOf(query) {
+    const method = methodTypes(query.method_type);
+    const loose = query.direction ?? 'any';
+    return [
+        ...group(query.all, 'all', { methodTypes: method.mixed }),
+        ...group(query.methods, 'methods', { kinds: ['method'], methodTypes: method.only }),
+        ...group(query.types, 'types', { kinds: ['type'], directions: sides(loose) }),
+        ...group(query.input_types, 'input_types', { kinds: ['type'], directions: sides('input') }),
+        ...group(query.output_types, 'output_types', {
+            kinds: ['type'],
+            directions: sides('output'),
+        }),
+        ...group(query.properties, 'properties', {
+            kinds: ['property'],
+            directions: sides(loose),
+        }),
+        ...group(query.input_properties, 'input_properties', {
+            kinds: ['property'],
+            directions: sides('input'),
+        }),
+        ...group(query.output_properties, 'output_properties', {
+            kinds: ['property'],
+            directions: sides('output'),
+        }),
+    ];
+}
+function group(texts, field, filter) {
+    return (texts ?? [])
+        .map((text) => text.trim())
+        .filter(Boolean)
+        .map((text) => ({ field, text, filter }));
+}
+function sides(direction) {
+    return direction === 'any' ? undefined : [direction, 'both'];
+}
+/**
+ * A method-type filter is about methods. Applied whole to an unfiltered query
+ * it would also throw away every type and property, since those carry `n/a`.
+ */
+function methodTypes(filter) {
+    if (!filter || filter === 'any') {
+        return {};
+    }
+    return { only: [filter], mixed: [filter, 'n/a'] };
+}
+function exclusion(query) {
+    const ids = new Set(query.exclude_ids ?? []);
+    const methods = new Set(query.exclude_methods ?? []);
+    const types = new Set(query.exclude_types ?? []);
+    const properties = new Set(query.exclude_properties ?? []);
+    const size = ids.size + methods.size + types.size + properties.size;
+    return {
+        size,
+        has(record) {
+            if (ids.has(record.id)) {
+                return true;
+            }
+            switch (record.kind) {
+                case 'method':
+                    return methods.has(record.name);
+                case 'type':
+                    return types.has(record.name);
+                default:
+                    return properties.has(record.name);
+            }
+        },
+    };
+}
+//# sourceMappingURL=search.js.map