@zenera/rag 1.1.0

package/dist/schema/subgraph.js

@@ -0,0 +1,272 @@
+export const DEFAULT_MAX_HOPS = 3;
+export const DEFAULT_MAX_NODES = 200;
+/** How many fields of one type are pulled in when the type itself was the hit. */
+const FIELDS_PER_TYPE = 24;
+/** How many when one field was the hit and the rest of the type is only context. */
+const FIELDS_PER_OWNER = 8;
+/** How far containment is read backwards, how many calls are enough, how wide it may get. */
+const REACH_HOPS = 6;
+const REACH_CALLERS = 3;
+const REACH_VISITS = 512;
+const OWNS = new Set(['HAS_PROPERTY', 'HAS_PARAM']);
+const CALLS = new Set(['TAKES_INPUT', 'RETURNS_OUTPUT']);
+const OF_TYPE = new Set(['OF_TYPE']);
+const WITHIN = new Set(['COMPOSES', 'ITEM_OF']);
+/** Containment read backwards: a type to the fields holding it, a field to its owner. */
+const UPWARD = new Set([
+    'HAS_PROPERTY',
+    'HAS_PARAM',
+    'OF_TYPE',
+    'COMPOSES',
+    'ITEM_OF',
+]);
+/** Why a node is here. The budget is spent in this order, so fills go first. */
+const LINK = 0;
+const FILL = 1;
+export function stitch(graph, seeds, options = {}) {
+    const maxHops = options.maxHops ?? DEFAULT_MAX_HOPS;
+    const maxNodes = options.maxNodes ?? DEFAULT_MAX_NODES;
+    const live = seeds.filter((s) => graph.hasNode(s.id));
+    if (live.length === 0) {
+        return [];
+    }
+    const scores = new Map();
+    for (const seed of live) {
+        scores.set(seed.id, (scores.get(seed.id) ?? 0) + seed.score);
+    }
+    const keep = new Map();
+    const put = (id, tier) => keep.set(id, Math.min(keep.get(id) ?? tier, tier));
+    for (const id of scores.keys()) {
+        put(id, LINK);
+    }
+    for (const seed of live) {
+        for (const node of anchor(graph, seed.id)) {
+            put(node.id, node.tier);
+        }
+    }
+    for (const node of connect(graph, live, maxHops)) {
+        put(node, LINK);
+    }
+    return components(graph, keep, scores).map((part) => materialize(graph, part, scores, keep, maxNodes));
+}
+// ---------------------------------------------------------------------------
+/** The nodes that make one hit legible on its own. */
+function anchor(graph, id) {
+    const kind = graph.getNodeAttribute(id, 'kind');
+    const out = [];
+    const link = (ids) => out.push(...ids.map((n) => ({ id: n, tier: LINK })));
+    const fill = (ids) => out.push(...ids.map((n) => ({ id: n, tier: FILL })));
+    if (kind === 'property') {
+        for (const owner of related(graph, id, 'in', OWNS)) {
+            link([owner, ...reach(graph, owner)]);
+            fill(related(graph, owner, 'out', OWNS).slice(0, FIELDS_PER_OWNER));
+        }
+        // What the field is *of* matters as much as what holds it.
+        link(related(graph, id, 'out', OF_TYPE));
+        return out;
+    }
+    if (kind === 'type') {
+        link(reach(graph, id));
+        link(related(graph, id, 'out', WITHIN));
+        fill(related(graph, id, 'out', OWNS).slice(0, FIELDS_PER_TYPE));
+        return out;
+    }
+    for (const target of related(graph, id, 'out', new Set([...OWNS, ...CALLS]))) {
+        link([target]);
+        // A parameter typed `Species` is part of reading the call, and is one
+        // hop further out than anything else here.
+        if (graph.getNodeAttribute(target, 'kind') === 'property') {
+            link(related(graph, target, 'out', OF_TYPE));
+        }
+    }
+    return out;
+}
+/**
+ * The way in, and only the way in: containment read backwards from a type until
+ * it lands on the operations carrying it, keeping the nodes on the route and
+ * nothing beside them. A shared `Money` is held by half the spec, so this stops
+ * at the first few calls rather than reporting all of them.
+ */
+function reach(graph, from) {
+    const previous = new Map([[from, from]]);
+    const out = [];
+    let frontier = [from];
+    let found = 0;
+    for (let hop = 0; hop < REACH_HOPS && frontier.length > 0 && found < REACH_CALLERS; hop++) {
+        const next = [];
+        for (const node of frontier) {
+            for (const caller of related(graph, node, 'in', CALLS)) {
+                if (previous.has(caller)) {
+                    continue;
+                }
+                previous.set(caller, node);
+                out.push(...trace(previous, from, caller));
+                if (++found >= REACH_CALLERS) {
+                    break;
+                }
+            }
+            if (found >= REACH_CALLERS || previous.size >= REACH_VISITS) {
+                break;
+            }
+            for (const up of related(graph, node, 'in', UPWARD)) {
+                if (!previous.has(up)) {
+                    previous.set(up, node);
+                    next.push(up);
+                }
+            }
+        }
+        frontier = next;
+    }
+    return out;
+}
+function related(graph, id, side, relations) {
+    const edges = side === 'in' ? graph.inEdges(id) : graph.outEdges(id);
+    const out = [];
+    for (const edge of edges) {
+        if (relations.has(graph.getEdgeAttribute(edge, 'relation'))) {
+            out.push(side === 'in' ? graph.source(edge) : graph.target(edge));
+        }
+    }
+    return [...new Set(out)];
+}
+/**
+ * The joins between hits. Only pairs from different query terms are tried:
+ * two properties found by the same phrase are already one thought, and paying
+ * for a path between them buys nothing but hops.
+ */
+function connect(graph, seeds, maxHops) {
+    const out = new Set();
+    for (let i = 0; i < seeds.length; i++) {
+        for (let j = i + 1; j < seeds.length; j++) {
+            const a = seeds[i];
+            const b = seeds[j];
+            if (a.term === b.term || a.id === b.id) {
+                continue;
+            }
+            for (const node of path(graph, a.id, b.id, maxHops) ?? []) {
+                out.add(node);
+            }
+        }
+    }
+    return out;
+}
+/** Breadth-first, ignoring edge direction, giving up past `maxHops`. */
+export function path(graph, from, to, maxHops) {
+    if (from === to) {
+        return [from];
+    }
+    const previous = new Map([[from, from]]);
+    let frontier = [from];
+    for (let hop = 0; hop < maxHops && frontier.length > 0; hop++) {
+        const next = [];
+        for (const node of frontier) {
+            for (const neighbor of graph.neighbors(node)) {
+                if (previous.has(neighbor)) {
+                    continue;
+                }
+                previous.set(neighbor, node);
+                if (neighbor === to) {
+                    return trace(previous, from, to);
+                }
+                next.push(neighbor);
+            }
+        }
+        frontier = next;
+    }
+    return undefined;
+}
+function trace(previous, from, to) {
+    const out = [to];
+    while (out[0] !== from) {
+        out.unshift(previous.get(out[0]));
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+/** Connected pieces of the kept set, biggest score first. */
+function components(graph, keep, scores) {
+    const seen = new Set();
+    const out = [];
+    for (const start of keep.keys()) {
+        if (seen.has(start)) {
+            continue;
+        }
+        const part = new Set();
+        const stack = [start];
+        while (stack.length > 0) {
+            const node = stack.pop();
+            if (seen.has(node)) {
+                continue;
+            }
+            seen.add(node);
+            part.add(node);
+            for (const neighbor of graph.neighbors(node)) {
+                if (keep.has(neighbor) && !seen.has(neighbor)) {
+                    stack.push(neighbor);
+                }
+            }
+        }
+        out.push(part);
+    }
+    return out.sort((a, b) => total(b, scores) - total(a, scores));
+}
+function total(part, scores) {
+    let sum = 0;
+    for (const node of part) {
+        sum += scores.get(node) ?? 0;
+    }
+    return sum;
+}
+/**
+ * Turns one component into a result, spending the node budget on the hits
+ * first, then on what connects them to a call, and only then on the fields
+ * that came along for context — so what is dropped is always the part
+ * furthest from anything anyone asked about.
+ */
+function materialize(graph, part, scores, tiers, maxNodes) {
+    const hits = [...part].filter((id) => scores.has(id));
+    const kept = new Set(hits.slice(0, maxNodes));
+    const rest = [...part]
+        .filter((id) => !kept.has(id))
+        .sort((a, b) => (tiers.get(a) ?? FILL) - (tiers.get(b) ?? FILL) || a.localeCompare(b));
+    for (const id of rest) {
+        if (kept.size >= maxNodes) {
+            break;
+        }
+        kept.add(id);
+    }
+    const nodes = [...kept]
+        .map((id) => ({
+        id,
+        kind: graph.getNodeAttribute(id, 'kind'),
+        attributes: graph.getNodeAttributes(id),
+        hit: scores.has(id),
+        score: scores.get(id) ?? 0,
+    }))
+        .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+    const edges = [];
+    for (const id of kept) {
+        for (const edge of graph.outEdges(id)) {
+            const target = graph.target(edge);
+            if (!kept.has(target)) {
+                continue;
+            }
+            const attrs = graph.getEdgeAttributes(edge);
+            edges.push({
+                source: id,
+                target,
+                relation: attrs.relation,
+                status: attrs.status,
+                in: attrs.in,
+            });
+        }
+    }
+    return {
+        nodes,
+        edges: edges.sort((a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target)),
+        hits: hits.filter((id) => kept.has(id)),
+        score: total(part, scores),
+        truncated: kept.size < part.size,
+    };
+}
+//# sourceMappingURL=subgraph.js.map

package/dist/schema/tools.d.ts

@@ -0,0 +1,10 @@
+import { type AnyTool } from '@zenera/neo';
+import { type Format } from '../present.ts';
+import type { SchemaIndex } from './search.ts';
+export interface SchemaToolOptions {
+    /** what `format` defaults to when the model does not say */
+    format?: Format;
+    docs?: boolean;
+}
+export declare function schemaTools<TCtx = unknown>(index: SchemaIndex, options?: SchemaToolOptions): AnyTool<TCtx>[];
+//# sourceMappingURL=tools.d.ts.map

package/dist/schema/tools.js

@@ -0,0 +1,242 @@
+import { tool } from '@zenera/neo';
+import { FORMATS, isFormat, present } from "../present.js";
+import { isEmpty, parseQuery, QueryError } from "../query.js";
+import { toTypeScript } from "./hydrate.js";
+import { stitch } from "./subgraph.js";
+// ---------------------------------------------------------------------------
+// The same index, given to an agent
+//
+// Four tools over one engine. Three of them search, and the fourth deliberately
+// does not: `find_types_with_property` is a graph lookup, for the moment after
+// the compiler says `'password' does not exist in type 'PublicUserProfile'`.
+// At that point the model does not need to be reminded what a password is —
+// it needs the list of types that have one, and an embedding of the word will
+// only rank the guess it already made near the top again.
+// ---------------------------------------------------------------------------
+const GROUP = 'schema';
+/** Kept small on purpose: a tool result is prompt, and the model asked for one thing. */
+const DEFAULT_LIMIT = 4;
+const DEFAULT_MAX_NODES = 60;
+const MAX_CANDIDATES = 25;
+export function schemaTools(index, options = {}) {
+    const fallback = options.format ?? 'text';
+    const docs = options.docs ?? true;
+    const searchApi = tool({
+        name: 'search_api',
+        group: GROUP,
+        description: 'Searches the API description and answers with the connected piece of it that ' +
+            'matched: the operations, the schemas they carry and the fields inside them. ' +
+            'Put the intent in the field that matches what is wanted — a request field in ' +
+            'input_properties, a response field in output_properties — rather than putting ' +
+            'everything in `all`, which cannot filter. Pass the ids from a previous answer ' +
+            'in exclude_ids to be shown something new instead of the same thing again.',
+        parameters: {
+            type: 'object',
+            properties: {
+                all: list('Free search over operations, schemas and fields alike.'),
+                methods: list('What the call does, e.g. "reset a user password".'),
+                types: list('What the schema is, e.g. "a billing invoice".'),
+                input_types: list('Schemas a call accepts.'),
+                output_types: list('Schemas a call returns.'),
+                properties: list('Fields or parameters, either side.'),
+                input_properties: list('Fields in a request body, or query/path parameters.'),
+                output_properties: list('Fields in a response body.'),
+                direction: {
+                    type: 'string',
+                    enum: ['input', 'output', 'any'],
+                    description: 'Which side `types` and `properties` are read on.',
+                },
+                method_type: {
+                    type: 'string',
+                    enum: ['read_only', 'read_write', 'any'],
+                    description: 'read_only is GET/HEAD/OPTIONS; read_write is everything else.',
+                },
+                exclude_ids: list('Node ids already seen, as printed in an earlier answer.'),
+                exclude_methods: list('Operation names to leave out.'),
+                exclude_types: list('Schema names to leave out.'),
+                exclude_properties: list('Field names to leave out.'),
+                limit: {
+                    type: 'integer',
+                    description: `Results per phrase. Default ${DEFAULT_LIMIT}.`,
+                },
+                max_nodes: {
+                    type: 'integer',
+                    description: `Largest answer, in nodes. Default ${DEFAULT_MAX_NODES}.`,
+                },
+                format: {
+                    type: 'string',
+                    enum: [...FORMATS],
+                    description: `How to write it. Default ${fallback}. Use "ts" to get types to code against.`,
+                },
+            },
+            additionalProperties: false,
+        },
+        execute: async (args) => {
+            const { format, ...rest } = args;
+            let query;
+            try {
+                query = parseQuery(rest);
+            }
+            catch (err) {
+                return {
+                    error: err instanceof QueryError ? err.message : String(err),
+                };
+            }
+            if (isEmpty(query)) {
+                return { error: 'nothing was asked for', hint: 'fill at least one search field' };
+            }
+            const result = await index.search({
+                limit: DEFAULT_LIMIT,
+                max_nodes: DEFAULT_MAX_NODES,
+                ...query,
+            });
+            if (result.subgraphs.length === 0) {
+                return { found: 0, hint: 'try fewer words, or `all` instead of a narrower field' };
+            }
+            return {
+                found: result.subgraphs.length,
+                ids: result.subgraphs.flatMap((s) => s.hits),
+                truncated: result.subgraphs.some((s) => s.truncated),
+                api: await present(index, result.subgraphs, chosen(format, fallback), { docs }),
+            };
+        },
+    });
+    const describeTypes = tool({
+        name: 'describe_types',
+        group: GROUP,
+        description: 'Prints named schemas as TypeScript declarations, together with everything they ' +
+            'refer to, so the result compiles on its own. Use it once search has named a ' +
+            'schema and the exact fields are what is needed.',
+        parameters: {
+            type: 'object',
+            properties: {
+                names: list('Schema names, e.g. ["ResetPasswordPayload"]. Not node ids.'),
+                only: list('Restrict every schema to these field names.'),
+            },
+            required: ['names'],
+            additionalProperties: false,
+        },
+        execute: async ({ names, only }) => {
+            const known = names.filter((name) => index.graph.hasNode(`Type:${name}`));
+            const missing = names.filter((name) => !known.includes(name));
+            if (known.length === 0) {
+                return {
+                    error: `no such schema: ${names.join(', ')}`,
+                    hint: 'search for it first',
+                };
+            }
+            const schemas = await index.schemas();
+            const sub = subgraphOf(index, known.map((name) => `Type:${name}`));
+            const code = toTypeScript(sub, schemas, { docs, onlyHits: false });
+            return {
+                typescript: only?.length ? narrow(code, only) : code,
+                ...(missing.length > 0 ? { missing } : {}),
+            };
+        },
+    });
+    const findTypesWithProperty = tool({
+        name: 'find_types_with_property',
+        group: GROUP,
+        description: 'Lists every schema that has a field of this name. Exact lookup, no searching: ' +
+            'reach for it when a field was put on the wrong type and the right one has to be ' +
+            'found, for instance after a compiler error saying the property does not exist.',
+        parameters: {
+            type: 'object',
+            properties: {
+                property: { type: 'string', description: 'The field name, e.g. "password".' },
+                direction: {
+                    type: 'string',
+                    enum: ['input', 'output', 'any'],
+                    description: 'Only schemas used on this side of a call.',
+                },
+            },
+            required: ['property'],
+            additionalProperties: false,
+        },
+        execute: async ({ property, direction }) => {
+            const wanted = property.toLowerCase();
+            const candidates = [];
+            index.graph.forEachNode((id, a) => {
+                if (a.kind !== 'property' || a.name.toLowerCase() !== wanted || !a.parent) {
+                    return;
+                }
+                const owner = index.graph.getNodeAttributes(`Type:${a.parent}`) ?? a;
+                if (direction && direction !== 'any' && !onSide(owner.direction, direction)) {
+                    return;
+                }
+                candidates.push({
+                    type: a.parent,
+                    id,
+                    signature: a.signature,
+                    required: a.required,
+                    direction: owner.direction,
+                    doc: a.doc || owner.doc,
+                });
+            });
+            return candidates.length === 0
+                ? { found: 0, hint: 'try search_api with the field in input_properties' }
+                : { found: candidates.length, candidates: candidates.slice(0, MAX_CANDIDATES) };
+        },
+    });
+    const listMethods = tool({
+        name: 'list_methods',
+        group: GROUP,
+        description: 'Lists operations by path, with no searching. Use it to see the shape of the API ' +
+            'before deciding what to ask for.',
+        parameters: {
+            type: 'object',
+            properties: {
+                contains: { type: 'string', description: 'Only paths holding this text.' },
+                method_type: {
+                    type: 'string',
+                    enum: ['read_only', 'read_write', 'any'],
+                },
+            },
+            additionalProperties: false,
+        },
+        execute: async ({ contains, method_type }) => {
+            const needle = contains?.toLowerCase();
+            const rows = [];
+            index.graph.forEachNode((_id, a) => {
+                if (a.kind !== 'method') {
+                    return;
+                }
+                if (needle && !a.path.toLowerCase().includes(needle)) {
+                    return;
+                }
+                if (method_type && method_type !== 'any' && a.methodType !== method_type) {
+                    return;
+                }
+                rows.push(`${a.httpMethod} ${a.path}  ${a.name}${a.doc ? ` — ${a.doc}` : ''}`);
+            });
+            return { found: rows.length, methods: rows.sort() };
+        },
+    });
+    return [searchApi, describeTypes, findTypesWithProperty, listMethods];
+}
+// ---------------------------------------------------------------------------
+function list(description) {
+    return { type: 'array', items: { type: 'string' }, description };
+}
+function chosen(format, fallback) {
+    return format && isFormat(format) ? format : fallback;
+}
+function onSide(direction, wanted) {
+    return direction === wanted || direction === 'both';
+}
+function subgraphOf(index, ids) {
+    const [first] = stitch(index.graph, ids.map((id) => ({ id, term: id, field: 'describe', score: 1 })), { maxNodes: DEFAULT_MAX_NODES * ids.length });
+    return first ?? { nodes: [], edges: [], hits: [], score: 0, truncated: false };
+}
+/** Keeps the declarations, drops the field lines nobody asked about. */
+function narrow(code, only) {
+    const wanted = new Set(only);
+    return code
+        .split('\n')
+        .filter((line) => {
+        const field = /^\s{4}'?([A-Za-z0-9_$-]+)'?\??:/.exec(line);
+        return !field || wanted.has(field[1]);
+    })
+        .join('\n');
+}
+//# sourceMappingURL=tools.js.map

package/dist/schema/typescript.d.ts

@@ -0,0 +1,22 @@
+import { type Schema } from './schema.ts';
+export interface PrintOptions {
+    /** emit `/** … *\/` above types and properties */
+    docs?: boolean;
+    /** truncate one comment to this many characters */
+    maxDoc?: number;
+    /** when given, only these properties of an object are printed */
+    only?: ReadonlySet<string>;
+}
+export declare class Printer {
+    #private;
+    constructor(types: Readonly<Record<string, Schema>>);
+    /** The TypeScript identifier a type id is printed as. */
+    identifier(id: string): string;
+    /** An inline type expression — what goes to the right of a colon. */
+    signature(schema: unknown, depth?: number): string;
+    /** A whole `export interface` / `export type` declaration for one id. */
+    declaration(id: string, options?: PrintOptions): string;
+}
+/** A key that is not a plain identifier has to be quoted. */
+export declare function property(key: string): string;
+//# sourceMappingURL=typescript.d.ts.map