@zenera/rag 1.1.5 → 1.1.8

package/dist/schema/tools.js

@@ -1,15 +1,17 @@
 import { tool } from '@zenera/neo';
-import { FORMATS, isFormat, present } from "../present.js";
-import { isEmpty, parseQuery, QueryError } from "../query.js";
+import { loose, matcher, PatternError } from "../common/match.js";
+import { FORMATS, isFormat, present } from "./present.js";
+import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { toTypeScript } from "./hydrate.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
-import { loose, matcher, PatternError } from "./match.js";
+import { sourceTag } from "./render.js";
 import { stitch } from "./subgraph.js";
+import { chainOf, traceNodes } from "./trace.js";
 // ---------------------------------------------------------------------------
 // The same index, given to an agent
 //
-// Five tools over one engine, and only one of them ranks anything. `search_api`
-// is the way in when the question is vague; the other four are exact, because
+// Six tools over one engine, and only one of them ranks anything. `search_api`
+// is the way in when the question is vague; the other five are exact, because
 // a model that has been told "no results" by a vector search has learned
 // nothing — a ranking returns the top of a list, so an empty answer and an
 // absent thing look identical.
@@ -19,6 +21,12 @@ import { stitch } from "./subgraph.js";
 // model does not need to be reminded what a password is — it needs the list of
 // types that have one. `grep_api` is the same instinct widened: every literal
 // occurrence, counted in full, so "it is not there" can actually be concluded.
+//
+// `trace_api` is the other direction entirely. Having found a field, the next
+// question is always which call carries it. `search_api` stitches part of the
+// way there, but only between the nodes that ranked — and the operation and
+// the field usually share no word at all, which is why the edge between them
+// was built. `trace_api` follows that edge instead of ranking anything.
 // ---------------------------------------------------------------------------
 const GROUP = 'schema';
 /** Kept small on purpose: a tool result is prompt, and the model asked for one thing. */
@@ -28,9 +36,17 @@ const MAX_CANDIDATES = 25;
 /** A listing is lines rather than subgraphs, so it can afford more of them. */
 const DEFAULT_ROWS = 50;
 const MAX_ROWS = 200;
+/** A trace is a paragraph per node, so fewer of them, and fewer routes each. */
+const DEFAULT_TRACES = 5;
+const MAX_TRACES = 25;
+const DEFAULT_ROUTES = 10;
 export function schemaTools(index, options = {}) {
     const fallback = options.format ?? 'text';
     const docs = options.docs ?? true;
+    // With one document there is nothing to disambiguate and naming it on every
+    // line is prompt spent saying the same word; with several it is the only
+    // way to tell two revisions of one API apart.
+    const source = options.source ?? index.manifest.sources.length > 1;
     const searchApi = tool({
         name: 'search_api',
         group: GROUP,
@@ -107,7 +123,10 @@ export function schemaTools(index, options = {}) {
                 found: result.subgraphs.length,
                 ids: result.subgraphs.flatMap((s) => s.hits),
                 truncated: result.subgraphs.some((s) => s.truncated),
-                api: await present(index, result.subgraphs, chosen(format, fallback), { docs }),
+                api: await present(index, result.subgraphs, chosen(format, fallback), {
+                    docs,
+                    source,
+                }),
             };
         },
     });
@@ -209,12 +228,24 @@ export function schemaTools(index, options = {}) {
                     description: 'Match the name. A plain word matches anywhere in it; use * and ? ' +
                         'to match the whole name, e.g. "*Password*".',
                 },
-                path: { type: 'string', description: 'Match the route, e.g. "/users*".' },
+                path: {
+                    type: 'string',
+                    description: 'Match the route, e.g. "/users*". Keeps operations and their ' +
+                        'parameters; a schema sits on no one route, so it is left out.',
+                },
+                regex: {
+                    type: 'boolean',
+                    description: 'Read `name` and `path` as regular expressions instead.',
+                },
                 method_type: {
                     type: 'string',
                     enum: ['read_only', 'read_write', 'any'],
                 },
                 direction: { type: 'string', enum: ['input', 'output', 'any'] },
+                source: {
+                    type: 'string',
+                    description: 'Only this document, when the index holds more than one.',
+                },
                 limit: {
                     type: 'integer',
                     description: `Rows to return. Default ${DEFAULT_ROWS}.`,
@@ -222,7 +253,7 @@ export function schemaTools(index, options = {}) {
             },
             additionalProperties: false,
         },
-        execute: async ({ kind, name, path, method_type, direction, limit }) => {
+        execute: async ({ kind, name, path, regex, method_type, direction, source: only, limit, }) => {
             const subject = SUBJECTS[kind ?? 'methods'];
             if (!subject) {
                 return { error: `cannot list "${kind}"`, hint: 'kind is methods, types or fields' };
@@ -231,8 +262,9 @@ export function schemaTools(index, options = {}) {
             try {
                 result = listNodes(index.graph, {
                     kind: subject,
-                    name: name ? [loose(name)] : undefined,
-                    path: path ? [loose(path)] : undefined,
+                    name: name ? [loose(name, { regex })] : undefined,
+                    path: path ? [loose(path, { regex })] : undefined,
+                    source: only,
                     methodType: enumerated(method_type),
                     direction: enumerated(direction),
                     limit: Math.min(limit ?? DEFAULT_ROWS, MAX_ROWS),
@@ -244,7 +276,7 @@ export function schemaTools(index, options = {}) {
             return {
                 found: result.found,
                 truncated: result.truncated,
-                [PLURALS[subject]]: result.rows.map((r) => line(index.graph, subject, r)),
+                [PLURALS[subject]]: result.rows.map((r) => line(index.graph, subject, r, source)),
             };
         },
     });
@@ -255,7 +287,8 @@ export function schemaTools(index, options = {}) {
             'operations, schemas and fields alike. No embeddings and no ranking, so ' +
             'nothing is missed for being an unusual word or an odd spelling. This is the ' +
             'tool for "does X exist anywhere", and for checking that a search which ' +
-            'returned nothing really means there is nothing.',
+            'returned nothing really means there is nothing. Narrow it with `path` or ' +
+            '`name` when the word is common and only one corner of the API is meant.',
         parameters: {
             type: 'object',
             properties: {
@@ -268,6 +301,18 @@ export function schemaTools(index, options = {}) {
                     description: 'Read the pattern as a regular expression instead.',
                 },
                 kind: { type: 'string', enum: ['method', 'type', 'property'] },
+                name: {
+                    type: 'string',
+                    description: 'Only nodes whose own name matches this. * and ? allowed.',
+                },
+                path: {
+                    type: 'string',
+                    description: 'Only what sits on a matching route, e.g. "/users*".',
+                },
+                source: {
+                    type: 'string',
+                    description: 'Only this document, when the index holds more than one.',
+                },
                 limit: {
                     type: 'integer',
                     description: `Matches to return. Default ${DEFAULT_ROWS}. \`found\` always counts them all.`,
@@ -276,11 +321,14 @@ export function schemaTools(index, options = {}) {
             required: ['pattern'],
             additionalProperties: false,
         },
-        execute: async ({ pattern, regex, kind, limit }) => {
+        execute: async ({ pattern, regex, kind, name, path, source: only, limit }) => {
             let result;
             try {
                 result = grepNodes(index.graph, matcher(pattern, { regex }), {
                     kinds: kind ? [kind] : undefined,
+                    name: name ? [loose(name)] : undefined,
+                    path: path ? [loose(path)] : undefined,
+                    source: only,
                     limit: Math.min(limit ?? DEFAULT_ROWS, MAX_ROWS),
                 });
             }
@@ -296,11 +344,95 @@ export function schemaTools(index, options = {}) {
             return {
                 found: result.found,
                 truncated: result.truncated,
-                matches: result.matches.map((m) => ({ id: m.id, text: m.text })),
+                matches: result.matches.map((m) => ({
+                    id: m.id,
+                    ...(source ? { source: m.attributes.source } : {}),
+                    text: m.text,
+                })),
+            };
+        },
+    });
+    const traceApi = tool({
+        name: 'trace_api',
+        group: GROUP,
+        description: 'Answers "which calls can reach this?" for a field or a schema: walks up the ' +
+            'graph from everything of that name to the operations that accept or return it, ' +
+            'and gives the chain in between. Use it whenever a field has been found and the ' +
+            'endpoint to call is what is actually wanted — searching for the operation will ' +
+            'not work, because a call almost never repeats the name of a field nested inside ' +
+            'its body. An empty answer means nothing in the API carries it.',
+        parameters: {
+            type: 'object',
+            properties: {
+                of: {
+                    type: 'string',
+                    description: 'The field or schema name to trace up from, or a node id such as ' +
+                        '"Type:User". A plain word matches anywhere in the name; * and ? ' +
+                        'match the whole of it.',
+                },
+                kind: {
+                    type: 'string',
+                    enum: ['type', 'property'],
+                    description: 'Only trace from schemas, or only from fields.',
+                },
+                direction: {
+                    type: 'string',
+                    enum: ['input', 'output', 'any'],
+                    description: 'Keep only the calls that accept it, or that return it.',
+                },
+                source: {
+                    type: 'string',
+                    description: 'Only this document, when the index holds more than one.',
+                },
+                limit: {
+                    type: 'integer',
+                    description: `Nodes to trace from. Default ${DEFAULT_TRACES}.`,
+                },
+                routes: {
+                    type: 'integer',
+                    description: `Operations per node. Default ${DEFAULT_ROUTES}.`,
+                },
+            },
+            required: ['of'],
+            additionalProperties: false,
+        },
+        execute: async ({ of: wanted, kind, direction, source: only, limit, routes }) => {
+            let result;
+            try {
+                result = traceNodes(index.graph, {
+                    ids: index.graph.hasNode(wanted) ? [wanted] : undefined,
+                    kinds: kind ? [kind] : undefined,
+                    name: index.graph.hasNode(wanted) ? undefined : [loose(wanted)],
+                    source: only,
+                    limit: Math.min(limit ?? DEFAULT_TRACES, MAX_TRACES),
+                    maxRoutes: Math.min(routes ?? DEFAULT_ROUTES, MAX_ROWS),
+                });
+            }
+            catch (err) {
+                return { error: err instanceof PatternError ? err.message : String(err) };
+            }
+            if (result.found === 0) {
+                return {
+                    found: 0,
+                    hint: `nothing in the API is called "${wanted}" — try grep_api for it`,
+                };
+            }
+            const side = enumerated(direction);
+            return {
+                found: result.found,
+                truncated: result.truncated,
+                traced: result.traces.map((t) => ({
+                    id: t.id,
+                    reached_by: t.found,
+                    operations: t.routes
+                        .filter((r) => !side || r.direction === side || r.direction === 'both')
+                        .map((r) => `${r.attributes.httpMethod} ${r.attributes.path}  ` +
+                        `${r.attributes.name}  (${r.direction})${source ? `  ${sourceTag(r.attributes.source)}` : ''}  ${chainOf(index.graph, t.id, r)}`),
+                })),
             };
         },
     });
-    return [searchApi, describeTypes, findTypesWithProperty, listApi, grepApi];
+    return [searchApi, describeTypes, findTypesWithProperty, listApi, grepApi, traceApi];
 }
 // ---------------------------------------------------------------------------
 const SUBJECTS = {
@@ -315,16 +447,17 @@ const PLURALS = {
     property: 'properties',
 };
 /** One row, as the line a model reads rather than an object it has to walk. */
-function line(graph, kind, row) {
+function line(graph, kind, row, source = false) {
+    const from = source ? `  ${sourceTag(row.source)}` : '';
     if (kind === 'method') {
-        return `${row.httpMethod} ${row.path}  ${row.name}${row.doc ? ` — ${row.doc}` : ''}`;
+        return `${row.httpMethod} ${row.path}  ${row.name}${from}${row.doc ? ` — ${row.doc}` : ''}`;
     }
     if (kind === 'type') {
         const side = row.direction === 'none' ? '' : ` (${row.direction})`;
-        return `${row.name}${side}  ${fields(propertyCount(graph, row.id))}${row.doc ? ` — ${row.doc}` : ''}`;
+        return `${row.name}${side}  ${fields(propertyCount(graph, row.id))}${from}${row.doc ? ` — ${row.doc}` : ''}`;
     }
     const owner = row.parent ? `${row.parent}.` : '';
-    return `${owner}${row.name}${row.required ? '' : '?'}: ${row.signature || 'unknown'}`;
+    return `${owner}${row.name}${row.required ? '' : '?'}: ${row.signature || 'unknown'}${from}`;
 }
 const enumerated = (value) => value && value !== 'any' ? value : undefined;
 function list(description) {

package/dist/schema/trace.d.ts

@@ -0,0 +1,52 @@
+import type { Matcher } from '../common/match.ts';
+import type { ApiGraph, NodeAttrs, NodeKind } from './graph.ts';
+export type Side = 'input' | 'output' | 'both';
+export interface Route {
+    /** the operation node */
+    id: string;
+    attributes: NodeAttrs;
+    /** whether the call accepts what was traced, returns it, or both */
+    direction: Side;
+    /** the nodes in between, nearest the operation first */
+    via: string[];
+    hops: number;
+}
+export interface Trace {
+    id: string;
+    attributes: NodeAttrs;
+    /** how many operations reach it, whatever was kept */
+    found: number;
+    routes: Route[];
+    truncated: boolean;
+}
+export interface Traced {
+    /** how many nodes were traced, whatever was kept */
+    found: number;
+    traces: Trace[];
+    truncated: boolean;
+}
+export interface TraceFilter {
+    /** node ids to start from, taken as given */
+    ids?: readonly string[];
+    /** which kinds a `name` may match; all three when unset */
+    kinds?: readonly NodeKind[];
+    name?: readonly Matcher[];
+    source?: string;
+    /** starting nodes kept */
+    limit?: number;
+    /** routes kept per starting node */
+    maxRoutes?: number;
+    maxHops?: number;
+}
+/** How far containment is read backwards before a route is too indirect to mean anything. */
+export declare const DEFAULT_TRACE_HOPS = 8;
+export declare function traceNodes(graph: ApiGraph, filter: TraceFilter): Traced;
+export declare function traceOne(graph: ApiGraph, id: string, filter?: TraceFilter): Trace;
+/**
+ * The chain from an operation down to the node that was traced, as one line.
+ * A field is written onto the type above it (`User.email`) and a parameter is
+ * marked (`?page_size`), so the shape of the call can be read off the route
+ * without going and looking any of it up.
+ */
+export declare function chainOf(graph: ApiGraph, start: string, route: Route): string;
+//# sourceMappingURL=trace.d.ts.map

package/dist/schema/trace.js

@@ -0,0 +1,144 @@
+import { listNodes } from "./lookup.js";
+/** How far containment is read backwards before a route is too indirect to mean anything. */
+export const DEFAULT_TRACE_HOPS = 8;
+/** A walk is bounded; a document can always be larger than the one this was written against. */
+const MAX_VISITS = 20_000;
+/**
+ * What a bare pattern matches. Operations are left out on purpose: they are
+ * where a trace ends, so starting one at an operationId that happens to share
+ * a word adds a row saying only that it found itself. Naming a `Method:` id
+ * outright still works.
+ */
+const KINDS = ['type', 'property'];
+/** The edges an operation holds its payload by. Reaching one of these is arriving. */
+const ENTRY = new Set(['TAKES_INPUT', 'RETURNS_OUTPUT', 'HAS_PARAM']);
+/** Containment, read backwards: a type to the fields that hold it, a field to its owner. */
+const UPWARD = new Set(['HAS_PROPERTY', 'OF_TYPE', 'COMPOSES', 'ITEM_OF']);
+export function traceNodes(graph, filter) {
+    const starts = select(graph, filter);
+    const kept = filter.limit && filter.limit < starts.length ? starts.slice(0, filter.limit) : starts;
+    return {
+        found: starts.length,
+        truncated: kept.length < starts.length,
+        traces: kept.map((id) => traceOne(graph, id, filter)),
+    };
+}
+export function traceOne(graph, id, filter = {}) {
+    const attributes = graph.getNodeAttributes(id);
+    // An operation is already where a trace ends; it reaches itself and nothing above it.
+    const routes = attributes.kind === 'method'
+        ? [{ id, attributes, direction: 'both', via: [], hops: 0 }]
+        : climb(graph, id, filter.maxHops ?? DEFAULT_TRACE_HOPS);
+    const limit = filter.maxRoutes;
+    return {
+        id,
+        attributes,
+        found: routes.length,
+        routes: limit && limit < routes.length ? routes.slice(0, limit) : routes,
+        truncated: Boolean(limit && limit < routes.length),
+    };
+}
+/**
+ * The chain from an operation down to the node that was traced, as one line.
+ * A field is written onto the type above it (`User.email`) and a parameter is
+ * marked (`?page_size`), so the shape of the call can be read off the route
+ * without going and looking any of it up.
+ */
+export function chainOf(graph, start, route) {
+    let out = '';
+    for (const id of [...route.via, start]) {
+        const label = labelOf(graph, id);
+        out += out === '' || label.startsWith('.') ? label : ` → ${label}`;
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+function labelOf(graph, id) {
+    const a = graph.getNodeAttributes(id);
+    if (a.kind !== 'property') {
+        return a.name;
+    }
+    // A parameter belongs to a call rather than to a type, so it cannot be
+    // written as a field of the thing before it.
+    return id.includes('#') ? `?${a.name}` : `.${a.name}`;
+}
+function select(graph, filter) {
+    const out = new Set((filter.ids ?? []).filter((id) => graph.hasNode(id)));
+    if (filter.name) {
+        for (const kind of filter.kinds ?? KINDS) {
+            for (const row of listNodes(graph, { kind, name: filter.name, source: filter.source })
+                .rows) {
+                out.add(row.id);
+            }
+        }
+    }
+    return [...out].sort((a, b) => a.localeCompare(b));
+}
+/**
+ * Breadth-first up containment, recording an operation the moment one is
+ * reached. Breadth-first is what makes the answer the *shortest* way in: a
+ * type held by a wrapper held by a request body should be reported through
+ * the body, not through whichever branch happened to be walked first.
+ */
+function climb(graph, start, maxHops) {
+    const previous = new Map();
+    const seen = new Set([start]);
+    const found = new Map();
+    let frontier = [start];
+    for (let hop = 0; hop <= maxHops && frontier.length > 0 && seen.size < MAX_VISITS; hop++) {
+        const next = [];
+        for (const node of frontier) {
+            for (const edge of graph.inEdges(node)) {
+                const relation = graph.getEdgeAttribute(edge, 'relation');
+                const from = graph.source(edge);
+                if (ENTRY.has(relation)) {
+                    arrive(found, {
+                        id: from,
+                        attributes: graph.getNodeAttributes(from),
+                        direction: sideOf(relation),
+                        via: upward(previous, start, node),
+                        hops: hop + 1,
+                    });
+                }
+                else if (UPWARD.has(relation) && !seen.has(from)) {
+                    seen.add(from);
+                    previous.set(from, node);
+                    next.push(from);
+                }
+            }
+        }
+        frontier = next;
+    }
+    return [...found.values()].sort((a, b) => a.hops - b.hops ||
+        a.attributes.path.localeCompare(b.attributes.path) ||
+        a.attributes.httpMethod.localeCompare(b.attributes.httpMethod));
+}
+/**
+ * One operation, once. A call that both accepts and returns the same type is
+ * two edges and one answer, and the honest word for that answer is `both`.
+ */
+function arrive(found, route) {
+    const existing = found.get(route.id);
+    if (!existing) {
+        found.set(route.id, route);
+    }
+    else if (existing.direction !== route.direction) {
+        existing.direction = 'both';
+    }
+}
+const sideOf = (relation) => (relation === 'RETURNS_OUTPUT' ? 'output' : 'input');
+/** The nodes walked through, from the one holding the entry edge back down towards the start. */
+function upward(previous, start, from) {
+    const out = [];
+    let at = from;
+    while (at !== start) {
+        out.push(at);
+        const below = previous.get(at);
+        if (below === undefined) {
+            break;
+        }
+        at = below;
+    }
+    return out;
+}
+//# sourceMappingURL=trace.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.5",
+    "version": "1.1.8",
     "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
     "keywords": [
         "agents",
@@ -53,7 +53,7 @@
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
         "graphology": "^0.26.0",
-        "@zenera/cli": "^1.1.5",
-        "@zenera/neo": "^1.1.5"
+        "@zenera/cli": "^1.1.8",
+        "@zenera/neo": "^1.1.8"
     }
 }

package/dist/schema/progress.d.ts

@@ -1,26 +0,0 @@
-import type { Counts, Manifest } from './files.ts';
-export declare const LOCK_FILE = ".lock";
-export declare const README_FILE = "README.md";
-export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
-export interface BuildPlan {
-    /** where the index goes */
-    dir: string;
-    files: readonly string[];
-    /** the embedding reference as it was typed */
-    embedding: string;
-    indexer: string;
-}
-export interface Journal {
-    phase(name: Phase): void;
-    /** what the documents turned out to hold, once they have been read */
-    read(counts: Counts): void;
-    progress(done: number, total: number): void;
-    finish(manifest: Manifest): void;
-    fail(reason: unknown): void;
-}
-/**
- * Takes the directory, or refuses it. Two builds writing one index would
- * interleave their LanceDB writes and leave a store neither of them describes.
- */
-export declare function beginBuild(plan: BuildPlan): Journal;
-//# sourceMappingURL=progress.d.ts.map