@zenera/rag 1.1.4 → 1.1.6

package/dist/schema/tools.js

@@ -2,25 +2,40 @@ import { tool } from '@zenera/neo';
 import { FORMATS, isFormat, present } from "../present.js";
 import { isEmpty, parseQuery, QueryError } from "../query.js";
 import { toTypeScript } from "./hydrate.js";
+import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
+import { loose, matcher, PatternError } from "./match.js";
+import { sourceTag } from "./render.js";
 import { stitch } from "./subgraph.js";
 // ---------------------------------------------------------------------------
 // The same index, given to an agent
 //
-// Four tools over one engine. Three of them search, and the fourth deliberately
-// does not: `find_types_with_property` is a graph lookup, for the moment after
-// the compiler says `'password' does not exist in type 'PublicUserProfile'`.
-// At that point the model does not need to be reminded what a password is —
-// it needs the list of types that have one, and an embedding of the word will
-// only rank the guess it already made near the top again.
+// Five tools over one engine, and only one of them ranks anything. `search_api`
+// is the way in when the question is vague; the other four are exact, because
+// a model that has been told "no results" by a vector search has learned
+// nothing — a ranking returns the top of a list, so an empty answer and an
+// absent thing look identical.
+//
+// `find_types_with_property` is for the moment after the compiler says
+// `'password' does not exist in type 'PublicUserProfile'`. At that point the
+// model does not need to be reminded what a password is — it needs the list of
+// types that have one. `grep_api` is the same instinct widened: every literal
+// occurrence, counted in full, so "it is not there" can actually be concluded.
 // ---------------------------------------------------------------------------
 const GROUP = 'schema';
 /** Kept small on purpose: a tool result is prompt, and the model asked for one thing. */
 const DEFAULT_LIMIT = 4;
 const DEFAULT_MAX_NODES = 60;
 const MAX_CANDIDATES = 25;
+/** A listing is lines rather than subgraphs, so it can afford more of them. */
+const DEFAULT_ROWS = 50;
+const MAX_ROWS = 200;
 export function schemaTools(index, options = {}) {
     const fallback = options.format ?? 'text';
     const docs = options.docs ?? true;
+    // With one document there is nothing to disambiguate and naming it on every
+    // line is prompt spent saying the same word; with several it is the only
+    // way to tell two revisions of one API apart.
+    const source = options.source ?? index.manifest.sources.length > 1;
     const searchApi = tool({
         name: 'search_api',
         group: GROUP,
@@ -97,7 +112,10 @@ export function schemaTools(index, options = {}) {
                 found: result.subgraphs.length,
                 ids: result.subgraphs.flatMap((s) => s.hits),
                 truncated: result.subgraphs.some((s) => s.truncated),
-                api: await present(index, result.subgraphs, chosen(format, fallback), { docs }),
+                api: await present(index, result.subgraphs, chosen(format, fallback), {
+                    docs,
+                    source,
+                }),
             };
         },
     });
@@ -178,43 +196,179 @@ export function schemaTools(index, options = {}) {
                 : { found: candidates.length, candidates: candidates.slice(0, MAX_CANDIDATES) };
         },
     });
-    const listMethods = tool({
-        name: 'list_methods',
+    const listApi = tool({
+        name: 'list_api',
         group: GROUP,
-        description: 'Lists operations by path, with no searching. Use it to see the shape of the API ' +
-            'before deciding what to ask for.',
+        description: 'Lists operations, schemas or fields by name, with no searching and no ranking. ' +
+            'The answer is complete: every match is counted, so `found` tells you how many ' +
+            'exist even when the list was shortened. Use it to see the shape of the API ' +
+            'before deciding what to ask for, and to settle whether something exists at all — ' +
+            'search can only ever return its best guesses, so it cannot answer that.',
         parameters: {
             type: 'object',
             properties: {
-                contains: { type: 'string', description: 'Only paths holding this text.' },
+                kind: {
+                    type: 'string',
+                    enum: ['methods', 'types', 'properties'],
+                    description: 'What to list. Default methods.',
+                },
+                name: {
+                    type: 'string',
+                    description: 'Match the name. A plain word matches anywhere in it; use * and ? ' +
+                        'to match the whole name, e.g. "*Password*".',
+                },
+                path: {
+                    type: 'string',
+                    description: 'Match the route, e.g. "/users*". Keeps operations and their ' +
+                        'parameters; a schema sits on no one route, so it is left out.',
+                },
+                regex: {
+                    type: 'boolean',
+                    description: 'Read `name` and `path` as regular expressions instead.',
+                },
                 method_type: {
                     type: 'string',
                     enum: ['read_only', 'read_write', 'any'],
                 },
+                direction: { type: 'string', enum: ['input', 'output', 'any'] },
+                source: {
+                    type: 'string',
+                    description: 'Only this document, when the index holds more than one.',
+                },
+                limit: {
+                    type: 'integer',
+                    description: `Rows to return. Default ${DEFAULT_ROWS}.`,
+                },
             },
             additionalProperties: false,
         },
-        execute: async ({ contains, method_type }) => {
-            const needle = contains?.toLowerCase();
-            const rows = [];
-            index.graph.forEachNode((_id, a) => {
-                if (a.kind !== 'method') {
-                    return;
-                }
-                if (needle && !a.path.toLowerCase().includes(needle)) {
-                    return;
-                }
-                if (method_type && method_type !== 'any' && a.methodType !== method_type) {
-                    return;
-                }
-                rows.push(`${a.httpMethod} ${a.path}  ${a.name}${a.doc ? ` — ${a.doc}` : ''}`);
-            });
-            return { found: rows.length, methods: rows.sort() };
+        execute: async ({ kind, name, path, regex, method_type, direction, source: only, limit, }) => {
+            const subject = SUBJECTS[kind ?? 'methods'];
+            if (!subject) {
+                return { error: `cannot list "${kind}"`, hint: 'kind is methods, types or fields' };
+            }
+            let result;
+            try {
+                result = listNodes(index.graph, {
+                    kind: subject,
+                    name: name ? [loose(name, { regex })] : undefined,
+                    path: path ? [loose(path, { regex })] : undefined,
+                    source: only,
+                    methodType: enumerated(method_type),
+                    direction: enumerated(direction),
+                    limit: Math.min(limit ?? DEFAULT_ROWS, MAX_ROWS),
+                });
+            }
+            catch (err) {
+                return { error: err instanceof PatternError ? err.message : String(err) };
+            }
+            return {
+                found: result.found,
+                truncated: result.truncated,
+                [PLURALS[subject]]: result.rows.map((r) => line(index.graph, subject, r, source)),
+            };
+        },
+    });
+    const grepApi = tool({
+        name: 'grep_api',
+        group: GROUP,
+        description: 'Finds every literal occurrence of a string across the whole API description — ' +
+            'operations, schemas and fields alike. No embeddings and no ranking, so ' +
+            'nothing is missed for being an unusual word or an odd spelling. This is the ' +
+            'tool for "does X exist anywhere", and for checking that a search which ' +
+            'returned nothing really means there is nothing. Narrow it with `path` or ' +
+            '`name` when the word is common and only one corner of the API is meant.',
+        parameters: {
+            type: 'object',
+            properties: {
+                pattern: {
+                    type: 'string',
+                    description: 'The text to find. Matched anywhere, ignoring case.',
+                },
+                regex: {
+                    type: 'boolean',
+                    description: 'Read the pattern as a regular expression instead.',
+                },
+                kind: { type: 'string', enum: ['method', 'type', 'property'] },
+                name: {
+                    type: 'string',
+                    description: 'Only nodes whose own name matches this. * and ? allowed.',
+                },
+                path: {
+                    type: 'string',
+                    description: 'Only what sits on a matching route, e.g. "/users*".',
+                },
+                source: {
+                    type: 'string',
+                    description: 'Only this document, when the index holds more than one.',
+                },
+                limit: {
+                    type: 'integer',
+                    description: `Matches to return. Default ${DEFAULT_ROWS}. \`found\` always counts them all.`,
+                },
+            },
+            required: ['pattern'],
+            additionalProperties: false,
+        },
+        execute: async ({ pattern, regex, kind, name, path, source: only, limit }) => {
+            let result;
+            try {
+                result = grepNodes(index.graph, matcher(pattern, { regex }), {
+                    kinds: kind ? [kind] : undefined,
+                    name: name ? [loose(name)] : undefined,
+                    path: path ? [loose(path)] : undefined,
+                    source: only,
+                    limit: Math.min(limit ?? DEFAULT_ROWS, MAX_ROWS),
+                });
+            }
+            catch (err) {
+                return { error: err instanceof PatternError ? err.message : String(err) };
+            }
+            if (result.found === 0) {
+                return {
+                    found: 0,
+                    hint: 'nothing in the description contains it — it is not there under this name',
+                };
+            }
+            return {
+                found: result.found,
+                truncated: result.truncated,
+                matches: result.matches.map((m) => ({
+                    id: m.id,
+                    ...(source ? { source: m.attributes.source } : {}),
+                    text: m.text,
+                })),
+            };
         },
     });
-    return [searchApi, describeTypes, findTypesWithProperty, listMethods];
+    return [searchApi, describeTypes, findTypesWithProperty, listApi, grepApi];
 }
 // ---------------------------------------------------------------------------
+const SUBJECTS = {
+    methods: 'method',
+    types: 'type',
+    properties: 'property',
+    fields: 'property',
+};
+const PLURALS = {
+    method: 'methods',
+    type: 'types',
+    property: 'properties',
+};
+/** One row, as the line a model reads rather than an object it has to walk. */
+function line(graph, kind, row, source = false) {
+    const from = source ? `  ${sourceTag(row.source)}` : '';
+    if (kind === 'method') {
+        return `${row.httpMethod} ${row.path}  ${row.name}${from}${row.doc ? ` — ${row.doc}` : ''}`;
+    }
+    if (kind === 'type') {
+        const side = row.direction === 'none' ? '' : ` (${row.direction})`;
+        return `${row.name}${side}  ${fields(propertyCount(graph, row.id))}${from}${row.doc ? ` — ${row.doc}` : ''}`;
+    }
+    const owner = row.parent ? `${row.parent}.` : '';
+    return `${owner}${row.name}${row.required ? '' : '?'}: ${row.signature || 'unknown'}${from}`;
+}
+const enumerated = (value) => value && value !== 'any' ? value : undefined;
 function list(description) {
     return { type: 'array', items: { type: 'string' }, description };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.4",
+    "version": "1.1.6",
     "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
     "keywords": [
         "agents",
@@ -53,7 +53,7 @@
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
         "graphology": "^0.26.0",
-        "@zenera/cli": "^1.1.4",
-        "@zenera/neo": "^1.1.4"
+        "@zenera/cli": "^1.1.6",
+        "@zenera/neo": "^1.1.6"
     }
 }