@zenera/rag 1.1.6 → 1.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +28 -2
  2. package/dist/command.js +41 -727
  3. package/dist/common/embedder.d.ts +3 -0
  4. package/dist/common/embedder.js +64 -0
  5. package/dist/common/locate.d.ts +18 -0
  6. package/dist/{schema → common}/locate.js +42 -28
  7. package/dist/common/manifest.d.ts +50 -0
  8. package/dist/common/manifest.js +62 -0
  9. package/dist/common/progress.d.ts +57 -0
  10. package/dist/common/progress.js +155 -0
  11. package/dist/common/prose.d.ts +13 -0
  12. package/dist/common/prose.js +56 -0
  13. package/dist/index.d.ts +6 -4
  14. package/dist/index.js +6 -4
  15. package/dist/schema/build.js +6 -2
  16. package/dist/schema/command.d.ts +3 -0
  17. package/dist/schema/command.js +819 -0
  18. package/dist/schema/files.d.ts +5 -27
  19. package/dist/schema/files.js +9 -29
  20. package/dist/schema/lookup.d.ts +1 -1
  21. package/dist/schema/lookup.js +1 -1
  22. package/dist/{present.d.ts → schema/present.d.ts} +5 -5
  23. package/dist/{present.js → schema/present.js} +2 -2
  24. package/dist/{query.d.ts → schema/query.d.ts} +1 -1
  25. package/dist/schema/readme.d.ts +6 -0
  26. package/dist/schema/readme.js +122 -0
  27. package/dist/{repl.d.ts → schema/repl.d.ts} +1 -1
  28. package/dist/schema/search.js +2 -1
  29. package/dist/schema/tools.d.ts +1 -1
  30. package/dist/schema/tools.js +97 -6
  31. package/dist/schema/trace.d.ts +52 -0
  32. package/dist/schema/trace.js +144 -0
  33. package/package.json +3 -3
  34. package/dist/schema/locate.d.ts +0 -20
  35. package/dist/schema/progress.d.ts +0 -26
  36. package/dist/schema/progress.js +0 -316
  37. /package/dist/{schema → common}/match.d.ts +0 -0
  38. /package/dist/{schema → common}/match.js +0 -0
  39. /package/dist/{query.js → schema/query.js} +0 -0
  40. /package/dist/{repl.js → schema/repl.js} +0 -0
@@ -1,8 +1,10 @@
1
+ import { type IndexHead, type IndexSpec } from '../common/manifest.ts';
1
2
  import type { ApiGraph } from './graph.ts';
2
3
  import type { Schema } from './schema.ts';
3
4
  import type { Operation } from './spec.ts';
4
5
  export declare const INDEX_VERSION = 3;
5
- export declare const MANIFEST_FILE = "manifest.json";
6
+ /** How a schema index is found, read, and refused. */
7
+ export declare const SCHEMA_INDEX: IndexSpec;
6
8
  export declare const GRAPH_FILE = "graph.json";
7
9
  export declare const SCHEMAS_FILE = "schemas.json";
8
10
  export declare const OPERATIONS_FILE = "operations.json";
@@ -30,23 +32,9 @@ export interface Counts {
30
32
  properties: number;
31
33
  entities: number;
32
34
  }
33
- export interface Manifest {
34
- version: number;
35
- createdAt: string;
36
- indexer: string;
37
- /** `ref` as it was typed, `id` as the embedder answers to it */
38
- embedding: {
39
- ref: string;
40
- id: string;
41
- dimensions: number;
42
- };
35
+ export interface Manifest extends IndexHead {
43
36
  sources: SourceRecord[];
44
37
  counts: Counts;
45
- /** whether the table carries an fts index, and whether it carries a vector one */
46
- indexes: {
47
- fts: boolean;
48
- vector: boolean;
49
- };
50
38
  }
51
39
  export interface WrittenIndex {
52
40
  manifest: Manifest;
@@ -75,15 +63,5 @@ export declare function openIndex(dir: string): Promise<OpenIndex>;
75
63
  * of what the caller can do next.
76
64
  */
77
65
  export declare function readSource(dir: string, name: string): Promise<string | undefined>;
78
- export declare function readManifest(dir: string): Promise<Manifest>;
79
- /**
80
- * A store answers with the neighbours of a vector, and a vector means nothing
81
- * without the model that produced it. Asking one model's index a question
82
- * embedded by another returns rows, in an order that is noise.
83
- *
84
- * Either spelling is accepted, because `openai:text-embedding-3-small` and
85
- * `text-embedding-3-small` are one model and which of them was typed is not
86
- * something anyone should have to remember.
87
- */
88
- export declare function assertSameEmbedding(manifest: Manifest, ref: string): void;
66
+ export declare const readManifest: (dir: string) => Promise<Manifest>;
89
67
  //# sourceMappingURL=files.d.ts.map
@@ -2,6 +2,7 @@ import { CliError, EXIT } from '@zenera/cli/lib';
2
2
  import { MultiDirectedGraph } from 'graphology';
3
3
  import { mkdir, readFile, writeFile } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
+ import { MANIFEST_FILE, readHead } from "../common/manifest.js";
5
6
  // ---------------------------------------------------------------------------
6
7
  // What an index is, on disk
7
8
  //
@@ -20,7 +21,13 @@ import { join } from 'node:path';
20
21
  // model — without going looking for the files it was made from.
21
22
  // ---------------------------------------------------------------------------
22
23
  export const INDEX_VERSION = 3;
23
- export const MANIFEST_FILE = 'manifest.json';
24
+ /** How a schema index is found, read, and refused. */
25
+ export const SCHEMA_INDEX = {
26
+ kind: 'schema',
27
+ version: INDEX_VERSION,
28
+ defaultDir: './schema-db',
29
+ envName: 'ZEN_SCHEMA_DB',
30
+ };
24
31
  export const GRAPH_FILE = 'graph.json';
25
32
  export const SCHEMAS_FILE = 'schemas.json';
26
33
  export const OPERATIONS_FILE = 'operations.json';
@@ -77,32 +84,5 @@ export async function readSource(dir, name) {
77
84
  }
78
85
  return await readFile(join(dir, record.path), 'utf8');
79
86
  }
80
- export async function readManifest(dir) {
81
- let text;
82
- try {
83
- text = await readFile(join(dir, MANIFEST_FILE), 'utf8');
84
- }
85
- catch {
86
- throw new CliError(`${dir} does not hold an index`, EXIT.invalid, 'build one with `zen rag schema index`, or name an existing one with --dir or $ZEN_SCHEMA_DB');
87
- }
88
- const manifest = JSON.parse(text);
89
- if (manifest.version !== INDEX_VERSION) {
90
- throw new CliError(`${dir} is a version ${manifest.version} index, and this indexer reads version ${INDEX_VERSION}`, EXIT.invalid, 'rebuild it with `zen rag schema index`');
91
- }
92
- return manifest;
93
- }
94
- /**
95
- * A store answers with the neighbours of a vector, and a vector means nothing
96
- * without the model that produced it. Asking one model's index a question
97
- * embedded by another returns rows, in an order that is noise.
98
- *
99
- * Either spelling is accepted, because `openai:text-embedding-3-small` and
100
- * `text-embedding-3-small` are one model and which of them was typed is not
101
- * something anyone should have to remember.
102
- */
103
- export function assertSameEmbedding(manifest, ref) {
104
- if (ref !== manifest.embedding.ref && ref !== manifest.embedding.id) {
105
- throw new CliError(`this index was built with ${manifest.embedding.ref}, not ${ref}`, EXIT.invalid, `search it with --embedding ${manifest.embedding.ref}, or rebuild it`);
106
- }
107
- }
87
+ export const readManifest = (dir) => readHead(dir, SCHEMA_INDEX);
108
88
  //# sourceMappingURL=files.js.map
@@ -1,5 +1,5 @@
1
+ import { type Matcher } from '../common/match.ts';
1
2
  import { type ApiGraph, type NodeAttrs, type NodeKind } from './graph.ts';
2
- import { type Matcher } from './match.ts';
3
3
  export interface Row extends NodeAttrs {
4
4
  id: string;
5
5
  }
@@ -1,6 +1,6 @@
1
+ import { PatternError } from "../common/match.js";
1
2
  import { textOf } from "./entities.js";
2
3
  import { methodId } from "./graph.js";
3
- import { PatternError } from "./match.js";
4
4
  /** A scan is bounded, because a pattern may have come from a model. */
5
5
  const DEADLINE_MS = 2000;
6
6
  const DEADLINE_EVERY = 500;
@@ -1,8 +1,8 @@
1
- import { type HydrateOptions } from './schema/hydrate.ts';
2
- import { type RenderOptions } from './schema/render.ts';
3
- import type { Schema } from './schema/schema.ts';
4
- import type { Operation } from './schema/spec.ts';
5
- import type { Subgraph } from './schema/subgraph.ts';
1
+ import { type HydrateOptions } from './hydrate.ts';
2
+ import { type RenderOptions } from './render.ts';
3
+ import type { Schema } from './schema.ts';
4
+ import type { Operation } from './spec.ts';
5
+ import type { Subgraph } from './subgraph.ts';
6
6
  export declare const FORMATS: readonly ["text", "mermaid", "mermaid-flowchart", "ts", "openapi"];
7
7
  export type Format = (typeof FORMATS)[number];
8
8
  export declare function isFormat(value: string): value is Format;
@@ -1,5 +1,5 @@
1
- import { toOpenApi, toTypeScript } from "./schema/hydrate.js";
2
- import { render } from "./schema/render.js";
1
+ import { toOpenApi, toTypeScript } from "./hydrate.js";
2
+ import { render } from "./render.js";
3
3
  // ---------------------------------------------------------------------------
4
4
  // One subgraph, in whichever of the four shapes was asked for
5
5
  //
@@ -1,4 +1,4 @@
1
- import type { SchemaQuery } from './schema/search.ts';
1
+ import type { SchemaQuery } from './search.ts';
2
2
  export declare class QueryError extends Error {
3
3
  }
4
4
  export declare function parseQuery(value: unknown): SchemaQuery;
@@ -0,0 +1,6 @@
1
+ import type { Report } from '../common/progress.ts';
2
+ import type { Counts, Manifest } from './files.ts';
3
+ export type Phase = 'reading' | 'graph' | 'embedding' | 'writing';
4
+ export declare const PHASES: Record<Phase, string>;
5
+ export declare const SCHEMA_REPORT: Report<Counts, Manifest>;
6
+ //# sourceMappingURL=readme.d.ts.map
@@ -0,0 +1,122 @@
1
+ import { basename } from 'node:path';
2
+ import { INTERVAL_MS } from "../common/progress.js";
3
+ import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
4
+ export const PHASES = {
5
+ reading: 'reading the documents',
6
+ graph: 'building the graph',
7
+ embedding: 'embedding',
8
+ writing: 'writing the store',
9
+ };
10
+ export const SCHEMA_REPORT = { building, complete, failed };
11
+ function building(state) {
12
+ const rows = [
13
+ ['documents', state.documents.join(', ')],
14
+ ['embedding', state.embedding],
15
+ [
16
+ 'started',
17
+ `${new Date(state.started).toISOString()} (${duration(state.now - state.started)} ago)`,
18
+ ],
19
+ ['step', state.step],
20
+ ];
21
+ if (state.summary) {
22
+ rows.push(['found', entities(state.summary)]);
23
+ }
24
+ if (state.total > 0) {
25
+ const percent = Math.round((state.done / state.total) * 100);
26
+ rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
27
+ }
28
+ rows.push(['updated', new Date(state.now).toISOString()]);
29
+ return [
30
+ '# Schema index — being built',
31
+ '',
32
+ 'A searchable index of the API documents named below, written by `zen rag schema index`.',
33
+ '**It is incomplete. Nothing should read it yet.**',
34
+ '',
35
+ ...fields(rows),
36
+ '',
37
+ `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
38
+ 'the whole file is replaced by a description of the index when it finishes. If it still says',
39
+ '"being built" and `.lock` names no living process, the build died part way.',
40
+ '',
41
+ ].join('\n');
42
+ }
43
+ function complete(state) {
44
+ const { manifest } = state;
45
+ const titles = manifest.sources.map((s) => s.title).filter(Boolean);
46
+ const what = titles.length > 0 ? titles.join(', ') : basename(state.dir);
47
+ return [
48
+ `# Schema index — ${what}`,
49
+ '',
50
+ `A searchable index of ${plural(manifest.sources.length, 'API document')}, built with`,
51
+ `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(state.ms)}.`,
52
+ 'Ask it for the operations and types behind a question and it answers with a subgraph:',
53
+ 'the endpoints that match, the schemas they carry, and the fields inside those — printed',
54
+ 'as text, Mermaid, TypeScript or OpenAPI.',
55
+ '',
56
+ '## What it covers',
57
+ '',
58
+ ...sourceTable(manifest.sources),
59
+ '',
60
+ `${entities(manifest.counts)},`,
61
+ `${searched(manifest.indexes)}.`,
62
+ '',
63
+ '## Files',
64
+ '',
65
+ ...fields([
66
+ ['manifest.json', 'what this index is and what built it — read this first'],
67
+ ['graph.json', 'the nodes and edges: operations, types, fields'],
68
+ ['schemas.json', 'the JSON Schema of every type'],
69
+ ['operations.json', 'every operation, with its parameters and responses'],
70
+ ...(manifest.sources.some((s) => s.path)
71
+ ? [['sources/', 'the documents themselves, bundled, exactly as indexed']]
72
+ : []),
73
+ ['lance/', 'the LanceDB table: the search text, the vectors, the filter columns'],
74
+ ]),
75
+ '',
76
+ '## Asking it something',
77
+ '',
78
+ 'From this directory:',
79
+ '',
80
+ '```',
81
+ 'zen rag schema search --dir . --all "how do I cancel a subscription"',
82
+ '```',
83
+ '',
84
+ `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
85
+ '',
86
+ ].join('\n');
87
+ }
88
+ function failed(state) {
89
+ return [
90
+ '# Schema index — failed',
91
+ '',
92
+ 'This index was not finished and what is here is incomplete. Nothing should read it;',
93
+ 'build it again with `zen rag schema index`.',
94
+ '',
95
+ ...fields([
96
+ ['documents', state.documents.join(', ')],
97
+ ['step', state.step],
98
+ ['reason', message(state.reason)],
99
+ ['started', new Date(state.started).toISOString()],
100
+ [
101
+ 'failed',
102
+ `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
103
+ ],
104
+ ]),
105
+ '',
106
+ ].join('\n');
107
+ }
108
+ // ---------------------------------------------------------------------------
109
+ const HEADERS = ['document', 'dialect', 'paths', 'operations', 'schemas', 'fields'];
110
+ const sourceTable = (sources) => grid(HEADERS, sources.map((s) => [
111
+ s.file,
112
+ s.dialect,
113
+ String(s.paths),
114
+ String(s.methods),
115
+ String(s.types),
116
+ String(s.properties),
117
+ ]));
118
+ function entities(counts) {
119
+ return (`${plural(counts.entities, 'entity', 'entities')}: ` +
120
+ `${counts.methods} operations, ${counts.types} schemas, ${counts.properties} fields`);
121
+ }
122
+ //# sourceMappingURL=readme.js.map
@@ -1,5 +1,5 @@
1
1
  import { type Format, type OutputOptions } from './present.ts';
2
- import type { SchemaIndex, SchemaQuery } from './schema/search.ts';
2
+ import type { SchemaIndex, SchemaQuery } from './search.ts';
3
3
  interface Settings extends OutputOptions {
4
4
  format: Format;
5
5
  }
@@ -1,4 +1,5 @@
1
- import { assertSameEmbedding, openIndex } from "./files.js";
1
+ import { assertSameEmbedding } from "../common/manifest.js";
2
+ import { openIndex } from "./files.js";
2
3
  import { EntityStore } from "./store.js";
3
4
  import { DEFAULT_MAX_HOPS, DEFAULT_MAX_NODES, stitch, } from "./subgraph.js";
4
5
  export const DEFAULT_LIMIT = 5;
@@ -1,5 +1,5 @@
1
1
  import { type AnyTool } from '@zenera/neo';
2
- import { type Format } from '../present.ts';
2
+ import { type Format } from './present.ts';
3
3
  import type { SchemaIndex } from './search.ts';
4
4
  export interface SchemaToolOptions {
5
5
  /** what `format` defaults to when the model does not say */
@@ -1,16 +1,17 @@
1
1
  import { tool } from '@zenera/neo';
2
- import { FORMATS, isFormat, present } from "../present.js";
3
- import { isEmpty, parseQuery, QueryError } from "../query.js";
2
+ import { loose, matcher, PatternError } from "../common/match.js";
3
+ import { FORMATS, isFormat, present } from "./present.js";
4
+ import { isEmpty, parseQuery, QueryError } from "./query.js";
4
5
  import { toTypeScript } from "./hydrate.js";
5
6
  import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
6
- import { loose, matcher, PatternError } from "./match.js";
7
7
  import { sourceTag } from "./render.js";
8
8
  import { stitch } from "./subgraph.js";
9
+ import { chainOf, traceNodes } from "./trace.js";
9
10
  // ---------------------------------------------------------------------------
10
11
  // The same index, given to an agent
11
12
  //
12
- // Five tools over one engine, and only one of them ranks anything. `search_api`
13
- // is the way in when the question is vague; the other four are exact, because
13
+ // Six tools over one engine, and only one of them ranks anything. `search_api`
14
+ // is the way in when the question is vague; the other five are exact, because
14
15
  // a model that has been told "no results" by a vector search has learned
15
16
  // nothing — a ranking returns the top of a list, so an empty answer and an
16
17
  // absent thing look identical.
@@ -20,6 +21,12 @@ import { stitch } from "./subgraph.js";
20
21
  // model does not need to be reminded what a password is — it needs the list of
21
22
  // types that have one. `grep_api` is the same instinct widened: every literal
22
23
  // occurrence, counted in full, so "it is not there" can actually be concluded.
24
+ //
25
+ // `trace_api` is the other direction entirely. Having found a field, the next
26
+ // question is always which call carries it. `search_api` stitches part of the
27
+ // way there, but only between the nodes that ranked — and the operation and
28
+ // the field usually share no word at all, which is why the edge between them
29
+ // was built. `trace_api` follows that edge instead of ranking anything.
23
30
  // ---------------------------------------------------------------------------
24
31
  const GROUP = 'schema';
25
32
  /** Kept small on purpose: a tool result is prompt, and the model asked for one thing. */
@@ -29,6 +36,10 @@ const MAX_CANDIDATES = 25;
29
36
  /** A listing is lines rather than subgraphs, so it can afford more of them. */
30
37
  const DEFAULT_ROWS = 50;
31
38
  const MAX_ROWS = 200;
39
+ /** A trace is a paragraph per node, so fewer of them, and fewer routes each. */
40
+ const DEFAULT_TRACES = 5;
41
+ const MAX_TRACES = 25;
42
+ const DEFAULT_ROUTES = 10;
32
43
  export function schemaTools(index, options = {}) {
33
44
  const fallback = options.format ?? 'text';
34
45
  const docs = options.docs ?? true;
@@ -341,7 +352,87 @@ export function schemaTools(index, options = {}) {
341
352
  };
342
353
  },
343
354
  });
344
- return [searchApi, describeTypes, findTypesWithProperty, listApi, grepApi];
355
+ const traceApi = tool({
356
+ name: 'trace_api',
357
+ group: GROUP,
358
+ description: 'Answers "which calls can reach this?" for a field or a schema: walks up the ' +
359
+ 'graph from everything of that name to the operations that accept or return it, ' +
360
+ 'and gives the chain in between. Use it whenever a field has been found and the ' +
361
+ 'endpoint to call is what is actually wanted — searching for the operation will ' +
362
+ 'not work, because a call almost never repeats the name of a field nested inside ' +
363
+ 'its body. An empty answer means nothing in the API carries it.',
364
+ parameters: {
365
+ type: 'object',
366
+ properties: {
367
+ of: {
368
+ type: 'string',
369
+ description: 'The field or schema name to trace up from, or a node id such as ' +
370
+ '"Type:User". A plain word matches anywhere in the name; * and ? ' +
371
+ 'match the whole of it.',
372
+ },
373
+ kind: {
374
+ type: 'string',
375
+ enum: ['type', 'property'],
376
+ description: 'Only trace from schemas, or only from fields.',
377
+ },
378
+ direction: {
379
+ type: 'string',
380
+ enum: ['input', 'output', 'any'],
381
+ description: 'Keep only the calls that accept it, or that return it.',
382
+ },
383
+ source: {
384
+ type: 'string',
385
+ description: 'Only this document, when the index holds more than one.',
386
+ },
387
+ limit: {
388
+ type: 'integer',
389
+ description: `Nodes to trace from. Default ${DEFAULT_TRACES}.`,
390
+ },
391
+ routes: {
392
+ type: 'integer',
393
+ description: `Operations per node. Default ${DEFAULT_ROUTES}.`,
394
+ },
395
+ },
396
+ required: ['of'],
397
+ additionalProperties: false,
398
+ },
399
+ execute: async ({ of: wanted, kind, direction, source: only, limit, routes }) => {
400
+ let result;
401
+ try {
402
+ result = traceNodes(index.graph, {
403
+ ids: index.graph.hasNode(wanted) ? [wanted] : undefined,
404
+ kinds: kind ? [kind] : undefined,
405
+ name: index.graph.hasNode(wanted) ? undefined : [loose(wanted)],
406
+ source: only,
407
+ limit: Math.min(limit ?? DEFAULT_TRACES, MAX_TRACES),
408
+ maxRoutes: Math.min(routes ?? DEFAULT_ROUTES, MAX_ROWS),
409
+ });
410
+ }
411
+ catch (err) {
412
+ return { error: err instanceof PatternError ? err.message : String(err) };
413
+ }
414
+ if (result.found === 0) {
415
+ return {
416
+ found: 0,
417
+ hint: `nothing in the API is called "${wanted}" — try grep_api for it`,
418
+ };
419
+ }
420
+ const side = enumerated(direction);
421
+ return {
422
+ found: result.found,
423
+ truncated: result.truncated,
424
+ traced: result.traces.map((t) => ({
425
+ id: t.id,
426
+ reached_by: t.found,
427
+ operations: t.routes
428
+ .filter((r) => !side || r.direction === side || r.direction === 'both')
429
+ .map((r) => `${r.attributes.httpMethod} ${r.attributes.path} ` +
430
+ `${r.attributes.name} (${r.direction})${source ? ` ${sourceTag(r.attributes.source)}` : ''} ${chainOf(index.graph, t.id, r)}`),
431
+ })),
432
+ };
433
+ },
434
+ });
435
+ return [searchApi, describeTypes, findTypesWithProperty, listApi, grepApi, traceApi];
345
436
  }
346
437
  // ---------------------------------------------------------------------------
347
438
  const SUBJECTS = {
@@ -0,0 +1,52 @@
1
+ import type { Matcher } from '../common/match.ts';
2
+ import type { ApiGraph, NodeAttrs, NodeKind } from './graph.ts';
3
+ export type Side = 'input' | 'output' | 'both';
4
+ export interface Route {
5
+ /** the operation node */
6
+ id: string;
7
+ attributes: NodeAttrs;
8
+ /** whether the call accepts what was traced, returns it, or both */
9
+ direction: Side;
10
+ /** the nodes in between, nearest the operation first */
11
+ via: string[];
12
+ hops: number;
13
+ }
14
+ export interface Trace {
15
+ id: string;
16
+ attributes: NodeAttrs;
17
+ /** how many operations reach it, whatever was kept */
18
+ found: number;
19
+ routes: Route[];
20
+ truncated: boolean;
21
+ }
22
+ export interface Traced {
23
+ /** how many nodes were traced, whatever was kept */
24
+ found: number;
25
+ traces: Trace[];
26
+ truncated: boolean;
27
+ }
28
+ export interface TraceFilter {
29
+ /** node ids to start from, taken as given */
30
+ ids?: readonly string[];
31
+ /** which kinds a `name` may match; all three when unset */
32
+ kinds?: readonly NodeKind[];
33
+ name?: readonly Matcher[];
34
+ source?: string;
35
+ /** starting nodes kept */
36
+ limit?: number;
37
+ /** routes kept per starting node */
38
+ maxRoutes?: number;
39
+ maxHops?: number;
40
+ }
41
+ /** How far containment is read backwards before a route is too indirect to mean anything. */
42
+ export declare const DEFAULT_TRACE_HOPS = 8;
43
+ export declare function traceNodes(graph: ApiGraph, filter: TraceFilter): Traced;
44
+ export declare function traceOne(graph: ApiGraph, id: string, filter?: TraceFilter): Trace;
45
+ /**
46
+ * The chain from an operation down to the node that was traced, as one line.
47
+ * A field is written onto the type above it (`User.email`) and a parameter is
48
+ * marked (`?page_size`), so the shape of the call can be read off the route
49
+ * without going and looking any of it up.
50
+ */
51
+ export declare function chainOf(graph: ApiGraph, start: string, route: Route): string;
52
+ //# sourceMappingURL=trace.d.ts.map
@@ -0,0 +1,144 @@
1
+ import { listNodes } from "./lookup.js";
2
+ /** How far containment is read backwards before a route is too indirect to mean anything. */
3
+ export const DEFAULT_TRACE_HOPS = 8;
4
+ /** A walk is bounded; a document can always be larger than the one this was written against. */
5
+ const MAX_VISITS = 20_000;
6
+ /**
7
+ * What a bare pattern matches. Operations are left out on purpose: they are
8
+ * where a trace ends, so starting one at an operationId that happens to share
9
+ * a word adds a row saying only that it found itself. Naming a `Method:` id
10
+ * outright still works.
11
+ */
12
+ const KINDS = ['type', 'property'];
13
+ /** The edges an operation holds its payload by. Reaching one of these is arriving. */
14
+ const ENTRY = new Set(['TAKES_INPUT', 'RETURNS_OUTPUT', 'HAS_PARAM']);
15
+ /** Containment, read backwards: a type to the fields that hold it, a field to its owner. */
16
+ const UPWARD = new Set(['HAS_PROPERTY', 'OF_TYPE', 'COMPOSES', 'ITEM_OF']);
17
+ export function traceNodes(graph, filter) {
18
+ const starts = select(graph, filter);
19
+ const kept = filter.limit && filter.limit < starts.length ? starts.slice(0, filter.limit) : starts;
20
+ return {
21
+ found: starts.length,
22
+ truncated: kept.length < starts.length,
23
+ traces: kept.map((id) => traceOne(graph, id, filter)),
24
+ };
25
+ }
26
+ export function traceOne(graph, id, filter = {}) {
27
+ const attributes = graph.getNodeAttributes(id);
28
+ // An operation is already where a trace ends; it reaches itself and nothing above it.
29
+ const routes = attributes.kind === 'method'
30
+ ? [{ id, attributes, direction: 'both', via: [], hops: 0 }]
31
+ : climb(graph, id, filter.maxHops ?? DEFAULT_TRACE_HOPS);
32
+ const limit = filter.maxRoutes;
33
+ return {
34
+ id,
35
+ attributes,
36
+ found: routes.length,
37
+ routes: limit && limit < routes.length ? routes.slice(0, limit) : routes,
38
+ truncated: Boolean(limit && limit < routes.length),
39
+ };
40
+ }
41
+ /**
42
+ * The chain from an operation down to the node that was traced, as one line.
43
+ * A field is written onto the type above it (`User.email`) and a parameter is
44
+ * marked (`?page_size`), so the shape of the call can be read off the route
45
+ * without going and looking any of it up.
46
+ */
47
+ export function chainOf(graph, start, route) {
48
+ let out = '';
49
+ for (const id of [...route.via, start]) {
50
+ const label = labelOf(graph, id);
51
+ out += out === '' || label.startsWith('.') ? label : ` → ${label}`;
52
+ }
53
+ return out;
54
+ }
55
+ // ---------------------------------------------------------------------------
56
+ function labelOf(graph, id) {
57
+ const a = graph.getNodeAttributes(id);
58
+ if (a.kind !== 'property') {
59
+ return a.name;
60
+ }
61
+ // A parameter belongs to a call rather than to a type, so it cannot be
62
+ // written as a field of the thing before it.
63
+ return id.includes('#') ? `?${a.name}` : `.${a.name}`;
64
+ }
65
+ function select(graph, filter) {
66
+ const out = new Set((filter.ids ?? []).filter((id) => graph.hasNode(id)));
67
+ if (filter.name) {
68
+ for (const kind of filter.kinds ?? KINDS) {
69
+ for (const row of listNodes(graph, { kind, name: filter.name, source: filter.source })
70
+ .rows) {
71
+ out.add(row.id);
72
+ }
73
+ }
74
+ }
75
+ return [...out].sort((a, b) => a.localeCompare(b));
76
+ }
77
+ /**
78
+ * Breadth-first up containment, recording an operation the moment one is
79
+ * reached. Breadth-first is what makes the answer the *shortest* way in: a
80
+ * type held by a wrapper held by a request body should be reported through
81
+ * the body, not through whichever branch happened to be walked first.
82
+ */
83
+ function climb(graph, start, maxHops) {
84
+ const previous = new Map();
85
+ const seen = new Set([start]);
86
+ const found = new Map();
87
+ let frontier = [start];
88
+ for (let hop = 0; hop <= maxHops && frontier.length > 0 && seen.size < MAX_VISITS; hop++) {
89
+ const next = [];
90
+ for (const node of frontier) {
91
+ for (const edge of graph.inEdges(node)) {
92
+ const relation = graph.getEdgeAttribute(edge, 'relation');
93
+ const from = graph.source(edge);
94
+ if (ENTRY.has(relation)) {
95
+ arrive(found, {
96
+ id: from,
97
+ attributes: graph.getNodeAttributes(from),
98
+ direction: sideOf(relation),
99
+ via: upward(previous, start, node),
100
+ hops: hop + 1,
101
+ });
102
+ }
103
+ else if (UPWARD.has(relation) && !seen.has(from)) {
104
+ seen.add(from);
105
+ previous.set(from, node);
106
+ next.push(from);
107
+ }
108
+ }
109
+ }
110
+ frontier = next;
111
+ }
112
+ return [...found.values()].sort((a, b) => a.hops - b.hops ||
113
+ a.attributes.path.localeCompare(b.attributes.path) ||
114
+ a.attributes.httpMethod.localeCompare(b.attributes.httpMethod));
115
+ }
116
+ /**
117
+ * One operation, once. A call that both accepts and returns the same type is
118
+ * two edges and one answer, and the honest word for that answer is `both`.
119
+ */
120
+ function arrive(found, route) {
121
+ const existing = found.get(route.id);
122
+ if (!existing) {
123
+ found.set(route.id, route);
124
+ }
125
+ else if (existing.direction !== route.direction) {
126
+ existing.direction = 'both';
127
+ }
128
+ }
129
+ const sideOf = (relation) => (relation === 'RETURNS_OUTPUT' ? 'output' : 'input');
130
+ /** The nodes walked through, from the one holding the entry edge back down towards the start. */
131
+ function upward(previous, start, from) {
132
+ const out = [];
133
+ let at = from;
134
+ while (at !== start) {
135
+ out.push(at);
136
+ const below = previous.get(at);
137
+ if (below === undefined) {
138
+ break;
139
+ }
140
+ at = below;
141
+ }
142
+ return out;
143
+ }
144
+ //# sourceMappingURL=trace.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zenera/rag",
3
- "version": "1.1.6",
3
+ "version": "1.1.8",
4
4
  "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
5
5
  "keywords": [
6
6
  "agents",
@@ -53,7 +53,7 @@
53
53
  "@apidevtools/swagger-parser": "^12.0.0",
54
54
  "@lancedb/lancedb": "^0.38.0",
55
55
  "graphology": "^0.26.0",
56
- "@zenera/cli": "^1.1.6",
57
- "@zenera/neo": "^1.1.6"
56
+ "@zenera/cli": "^1.1.8",
57
+ "@zenera/neo": "^1.1.8"
58
58
  }
59
59
  }