@zenera/rag 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +144 -0
- package/dist/command.d.ts +3 -0
- package/dist/command.js +436 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +16 -0
- package/dist/present.d.ts +17 -0
- package/dist/present.js +40 -0
- package/dist/query.d.ts +7 -0
- package/dist/query.js +88 -0
- package/dist/repl.d.ts +8 -0
- package/dist/repl.js +119 -0
- package/dist/schema/build.d.ts +28 -0
- package/dist/schema/build.js +79 -0
- package/dist/schema/entities.d.ts +18 -0
- package/dist/schema/entities.js +71 -0
- package/dist/schema/files.d.ts +74 -0
- package/dist/schema/files.js +79 -0
- package/dist/schema/graph.d.ts +48 -0
- package/dist/schema/graph.js +320 -0
- package/dist/schema/hydrate.d.ts +19 -0
- package/dist/schema/hydrate.js +182 -0
- package/dist/schema/render.d.ts +11 -0
- package/dist/schema/render.js +254 -0
- package/dist/schema/schema.d.ts +11 -0
- package/dist/schema/schema.js +189 -0
- package/dist/schema/search.d.ts +50 -0
- package/dist/schema/search.js +142 -0
- package/dist/schema/spec.d.ts +58 -0
- package/dist/schema/spec.js +309 -0
- package/dist/schema/store.d.ts +32 -0
- package/dist/schema/store.js +126 -0
- package/dist/schema/subgraph.d.ts +42 -0
- package/dist/schema/subgraph.js +272 -0
- package/dist/schema/tools.d.ts +10 -0
- package/dist/schema/tools.js +242 -0
- package/dist/schema/typescript.d.ts +22 -0
- package/dist/schema/typescript.js +246 -0
- package/package.json +59 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrey Ryabov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# @zenera/rag
|
|
2
|
+
|
|
3
|
+
**An OpenAPI description, indexed as a graph and searched by meaning — for
|
|
4
|
+
agents that have to call an API they have not read.**
|
|
5
|
+
|
|
6
|
+
[](https://github.com/andreyryabov/ZeneraNeo/blob/main/LICENSE)
|
|
7
|
+
[](https://nodejs.org)
|
|
8
|
+
|
|
9
|
+
> Part of [ZeneraNeo](https://github.com/andreyryabov/ZeneraNeo). It ships no
|
|
10
|
+
> binary of its own: installing it adds a `rag` **subcommand** to
|
|
11
|
+
> [`zen`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md),
|
|
12
|
+
> which is also where the credentials already are.
|
|
13
|
+
|
|
14
|
+
## Why
|
|
15
|
+
|
|
16
|
+
A large specification does not fit in a prompt, and the parts of it that answer
|
|
17
|
+
a question are scattered: the field is on a schema, the schema is on a request
|
|
18
|
+
body, the request body belongs to one operation out of three hundred. Vector
|
|
19
|
+
search finds the field. Only a graph gets from there to the call.
|
|
20
|
+
|
|
21
|
+
So this keeps both — a [graphology](https://graphology.github.io) graph for
|
|
22
|
+
structure and a [LanceDB](https://lancedb.com) table for hybrid vector +
|
|
23
|
+
full-text retrieval — and answers with the connected piece of the API that
|
|
24
|
+
matched, rendered as a tree, a Mermaid diagram, TypeScript declarations or a
|
|
25
|
+
standalone OpenAPI document.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
Node.js 24+. Install it alongside the CLI:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
npm i -g @zenera/cli @zenera/rag openai
|
|
33
|
+
zen key add openai # the keyring `zen` already uses
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Use
|
|
37
|
+
|
|
38
|
+
Index once, then ask:
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
zen rag schema index --embedding openai:text-embedding-3-small ./specs/*.yaml
|
|
42
|
+
zen rag schema search --output-property "user billing history"
|
|
43
|
+
zen rag schema search --input-property "password reset token" --format ts
|
|
44
|
+
zen rag schema search --interactive
|
|
45
|
+
zen rag schema stats # what is in the index, and what built it
|
|
46
|
+
zen rag schema show Type:Invoice # a named node, with no search in between
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Non-interactive search is a machine interface: every field is a flag, the whole
|
|
50
|
+
query can arrive as one JSON object, `--json` is a stable shape, no terminal is
|
|
51
|
+
required, and an empty result exits 0.
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
zen rag schema search --query - --format ts <<'JSON'
|
|
55
|
+
{
|
|
56
|
+
"input_properties": ["password reset token"],
|
|
57
|
+
"method_type": "read_write",
|
|
58
|
+
"exclude_ids": ["Type:PublicUserProfile"],
|
|
59
|
+
"limit": 3
|
|
60
|
+
}
|
|
61
|
+
JSON
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Commands
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
zen rag schema index <spec...> Read the documents and write a searchable index.
|
|
68
|
+
zen rag schema search Ask it something. --interactive for a prompt.
|
|
69
|
+
zen rag schema show <id...> Print named nodes, with no search in between.
|
|
70
|
+
zen rag schema stats What is in an index, and what built it.
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Search terms are one flag each — `--all`, `--method`, `--type`, `--input-type`,
|
|
74
|
+
`--output-type`, `--property`, `--input-property`, `--output-property` — shaped
|
|
75
|
+
by `--direction`, `--method-type`, `--limit`, `--max-hops`, `--max-nodes` and
|
|
76
|
+
the four `--exclude-*` filters, and rendered by `--format text | mermaid |
|
|
77
|
+
mermaid-flowchart | ts | openapi`. `zen help rag` prints the full table.
|
|
78
|
+
|
|
79
|
+
## From an agent
|
|
80
|
+
|
|
81
|
+
```ts
|
|
82
|
+
import { createEmbedder, loadProject } from '@zenera/neo';
|
|
83
|
+
import { SchemaIndex, schemaTools } from '@zenera/rag';
|
|
84
|
+
|
|
85
|
+
const index = await SchemaIndex.open(
|
|
86
|
+
'./schema-db',
|
|
87
|
+
createEmbedder('openai:text-embedding-3-small'),
|
|
88
|
+
);
|
|
89
|
+
const project = await loadProject('./my-project', { tools: schemaTools(index) });
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Four tools in the group `schema`, selectable as `schema:*`:
|
|
93
|
+
|
|
94
|
+
| Tool | For |
|
|
95
|
+
| -------------------------- | ------------------------------------------------------- |
|
|
96
|
+
| `search_api` | the connected piece of the API that matches an intent |
|
|
97
|
+
| `describe_types` | named schemas as declarations that compile on their own |
|
|
98
|
+
| `find_types_with_property` | which types have a field of this name — no search |
|
|
99
|
+
| `list_methods` | the shape of the API, by path |
|
|
100
|
+
|
|
101
|
+
`find_types_with_property` is the one for the repair loop: when `tsc` says
|
|
102
|
+
`'password' does not exist in type 'PublicUserProfile'`, the model does not
|
|
103
|
+
need the word explained again, it needs the list of types that have one.
|
|
104
|
+
|
|
105
|
+
## What an index is
|
|
106
|
+
|
|
107
|
+
```
|
|
108
|
+
schema-db/
|
|
109
|
+
├── manifest.json written last — its absence means "not indexed"
|
|
110
|
+
├── graph.json topology and light attributes, read whole
|
|
111
|
+
├── schemas.json the raw schemas, read on first hydrate
|
|
112
|
+
├── operations.json likewise, for the OpenAPI subset
|
|
113
|
+
└── lance/ one table: a row per node, one text column, one vector
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The manifest records which embedder made the vectors, and a search with a
|
|
117
|
+
different one is refused rather than answered with noise.
|
|
118
|
+
|
|
119
|
+
## Notes
|
|
120
|
+
|
|
121
|
+
- Documents are **bundled, not dereferenced**: `#/components/schemas/User` is
|
|
122
|
+
an edge and `User` is a node id. Swagger 2.0, OpenAPI 3.0 and 3.1 are
|
|
123
|
+
converted to 2020-12 on the way in, `discriminator` included — it is what
|
|
124
|
+
turns a `oneOf` into a TypeScript tagged union the compiler can narrow.
|
|
125
|
+
- A **query parameter is a property**, like any field in a body. Nobody should
|
|
126
|
+
have to know in advance which one `page_size` is.
|
|
127
|
+
- Every type carries a **direction** — `input`, `output` or `both` — worked out
|
|
128
|
+
by propagating from the operations through composition, so a shared DTO is
|
|
129
|
+
honestly both rather than whichever side was read last.
|
|
130
|
+
- Filters reaching the store are **closed enums only**. Exclusion lists are
|
|
131
|
+
applied in JavaScript afterwards, so nothing a model wrote ever reaches a SQL
|
|
132
|
+
predicate.
|
|
133
|
+
|
|
134
|
+
## The rest of the family
|
|
135
|
+
|
|
136
|
+
| Package | What it is |
|
|
137
|
+
| ----------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
|
|
138
|
+
| [`@zenera/cli`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md) | `zen` — agent projects on the command line |
|
|
139
|
+
| [`@zenera/neo`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/neo/README.md) | the runtime — agents, models, tools, skills, memory |
|
|
140
|
+
| [`@zenera/faker`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/faker/README.md) | `zen faker` — a mock API from the same kind of document |
|
|
141
|
+
|
|
142
|
+
## License
|
|
143
|
+
|
|
144
|
+
[MIT](https://github.com/andreyryabov/ZeneraNeo/blob/main/LICENSE).
|
package/dist/command.js
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import { relative, resolve } from 'node:path';
|
|
2
|
+
import { bold, CliError, cyan, dim, ensureHome, EXIT, isInteractive, json, KeyStore, note, parse, PROVIDERS, SHAPES, table, usageError, write, } from '@zenera/cli/lib';
|
|
3
|
+
import { createEmbedder } from '@zenera/neo';
|
|
4
|
+
import { isFormat, present } from "./present.js";
|
|
5
|
+
import { isEmpty, parseQuery, QueryError } from "./query.js";
|
|
6
|
+
import { repl } from "./repl.js";
|
|
7
|
+
import { buildIndex } from "./schema/build.js";
|
|
8
|
+
import { assertSameEmbedding, openIndex, readManifest } from "./schema/files.js";
|
|
9
|
+
import { SchemaIndex } from "./schema/search.js";
|
|
10
|
+
import { stitch } from "./schema/subgraph.js";
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// zen rag — an api description, as something to search
|
|
13
|
+
//
|
|
14
|
+
// `search` has two modes and neither is the afterthought. Interactively it is
|
|
15
|
+
// a loop with prompts; non-interactively it is a tool, and that is the mode
|
|
16
|
+
// that has to be exactly specified: every field settable from a flag, the whole
|
|
17
|
+
// query settable as one JSON object, a stable `--json` shape, no terminal
|
|
18
|
+
// required, and exit 0 when nothing matched — an empty answer is an answer, and
|
|
19
|
+
// a caller that has to tell "no results" from "the index is missing" by parsing
|
|
20
|
+
// stderr will get it wrong.
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
const USAGE = 'zen rag schema <index|search|show|stats> [spec...]';
|
|
23
|
+
const INDEX_USAGE = 'zen rag schema index --embedding <ref> [--out <dir>] <spec...>';
|
|
24
|
+
const SEARCH_USAGE = 'zen rag schema search [--dir <dir>] [query...]';
|
|
25
|
+
const DEFAULT_DIR = './schema-db';
|
|
26
|
+
export const command = {
|
|
27
|
+
summary: 'Search an openapi/swagger document as a graph.',
|
|
28
|
+
usage: USAGE,
|
|
29
|
+
details: [
|
|
30
|
+
'Commands',
|
|
31
|
+
...table([
|
|
32
|
+
[' index <spec...>', dim('Read the documents and write a searchable index.')],
|
|
33
|
+
[' search', dim('Ask it something. --interactive for a prompt.')],
|
|
34
|
+
[' show <id...>', dim('Print named nodes, with no search in between.')],
|
|
35
|
+
[' stats', dim('What is in an index, and what built it.')],
|
|
36
|
+
]),
|
|
37
|
+
'',
|
|
38
|
+
'Index',
|
|
39
|
+
...table([
|
|
40
|
+
[
|
|
41
|
+
' --embedding <ref>',
|
|
42
|
+
dim('Which embedder makes the vectors. Omit it to be shown the choices.'),
|
|
43
|
+
],
|
|
44
|
+
[' -o, --out <dir>', dim(`Where the index goes. Default ${DEFAULT_DIR}.`)],
|
|
45
|
+
[
|
|
46
|
+
' --batch <n>',
|
|
47
|
+
dim('Texts per embedding request, and how often progress prints. Default 96.'),
|
|
48
|
+
],
|
|
49
|
+
]),
|
|
50
|
+
'',
|
|
51
|
+
'Search terms (repeatable)',
|
|
52
|
+
...table([
|
|
53
|
+
[' <text>', dim('A bare phrase, the same as --all.')],
|
|
54
|
+
[' --all <q>', dim('Against everything, unfiltered.')],
|
|
55
|
+
[' --method <q>', dim('Operations.')],
|
|
56
|
+
[' --type <q>', dim('Schemas, on the side --direction names.')],
|
|
57
|
+
[' --input-type <q>', dim('Schemas a call accepts.')],
|
|
58
|
+
[' --output-type <q>', dim('Schemas a call returns.')],
|
|
59
|
+
[' --property <q>', dim('Fields and parameters, per --direction.')],
|
|
60
|
+
[' --input-property <q>', dim('Fields and parameters a call accepts.')],
|
|
61
|
+
[' --output-property <q>', dim('Fields a call returns.')],
|
|
62
|
+
[' --query <json|->', dim('A whole query object; - reads stdin.')],
|
|
63
|
+
]),
|
|
64
|
+
'',
|
|
65
|
+
'Search filters and shape',
|
|
66
|
+
...table([
|
|
67
|
+
[' -d, --dir <dir>', dim(`Which index. Default ${DEFAULT_DIR}.`)],
|
|
68
|
+
[' --embedding <ref>', dim('Must be the one the index was built with.')],
|
|
69
|
+
[' --direction <d>', dim('input | output | any. Default any.')],
|
|
70
|
+
[' --method-type <t>', dim('read_only | read_write | any. Default any.')],
|
|
71
|
+
[' --exclude-id <id>', dim('Drop a node. Repeatable, as are the three below.')],
|
|
72
|
+
[' --exclude-method <name>', dim('Drop an operation by name.')],
|
|
73
|
+
[' --exclude-type <name>', dim('Drop a schema by name.')],
|
|
74
|
+
[' --exclude-property <name>', dim('Drop a field by name.')],
|
|
75
|
+
[' --limit <n>', dim('Seeds kept per term. Default 5.')],
|
|
76
|
+
[' --max-hops <n>', dim('How far apart two hits may be. Default 3.')],
|
|
77
|
+
[' --max-nodes <n>', dim('Nodes per result. Default 200.')],
|
|
78
|
+
[' --format <f>', dim('text | mermaid | mermaid-flowchart | ts | openapi.')],
|
|
79
|
+
[' --no-docs', dim('Leave the descriptions out.')],
|
|
80
|
+
[' --interactive', dim('Prompt, search, refine. Needs a terminal.')],
|
|
81
|
+
[' --quiet', dim('No narration.')],
|
|
82
|
+
]),
|
|
83
|
+
'',
|
|
84
|
+
dim(`Credentials come from the ${cyan('zen')} keyring — try ${cyan('zen key ls')}.`),
|
|
85
|
+
],
|
|
86
|
+
async run(ctx) {
|
|
87
|
+
const [group, ...rest] = ctx.args;
|
|
88
|
+
// `schema` is the only subject so far; leaving it out is a courtesy,
|
|
89
|
+
// not a second spelling to support forever.
|
|
90
|
+
const [name, ...tail] = group === 'schema' ? rest : ctx.args;
|
|
91
|
+
switch (name) {
|
|
92
|
+
case 'index':
|
|
93
|
+
return await index(tail, ctx);
|
|
94
|
+
case 'search':
|
|
95
|
+
return await search(tail, ctx);
|
|
96
|
+
case 'show':
|
|
97
|
+
return await show(tail, ctx);
|
|
98
|
+
case 'stats':
|
|
99
|
+
return await stats(tail, ctx);
|
|
100
|
+
default:
|
|
101
|
+
throw usageError(name ? `unknown command "${name}"` : 'no command given', USAGE);
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
async function index(args, ctx) {
|
|
106
|
+
const { values, positionals } = parse(args, {
|
|
107
|
+
out: { type: 'string', short: 'o' },
|
|
108
|
+
embedding: { type: 'string' },
|
|
109
|
+
batch: { type: 'string' },
|
|
110
|
+
quiet: { type: 'boolean' },
|
|
111
|
+
}, INDEX_USAGE);
|
|
112
|
+
if (positionals.length === 0) {
|
|
113
|
+
throw usageError('no document given', INDEX_USAGE);
|
|
114
|
+
}
|
|
115
|
+
const out = resolve(ctx.cwd, values.out ?? DEFAULT_DIR);
|
|
116
|
+
const loud = !values.quiet && !ctx.json;
|
|
117
|
+
const chosen = await embedder(values.embedding);
|
|
118
|
+
const started = Date.now();
|
|
119
|
+
const { manifest } = await buildIndex({
|
|
120
|
+
files: positionals.map((file) => resolve(ctx.cwd, file)),
|
|
121
|
+
out,
|
|
122
|
+
embedder: chosen,
|
|
123
|
+
embeddingRef: values.embedding,
|
|
124
|
+
indexer: 'zenera-rag',
|
|
125
|
+
batch: values.batch ? count(values.batch, '--batch') : undefined,
|
|
126
|
+
onRead: loud
|
|
127
|
+
? (summary) => {
|
|
128
|
+
printSources(summary.sources, ctx.cwd);
|
|
129
|
+
// The first batch can take a while and says nothing while it
|
|
130
|
+
// does; this is the line that makes that a wait, not a hang.
|
|
131
|
+
note(dim(` embedding ${summary.counts.entities} entities with ${chosen.id} …`));
|
|
132
|
+
}
|
|
133
|
+
: undefined,
|
|
134
|
+
onProgress: loud
|
|
135
|
+
? (done, total) => note(dim(` embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
|
|
136
|
+
: undefined,
|
|
137
|
+
});
|
|
138
|
+
if (ctx.json) {
|
|
139
|
+
json({ out, manifest });
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
// stdout is the path and nothing else, so `DIR=$(zen rag schema index …)`
|
|
143
|
+
// works; what it means goes to stderr, where the narration lives.
|
|
144
|
+
note();
|
|
145
|
+
write(out);
|
|
146
|
+
note(` wrote ${bold(String(manifest.counts.entities))} entities to ${bold(out)}, ` +
|
|
147
|
+
`embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
|
|
148
|
+
const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
|
|
149
|
+
note(dim(` search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
|
|
150
|
+
}
|
|
151
|
+
const HEADERS = ['PATHS', 'OPERATIONS', 'SCHEMAS', 'FIELDS'];
|
|
152
|
+
function elapsed(since) {
|
|
153
|
+
const seconds = Math.round((Date.now() - since) / 1000);
|
|
154
|
+
return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
|
|
155
|
+
}
|
|
156
|
+
function printSources(sources, cwd) {
|
|
157
|
+
const rows = sources.map((s) => ({
|
|
158
|
+
name: relative(cwd, s.path) || s.path,
|
|
159
|
+
dialect: s.dialect,
|
|
160
|
+
cells: [s.paths, s.methods, s.types, s.properties],
|
|
161
|
+
}));
|
|
162
|
+
if (rows.length > 1) {
|
|
163
|
+
rows.push({
|
|
164
|
+
name: 'total',
|
|
165
|
+
dialect: '',
|
|
166
|
+
cells: HEADERS.map((_, i) => rows.reduce((n, r) => n + (r.cells[i] ?? 0), 0)),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
// Numbers are padded before they are styled: a colour code has no width,
|
|
170
|
+
// and `table` cannot know that.
|
|
171
|
+
const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r.cells[i]).length)));
|
|
172
|
+
note();
|
|
173
|
+
notes(table([
|
|
174
|
+
[bold('SPEC'), bold('DIALECT'), ...HEADERS.map((h, i) => bold(h.padStart(widths[i])))],
|
|
175
|
+
...rows.map((r) => [
|
|
176
|
+
r.name === 'total' ? dim(r.name) : r.name,
|
|
177
|
+
dim(r.dialect),
|
|
178
|
+
...r.cells.map((c, i) => String(c).padStart(widths[i])),
|
|
179
|
+
]),
|
|
180
|
+
]).map((line) => ` ${line}`));
|
|
181
|
+
note();
|
|
182
|
+
}
|
|
183
|
+
const MANY = { type: 'string', multiple: true };
|
|
184
|
+
const SEARCH_OPTIONS = {
|
|
185
|
+
dir: { type: 'string', short: 'd' },
|
|
186
|
+
embedding: { type: 'string' },
|
|
187
|
+
all: MANY,
|
|
188
|
+
method: MANY,
|
|
189
|
+
type: MANY,
|
|
190
|
+
'input-type': MANY,
|
|
191
|
+
'output-type': MANY,
|
|
192
|
+
property: MANY,
|
|
193
|
+
'input-property': MANY,
|
|
194
|
+
'output-property': MANY,
|
|
195
|
+
query: { type: 'string' },
|
|
196
|
+
direction: { type: 'string' },
|
|
197
|
+
'method-type': { type: 'string' },
|
|
198
|
+
'exclude-id': MANY,
|
|
199
|
+
'exclude-method': MANY,
|
|
200
|
+
'exclude-type': MANY,
|
|
201
|
+
'exclude-property': MANY,
|
|
202
|
+
limit: { type: 'string' },
|
|
203
|
+
'max-hops': { type: 'string' },
|
|
204
|
+
'max-nodes': { type: 'string' },
|
|
205
|
+
format: { type: 'string' },
|
|
206
|
+
'no-docs': { type: 'boolean' },
|
|
207
|
+
'only-hits': { type: 'boolean' },
|
|
208
|
+
interactive: { type: 'boolean' },
|
|
209
|
+
quiet: { type: 'boolean' },
|
|
210
|
+
};
|
|
211
|
+
async function search(args, ctx) {
|
|
212
|
+
const { values, positionals } = parse(args, SEARCH_OPTIONS, SEARCH_USAGE);
|
|
213
|
+
const dir = resolve(ctx.cwd, values.dir ?? DEFAULT_DIR);
|
|
214
|
+
const format = formatOf(values.format);
|
|
215
|
+
const options = { docs: !values['no-docs'], onlyHits: values['only-hits'] };
|
|
216
|
+
const query = { ...(await fromStdin(values.query)), ...fromFlags(values, positionals) };
|
|
217
|
+
// Everything that can be wrong about the invocation is settled before a
|
|
218
|
+
// credential is asked for, so a typo is a usage error and not a login.
|
|
219
|
+
if (values.interactive && !isInteractive()) {
|
|
220
|
+
throw usageError('--interactive needs a terminal', SEARCH_USAGE);
|
|
221
|
+
}
|
|
222
|
+
if (!values.interactive && isEmpty(query)) {
|
|
223
|
+
throw usageError('no query given', SEARCH_USAGE);
|
|
224
|
+
}
|
|
225
|
+
const manifest = await readManifest(dir);
|
|
226
|
+
const ref = values.embedding ?? manifest.embedding.ref;
|
|
227
|
+
assertSameEmbedding(manifest, ref);
|
|
228
|
+
const index = await SchemaIndex.open(dir, await embedder(ref));
|
|
229
|
+
try {
|
|
230
|
+
if (values.interactive) {
|
|
231
|
+
await repl(index, query, { format, ...options });
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const result = await index.search(query);
|
|
235
|
+
if (ctx.json) {
|
|
236
|
+
json({
|
|
237
|
+
seeds: result.seeds,
|
|
238
|
+
empty: result.empty,
|
|
239
|
+
subgraphs: result.subgraphs,
|
|
240
|
+
rendered: await present(index, result.subgraphs, format, options),
|
|
241
|
+
});
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const text = await present(index, result.subgraphs, format, options);
|
|
245
|
+
if (text) {
|
|
246
|
+
write(text);
|
|
247
|
+
}
|
|
248
|
+
if (!values.quiet) {
|
|
249
|
+
note(dim(` ${result.seeds.length} seed(s) · ${result.subgraphs.length} result(s)${result.empty.length > 0 ? ` · nothing for: ${result.empty.join(', ')}` : ''}`));
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
index.close();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
/** Flags win over `--query`: the more specific spelling is the later thought. */
|
|
257
|
+
function fromFlags(values, positionals = []) {
|
|
258
|
+
const query = {};
|
|
259
|
+
const put = (key, value) => {
|
|
260
|
+
if (value !== undefined && (!Array.isArray(value) || value.length > 0)) {
|
|
261
|
+
query[key] = value;
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
// A bare phrase is the unfiltered search; there is nothing else it could mean.
|
|
265
|
+
put('all', [...(values.all ?? []), ...positionals]);
|
|
266
|
+
put('methods', values.method);
|
|
267
|
+
put('types', values.type);
|
|
268
|
+
put('input_types', values['input-type']);
|
|
269
|
+
put('output_types', values['output-type']);
|
|
270
|
+
put('properties', values.property);
|
|
271
|
+
put('input_properties', values['input-property']);
|
|
272
|
+
put('output_properties', values['output-property']);
|
|
273
|
+
put('exclude_ids', values['exclude-id']);
|
|
274
|
+
put('exclude_methods', values['exclude-method']);
|
|
275
|
+
put('exclude_types', values['exclude-type']);
|
|
276
|
+
put('exclude_properties', values['exclude-property']);
|
|
277
|
+
put('direction', values.direction);
|
|
278
|
+
put('method_type', values['method-type']);
|
|
279
|
+
put('limit', values.limit && count(values.limit, '--limit'));
|
|
280
|
+
put('max_hops', values['max-hops'] && count(values['max-hops'], '--max-hops'));
|
|
281
|
+
put('max_nodes', values['max-nodes'] && count(values['max-nodes'], '--max-nodes'));
|
|
282
|
+
return check(query);
|
|
283
|
+
}
|
|
284
|
+
async function fromStdin(source) {
|
|
285
|
+
if (source === undefined) {
|
|
286
|
+
return {};
|
|
287
|
+
}
|
|
288
|
+
const text = source === '-' ? await readStdin() : source;
|
|
289
|
+
let parsed;
|
|
290
|
+
try {
|
|
291
|
+
parsed = JSON.parse(text);
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
throw usageError(`--query is not JSON: ${err.message}`, SEARCH_USAGE);
|
|
295
|
+
}
|
|
296
|
+
return check(parsed);
|
|
297
|
+
}
|
|
298
|
+
async function readStdin() {
|
|
299
|
+
const chunks = [];
|
|
300
|
+
for await (const chunk of process.stdin) {
|
|
301
|
+
chunks.push(chunk);
|
|
302
|
+
}
|
|
303
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
304
|
+
}
|
|
305
|
+
function check(value) {
|
|
306
|
+
try {
|
|
307
|
+
return parseQuery(value);
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
if (err instanceof QueryError) {
|
|
311
|
+
throw usageError(err.message, SEARCH_USAGE);
|
|
312
|
+
}
|
|
313
|
+
throw err;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
// ---------------------------------------------------------------------------
|
|
317
|
+
// show, stats
|
|
318
|
+
// ---------------------------------------------------------------------------
|
|
319
|
+
/**
|
|
320
|
+
* No embedder and no store: naming a node is a graph lookup, and asking for a
|
|
321
|
+
* credential to print something already on disk would be theatre.
|
|
322
|
+
*/
|
|
323
|
+
async function show(args, ctx) {
|
|
324
|
+
const usage = 'zen rag schema show <id...>';
|
|
325
|
+
const { values, positionals } = parse(args, SEARCH_OPTIONS, usage);
|
|
326
|
+
if (positionals.length === 0) {
|
|
327
|
+
throw usageError('no node named', usage);
|
|
328
|
+
}
|
|
329
|
+
const format = formatOf(values.format);
|
|
330
|
+
const index = await openIndex(resolve(ctx.cwd, values.dir ?? DEFAULT_DIR));
|
|
331
|
+
const missing = positionals.filter((id) => !index.graph.hasNode(id));
|
|
332
|
+
if (missing.length > 0) {
|
|
333
|
+
throw new CliError(`no such node: ${missing.join(', ')}`, EXIT.failed, 'ids look like `Type:User` or `Property:User.email`');
|
|
334
|
+
}
|
|
335
|
+
const subgraphs = stitch(index.graph, positionals.map((id) => ({ id, term: id, field: 'show', score: 1 })), { maxNodes: values['max-nodes'] ? count(values['max-nodes'], '--max-nodes') : undefined });
|
|
336
|
+
const text = await present(index, subgraphs, format, { docs: !values['no-docs'] });
|
|
337
|
+
if (ctx.json) {
|
|
338
|
+
json({ subgraphs, rendered: text });
|
|
339
|
+
}
|
|
340
|
+
else if (text) {
|
|
341
|
+
write(text);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
async function stats(args, ctx) {
|
|
345
|
+
const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag schema stats [--dir <dir>]');
|
|
346
|
+
const dir = resolve(ctx.cwd, values.dir ?? DEFAULT_DIR);
|
|
347
|
+
const manifest = await readManifest(dir);
|
|
348
|
+
if (ctx.json) {
|
|
349
|
+
json(manifest);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
note(bold(dir));
|
|
353
|
+
notes(table([
|
|
354
|
+
[' built', manifest.createdAt],
|
|
355
|
+
[' by', manifest.indexer],
|
|
356
|
+
[' embedder', `${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`],
|
|
357
|
+
[
|
|
358
|
+
' indexes',
|
|
359
|
+
`fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
|
|
360
|
+
],
|
|
361
|
+
]));
|
|
362
|
+
printSources(manifest.sources, ctx.cwd);
|
|
363
|
+
notes(table([[' entities', String(manifest.counts.entities)]]).map(dim));
|
|
364
|
+
}
|
|
365
|
+
function notes(lines) {
|
|
366
|
+
for (const line of lines) {
|
|
367
|
+
note(line);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
const yes = (value) => (value ? 'yes' : 'no');
|
|
371
|
+
// ---------------------------------------------------------------------------
|
|
372
|
+
/** The keyring is materialised here, and only here: `show` and `stats` read no vectors. */
|
|
373
|
+
async function embedder(ref) {
|
|
374
|
+
ensureHome();
|
|
375
|
+
const keys = await KeyStore.open();
|
|
376
|
+
// Asked before materialising, because materialising is what erases the
|
|
377
|
+
// difference between "the environment had it" and "the keyring supplied it".
|
|
378
|
+
const fromEnv = new Set(PROVIDERS.filter((p) => process.env[SHAPES[p].env]));
|
|
379
|
+
keys.materialize();
|
|
380
|
+
if (!ref) {
|
|
381
|
+
throw choices(keys, fromEnv);
|
|
382
|
+
}
|
|
383
|
+
return createEmbedder(ref);
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Well-known embedding models per provider. A list rather than a lookup: any
|
|
387
|
+
* ref the registry can parse works, and these are the ones worth typing.
|
|
388
|
+
* Anthropic is absent because it publishes no embeddings API at all.
|
|
389
|
+
*/
|
|
390
|
+
const EMBEDDINGS = {
|
|
391
|
+
openai: ['text-embedding-3-small', 'text-embedding-3-large'],
|
|
392
|
+
google: ['gemini-embedding-001'],
|
|
393
|
+
vertex: ['gemini-embedding-001', 'text-embedding-005'],
|
|
394
|
+
openrouter: ['openai/text-embedding-3-small'],
|
|
395
|
+
};
|
|
396
|
+
/** What could be passed, with the ones this machine can actually use first. */
|
|
397
|
+
function choices(keys, fromEnv) {
|
|
398
|
+
const rows = [];
|
|
399
|
+
const rest = [];
|
|
400
|
+
for (const provider of PROVIDERS) {
|
|
401
|
+
for (const model of EMBEDDINGS[provider] ?? []) {
|
|
402
|
+
const source = fromEnv.has(provider)
|
|
403
|
+
? 'environment'
|
|
404
|
+
: keys.active(provider)
|
|
405
|
+
? 'keyring'
|
|
406
|
+
: '';
|
|
407
|
+
const row = [` ${cyan(`${provider}:${model}`)}`, dim(source || SHAPES[provider].env)];
|
|
408
|
+
(source ? rows : rest).push(row);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
note(bold('Embeddings'));
|
|
412
|
+
notes(table([...rows, ...rest]));
|
|
413
|
+
note('');
|
|
414
|
+
if (rows.length === 0) {
|
|
415
|
+
note(dim(' no provider on this machine has a credential — try: zen key add openai'));
|
|
416
|
+
note('');
|
|
417
|
+
}
|
|
418
|
+
return usageError('no embedder named', 'pass --embedding <ref>, one of the above');
|
|
419
|
+
}
|
|
420
|
+
function formatOf(value) {
|
|
421
|
+
if (value === undefined) {
|
|
422
|
+
return 'text';
|
|
423
|
+
}
|
|
424
|
+
if (!isFormat(value)) {
|
|
425
|
+
throw usageError(`unknown format "${value}"`, 'expected text, mermaid, mermaid-flowchart, ts or openapi');
|
|
426
|
+
}
|
|
427
|
+
return value;
|
|
428
|
+
}
|
|
429
|
+
function count(value, flag) {
|
|
430
|
+
const number = Number(value);
|
|
431
|
+
if (!Number.isInteger(number) || number < 1) {
|
|
432
|
+
throw usageError(`${flag} must be a whole number of at least 1`, USAGE);
|
|
433
|
+
}
|
|
434
|
+
return number;
|
|
435
|
+
}
|
|
436
|
+
//# sourceMappingURL=command.js.map
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export * from './present.ts';
|
|
2
|
+
export * from './query.ts';
|
|
3
|
+
export * from './schema/build.ts';
|
|
4
|
+
export * from './schema/entities.ts';
|
|
5
|
+
export * from './schema/files.ts';
|
|
6
|
+
export * from './schema/graph.ts';
|
|
7
|
+
export * from './schema/hydrate.ts';
|
|
8
|
+
export * from './schema/render.ts';
|
|
9
|
+
export * from './schema/schema.ts';
|
|
10
|
+
export * from './schema/search.ts';
|
|
11
|
+
export * from './schema/spec.ts';
|
|
12
|
+
export * from './schema/store.ts';
|
|
13
|
+
export * from './schema/subgraph.ts';
|
|
14
|
+
export * from './schema/tools.ts';
|
|
15
|
+
export * from './schema/typescript.ts';
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export * from "./present.js";
|
|
2
|
+
export * from "./query.js";
|
|
3
|
+
export * from "./schema/build.js";
|
|
4
|
+
export * from "./schema/entities.js";
|
|
5
|
+
export * from "./schema/files.js";
|
|
6
|
+
export * from "./schema/graph.js";
|
|
7
|
+
export * from "./schema/hydrate.js";
|
|
8
|
+
export * from "./schema/render.js";
|
|
9
|
+
export * from "./schema/schema.js";
|
|
10
|
+
export * from "./schema/search.js";
|
|
11
|
+
export * from "./schema/spec.js";
|
|
12
|
+
export * from "./schema/store.js";
|
|
13
|
+
export * from "./schema/subgraph.js";
|
|
14
|
+
export * from "./schema/tools.js";
|
|
15
|
+
export * from "./schema/typescript.js";
|
|
16
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type HydrateOptions } from './schema/hydrate.ts';
|
|
2
|
+
import { type RenderOptions } from './schema/render.ts';
|
|
3
|
+
import type { Schema } from './schema/schema.ts';
|
|
4
|
+
import type { Operation } from './schema/spec.ts';
|
|
5
|
+
import type { Subgraph } from './schema/subgraph.ts';
|
|
6
|
+
export declare const FORMATS: readonly ["text", "mermaid", "mermaid-flowchart", "ts", "openapi"];
|
|
7
|
+
export type Format = (typeof FORMATS)[number];
|
|
8
|
+
export declare function isFormat(value: string): value is Format;
|
|
9
|
+
export interface OutputOptions extends RenderOptions, HydrateOptions {
|
|
10
|
+
}
|
|
11
|
+
/** Two of the four formats need what is on disk; an open index is enough. */
|
|
12
|
+
export interface SchemaSource {
|
|
13
|
+
schemas(): Promise<Record<string, Schema>>;
|
|
14
|
+
operations(): Promise<Operation[]>;
|
|
15
|
+
}
|
|
16
|
+
export declare function present(index: SchemaSource, subgraphs: readonly Subgraph[], format: Format, options?: OutputOptions): Promise<string>;
|
|
17
|
+
//# sourceMappingURL=present.d.ts.map
|