@zenera/rag 1.1.15 → 1.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +401 -295
- package/dist/command.js +15 -8
- package/dist/docs/command.js +58 -10
- package/dist/docs/search.d.ts +27 -0
- package/dist/docs/search.js +42 -5
- package/dist/docs/tools.js +20 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -12,54 +12,114 @@
|
|
|
12
12
|
|
|
13
13
|
# @zenera/rag
|
|
14
14
|
|
|
15
|
-
**
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
**Advanced indexing and retrieval for documents and API schemas, purpose-built
|
|
16
|
+
for agentic RAG.
|
|
17
|
+
Full-text, semantic, hybrid and graph search combine with
|
|
18
|
+
precise inspection and filtering so agents can verify every answer.**
|
|
18
19
|
|
|
19
20
|
> Part of [ZeneraNeo](https://github.com/andreyryabov/ZeneraNeo). It ships no
|
|
20
21
|
> binary of its own: installing it adds a `rag` **subcommand** to
|
|
21
22
|
> [`zen`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md),
|
|
22
|
-
>
|
|
23
|
+
> so indexing, retrieval and agent tools fit one developer workflow.
|
|
24
|
+
|
|
25
|
+
## Complete retrieval for agentic systems
|
|
26
|
+
|
|
27
|
+
`@zenera/rag` is a complete retrieval layer for agents that work with technical
|
|
28
|
+
knowledge. Index content once, then discover relevant context, narrow the scope,
|
|
29
|
+
inspect exact source material and return only the representation the next step
|
|
30
|
+
needs. The same workflow is available through CLI commands and agent tools.
|
|
31
|
+
|
|
32
|
+
### Document indexing and search
|
|
33
|
+
|
|
34
|
+
- **Semantic, full-text and hybrid search** across Markdown and plain-text
|
|
35
|
+
collections, with hybrid retrieval as the default.
|
|
36
|
+
- **Structure-aware indexing** for headings, sections, paragraphs, lists, code
|
|
37
|
+
blocks and tables instead of treating every document as undifferentiated text.
|
|
38
|
+
- **Precise scoping** by file, section and content kind, with iterative exclusion
|
|
39
|
+
of passages an agent has already considered.
|
|
40
|
+
- **Verbatim evidence** with source files, exact line ranges and configurable
|
|
41
|
+
surrounding context, ready for citation or follow-up reading.
|
|
42
|
+
- **Deterministic inspection** through file, section and table listing, literal
|
|
43
|
+
or regular-expression search, and direct reads by section or line range.
|
|
44
|
+
- **Multi-document indexing** with portable source-aware indexes, incremental
|
|
45
|
+
rebuilds and interactive or non-interactive search workflows.
|
|
46
|
+
|
|
47
|
+
### API schema indexing and search
|
|
48
|
+
|
|
49
|
+
- **OpenAPI and Swagger indexing** across one specification, many services or
|
|
50
|
+
multiple API revisions in a single searchable index.
|
|
51
|
+
- **Semantic, full-text and hybrid discovery** across operations, schemas,
|
|
52
|
+
properties, parameters, request bodies and responses.
|
|
53
|
+
- **Graph retrieval** that follows schema relationships and returns the connected
|
|
54
|
+
API context around a match instead of isolated search results.
|
|
55
|
+
- **Typed query controls** for methods, input and output types, properties,
|
|
56
|
+
direction, read-only or mutating operations, source and graph depth.
|
|
57
|
+
- **Relationship tracing** from a field or schema to every operation that accepts
|
|
58
|
+
or returns it, even when the operation and field share no searchable terms.
|
|
59
|
+
- **Multiple output formats** including readable trees, Mermaid diagrams,
|
|
60
|
+
self-contained TypeScript declarations and standalone OpenAPI documents.
|
|
61
|
+
- **Deterministic inspection** through exact method, type and property listing,
|
|
62
|
+
literal or regular-expression search, node lookup and source filtering.
|
|
63
|
+
|
|
64
|
+
### Agent-ready workflows
|
|
65
|
+
|
|
66
|
+
- **Discovery and verification in one workflow.** Search finds relevant context;
|
|
67
|
+
exact operations establish whether something exists and retrieve it directly.
|
|
68
|
+
- **Progressive narrowing.** Agents can refine by source, structure, direction,
|
|
69
|
+
type, path and previous results without rebuilding an index.
|
|
70
|
+
- **Stable machine interfaces.** Structured queries and JSON output support
|
|
71
|
+
repeatable automation, while typed toolsets expose the same capabilities to
|
|
72
|
+
ZeneraNeo agents.
|
|
73
|
+
- **Bounded context.** Result limits, graph depth, line ranges and output formats
|
|
74
|
+
keep retrieved context focused and suitable for model input.
|
|
75
|
+
|
|
76
|
+
## From relevant to verified
|
|
77
|
+
|
|
78
|
+
Search is how an agent starts with an incomplete question. Verification is how
|
|
79
|
+
it finishes with evidence. `grep`, `list`, `trace` and `show` inspect the index
|
|
80
|
+
directly, so an agent can confirm a field, identify the operations that carry
|
|
81
|
+
it, read the surrounding source and refine the next query from what it found.
|
|
82
|
+
|
|
83
|
+
That workflow avoids treating a similarity score as proof: retrieve a relevant
|
|
84
|
+
starting point, inspect the exact material, then return an answer with the
|
|
85
|
+
context needed for the next decision.
|
|
23
86
|
|
|
24
|
-
##
|
|
87
|
+
## Install
|
|
25
88
|
|
|
26
|
-
|
|
27
|
-
own flags — not a variation on one command, because what `list` means to an API
|
|
28
|
-
description is not what it means to a folder of notes.
|
|
89
|
+
Node.js 24+. Install it alongside the CLI:
|
|
29
90
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
91
|
+
```sh
|
|
92
|
+
npm i -g @zenera/cli @zenera/rag
|
|
93
|
+
zen key add openai # the keyring `zen` already uses
|
|
94
|
+
```
|
|
34
95
|
|
|
35
|
-
|
|
96
|
+
Use the CLI to create and query indexes, or expose an opened index as tools to a
|
|
97
|
+
ZeneraNeo agent.
|
|
36
98
|
|
|
37
|
-
|
|
38
|
-
and exact commands beside the ranking ones that need no credential at all.
|
|
99
|
+
## API schemas - `zen rag schema`
|
|
39
100
|
|
|
40
|
-
|
|
101
|
+
**OpenAPI 3.1, OpenAPI 3.0 and Swagger 2.0**, as `.json`, `.yaml` or `.yml`,
|
|
102
|
+
one document or a whole folder of them. External references and multiple
|
|
103
|
+
specification versions are indexed as one connected API surface, so a query can
|
|
104
|
+
follow relationships across services, revisions and shared definitions.
|
|
41
105
|
|
|
42
106
|
A large specification does not fit in a prompt, and the parts of it that answer
|
|
43
107
|
a question are scattered: the field is on a schema, the schema is on a request
|
|
44
|
-
body, the
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
structure and a [LanceDB](https://lancedb.com) table for hybrid vector +
|
|
49
|
-
full-text retrieval — and answers with the connected piece of the API that
|
|
50
|
-
matched, rendered as a tree, a Mermaid diagram, TypeScript declarations or a
|
|
108
|
+
body, the body belongs to one operation out of three hundred. Search finds the
|
|
109
|
+
field. Only a graph gets from there to the call - so this keeps both, and
|
|
110
|
+
answers with the connected piece of the API that matched, rendered as a tree, a
|
|
111
|
+
Mermaid diagram, TypeScript declarations that compile on their own, or a
|
|
51
112
|
standalone OpenAPI document.
|
|
52
113
|
|
|
53
|
-
## Install
|
|
54
|
-
|
|
55
|
-
Node.js 24+. Install it alongside the CLI:
|
|
56
|
-
|
|
57
|
-
```sh
|
|
58
|
-
npm i -g @zenera/cli @zenera/rag
|
|
59
|
-
zen key add openai # the keyring `zen` already uses
|
|
60
114
|
```
|
|
61
|
-
|
|
62
|
-
|
|
115
|
+
zen rag schema index <spec...> Read the documents and write a searchable index.
|
|
116
|
+
zen rag schema search Ask it something. --interactive for a prompt.
|
|
117
|
+
zen rag schema list <what> Every method, type or property. No ranking.
|
|
118
|
+
zen rag schema grep <pattern> Every literal match across the whole index.
|
|
119
|
+
zen rag schema trace <what> Up from a field to the calls that carry it.
|
|
120
|
+
zen rag schema show [id...] Print named nodes, with no search in between.
|
|
121
|
+
zen rag schema stats What is in an index, and what built it.
|
|
122
|
+
```
|
|
63
123
|
|
|
64
124
|
Index once, then ask:
|
|
65
125
|
|
|
@@ -68,69 +128,158 @@ zen rag schema index --embedding openai:text-embedding-3-small ./specs/*.yaml
|
|
|
68
128
|
zen rag schema search --output-property "user billing history"
|
|
69
129
|
zen rag schema search --input-property "password reset token" --format ts
|
|
70
130
|
zen rag schema search --interactive
|
|
71
|
-
zen rag schema stats # what is in the index, and what built it
|
|
72
131
|
zen rag schema show Type:Invoice # a named node, with no search in between
|
|
73
132
|
```
|
|
74
133
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
134
|
+
### Exact API inspection
|
|
135
|
+
|
|
136
|
+
Search finds a starting point; exact inspection answers whether something is in
|
|
137
|
+
the API and shows every matching result:
|
|
79
138
|
|
|
80
139
|
```sh
|
|
81
140
|
zen rag schema list methods --path "*/users*" # every route under /users
|
|
82
141
|
zen rag schema list types --name "*Password*" # every schema so named
|
|
83
142
|
zen rag schema grep password # every literal occurrence
|
|
84
143
|
zen rag schema grep "pass(word|phrase)" --regex
|
|
85
|
-
zen rag schema trace city # which calls can reach the field
|
|
86
144
|
zen rag schema show --method GetCurrentUserInfo --format openapi --exact
|
|
87
145
|
```
|
|
88
146
|
|
|
89
|
-
|
|
90
|
-
|
|
147
|
+
Nothing matching exits 0 - an empty answer is an answer, and here it is a
|
|
148
|
+
trustworthy one: if `grep` finds nothing, the word is not in the description.
|
|
149
|
+
|
|
150
|
+
### Which call carries this field
|
|
151
|
+
|
|
152
|
+
Finding a field is half the job; the other half is which call can reach it, and
|
|
153
|
+
that is a walk up the `$ref`s rather than a match on anything:
|
|
91
154
|
|
|
92
155
|
```sh
|
|
93
|
-
zen rag schema
|
|
94
|
-
zen rag schema grep status --path "/invoices/*" --kind property
|
|
156
|
+
zen rag schema trace city
|
|
95
157
|
```
|
|
96
158
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
159
|
+
```
|
|
160
|
+
Property:Address.city
|
|
161
|
+
GET /users/{userId} getUser output PublicUserProfile.address → Address.city
|
|
162
|
+
```
|
|
100
163
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
164
|
+
No search can be relied on for this - `getUser` and `city` share no word for
|
|
165
|
+
either of them to rank on. A node nothing carries says so, which is worth
|
|
166
|
+
knowing: it means no request in this document will ever carry it.
|
|
104
167
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
matching exits 0 — an empty answer is an answer, and here it is a trustworthy
|
|
108
|
-
one: if `grep` finds nothing, the word is not in the description.
|
|
168
|
+
<details>
|
|
169
|
+
<summary>Every flag</summary>
|
|
109
170
|
|
|
110
|
-
|
|
111
|
-
|
|
171
|
+
`zen rag schema --help` prints the same table.
|
|
172
|
+
|
|
173
|
+
**index**
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
--embedding <ref> Which embedder makes the vectors. Omit it to be shown the choices.
|
|
177
|
+
-o, --out <dir> Where the index goes. Default schema-db, or $ZEN_SCHEMA_DB.
|
|
178
|
+
--batch <n> Texts per embedding request. Default: the model's own cap.
|
|
179
|
+
--dimensions <n> Narrower vectors, if the model allows it.
|
|
180
|
+
--no-sources Do not keep a copy of each document in the index.
|
|
181
|
+
--no-cache Embed everything again, ignoring vectors this machine already has.
|
|
182
|
+
--cache-dir <dir> Keep the vectors somewhere other than the shared cache.
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
**search** - terms, one flag each, all repeatable
|
|
186
|
+
|
|
187
|
+
```
|
|
188
|
+
<text> A bare phrase, the same as --all.
|
|
189
|
+
--all <q> Against everything, unfiltered.
|
|
190
|
+
--method <q> Operations.
|
|
191
|
+
--type <q> Schemas, on the side --direction names.
|
|
192
|
+
--input-type <q> Schemas a call accepts.
|
|
193
|
+
--output-type <q> Schemas a call returns.
|
|
194
|
+
--property <q> Fields and parameters, per --direction.
|
|
195
|
+
--input-property <q> Fields and parameters a call accepts.
|
|
196
|
+
--output-property <q> Fields a call returns.
|
|
197
|
+
--query <json|-> A whole query object; - reads stdin.
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
**search** - filters and shape
|
|
201
|
+
|
|
202
|
+
```
|
|
203
|
+
-d, --dir <dir> Which index. Found from here if unset; see $ZEN_SCHEMA_DB.
|
|
204
|
+
--embedding <ref> Must be the one the index was built with.
|
|
205
|
+
--direction <d> input | output | any. Default any.
|
|
206
|
+
--method-type <t> read_only | read_write | any. Default any.
|
|
207
|
+
--exclude-id <id> Drop a node. Repeatable, as are the three below.
|
|
208
|
+
--exclude-method <name> Drop an operation by name.
|
|
209
|
+
--exclude-type <name> Drop a schema by name.
|
|
210
|
+
--exclude-property <name> Drop a field by name.
|
|
211
|
+
--source <name> Only this document, as stats names it. Repeatable.
|
|
212
|
+
--limit <n> Seeds kept per term. Default 5.
|
|
213
|
+
--max-hops <n> How far apart two hits may be. Default 3.
|
|
214
|
+
--max-nodes <n> Nodes per result. Default 200.
|
|
215
|
+
--format <f> text | mermaid | mermaid-flowchart | ts | openapi.
|
|
216
|
+
--show-source Name the document each operation and schema came from.
|
|
217
|
+
--no-docs Leave the descriptions out.
|
|
218
|
+
--interactive Prompt, search, refine. Needs a terminal.
|
|
219
|
+
--quiet No narration.
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
**list** and **grep**
|
|
223
|
+
|
|
224
|
+
```
|
|
225
|
+
list methods Operations. Filter with --path and --name.
|
|
226
|
+
list types Schemas. Filter with --name.
|
|
227
|
+
list properties Fields and parameters. Filter with --name and --path.
|
|
228
|
+
grep <pattern> Substring over every node; --regex for a regex.
|
|
229
|
+
--regex Read every pattern as a regex, list and grep alike.
|
|
230
|
+
--case-sensitive Match the capitals too.
|
|
231
|
+
--kind <k> grep: method | type | property. Repeatable.
|
|
232
|
+
--name <p> Only nodes whose name matches. Repeatable.
|
|
233
|
+
--path <p> Only what sits on a matching route.
|
|
234
|
+
--ids-only grep: bare ids, to pipe into show.
|
|
235
|
+
--source <name> Only this document, as stats names it.
|
|
236
|
+
--show-source Print which document each row came from.
|
|
237
|
+
--limit <n> Keep at most n; the count still reports them all.
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
A pattern with `*` or `?` is a glob over the whole name; otherwise it is a
|
|
241
|
+
substring, so `--name password` finds `ResetPasswordPayload`. Under `--regex` it
|
|
242
|
+
is a regular expression either way - the only way to say "one of these
|
|
243
|
+
prefixes". A `--path` selects on the route an operation sits on, and on the
|
|
244
|
+
route a parameter's operation sits on; a schema belongs to no one route, so it
|
|
245
|
+
never selects one.
|
|
112
246
|
|
|
113
247
|
```sh
|
|
114
|
-
zen rag schema
|
|
248
|
+
zen rag schema list methods --regex --path "^/(users|teams)/"
|
|
249
|
+
zen rag schema grep status --path "/invoices/*" --kind property
|
|
250
|
+
|
|
251
|
+
# Everything that mentions a token, rendered as TypeScript.
|
|
252
|
+
zen rag schema grep token --ids-only | xargs zen rag schema show --format ts
|
|
115
253
|
```
|
|
116
254
|
|
|
255
|
+
**trace**
|
|
256
|
+
|
|
117
257
|
```
|
|
118
|
-
|
|
119
|
-
|
|
258
|
+
<pattern|id...> What to trace up from. A name, a glob, or a node id.
|
|
259
|
+
--kind <k> method | type | property. Schemas and fields by default.
|
|
260
|
+
--direction <d> Keep only the calls that accept it, or return it.
|
|
261
|
+
--max-hops <n> How far up to walk. Default 8.
|
|
262
|
+
--limit <n> Trace at most n matching nodes.
|
|
263
|
+
--routes <n> Operations printed per node; the count still has them all.
|
|
264
|
+
--ids-only Bare operation ids, one per line, for piping into show.
|
|
265
|
+
--regex Read the pattern as a regex. --case-sensitive too.
|
|
266
|
+
--source <name> Only nodes from this document.
|
|
120
267
|
```
|
|
121
268
|
|
|
122
|
-
|
|
123
|
-
connected piece, but only between the nodes that ranked and only within
|
|
124
|
-
`--max-hops`, and `getUser` and `city` share no word for either of them to rank
|
|
125
|
-
on. `--direction` keeps just the calls that accept it or return it,
|
|
126
|
-
`--kind` narrows what a bare pattern may start from, `--routes` and `--limit`
|
|
127
|
-
cut the printed rows without lying about `found`, and `--ids-only` pipes the
|
|
128
|
-
operations onward. A node nothing carries says so, which is worth knowing: it
|
|
129
|
-
means no request in this document will ever carry it.
|
|
269
|
+
**show**
|
|
130
270
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
271
|
+
```
|
|
272
|
+
<id...> Node ids, e.g. Type:User or Property:User.email.
|
|
273
|
+
--method <name> An operation by name. * to take more. Repeatable.
|
|
274
|
+
--type <name> A schema by name. * to take more. Repeatable.
|
|
275
|
+
--source <name> A whole document, as it was indexed.
|
|
276
|
+
--show-source Name the document each node came from.
|
|
277
|
+
--exact Only what was named, without the neighbours.
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
Search is also a machine interface: every field is a flag, a whole query can
|
|
281
|
+
arrive as one JSON object, and `--json` returns a stable structure for
|
|
282
|
+
automation.
|
|
134
283
|
|
|
135
284
|
```sh
|
|
136
285
|
zen rag schema search --query - --format ts <<'JSON'
|
|
@@ -143,22 +292,34 @@ zen rag schema search --query - --format ts <<'JSON'
|
|
|
143
292
|
JSON
|
|
144
293
|
```
|
|
145
294
|
|
|
146
|
-
|
|
295
|
+
</details>
|
|
296
|
+
|
|
297
|
+
## Documents - `zen rag docs`
|
|
298
|
+
|
|
299
|
+
**Markdown and plain text** - `.md`, `.markdown`, `.txt` and `.text` - named as
|
|
300
|
+
files, directories or globs; hidden directories and `node_modules` are skipped.
|
|
301
|
+
|
|
302
|
+
The answer is the documents themselves: the passages that matched, quoted
|
|
303
|
+
verbatim with their line numbers, and a marker wherever something between two of
|
|
304
|
+
them was left out. Nothing is paraphrased, so the next question can be phrased
|
|
305
|
+
in line numbers and the file can be edited from the answer.
|
|
147
306
|
|
|
148
|
-
|
|
149
|
-
|
|
307
|
+
```
|
|
308
|
+
zen rag docs index <path...> Read the documents and write a searchable index.
|
|
309
|
+
zen rag docs search [text] Ask it something. --interactive for a prompt.
|
|
310
|
+
zen rag docs list <what> Every document, section or table. No ranking.
|
|
311
|
+
zen rag docs grep <pattern> Every matching line, with the section it sits in.
|
|
312
|
+
zen rag docs show <file> A document, a section of one, or a line range.
|
|
313
|
+
zen rag docs stats What is in an index, and what built it.
|
|
314
|
+
```
|
|
150
315
|
|
|
151
316
|
```sh
|
|
152
317
|
zen rag docs index --embedding openai:text-embedding-3-small ./docs
|
|
153
318
|
zen rag docs search "how are rate limits counted"
|
|
154
319
|
```
|
|
155
320
|
|
|
156
|
-
The answer is the documents themselves — the passages that matched, quoted
|
|
157
|
-
verbatim with their line numbers, and a marker wherever something between two
|
|
158
|
-
of them was left out:
|
|
159
|
-
|
|
160
321
|
```
|
|
161
|
-
## acme_4.2.0/api/routing.md
|
|
322
|
+
## acme_4.2.0/api/routing.md - 9 of 148 lines
|
|
162
323
|
|
|
163
324
|
5 | ## Rate limits
|
|
164
325
|
7 | Requests are counted per tenant and rejected past the limit.
|
|
@@ -168,9 +329,11 @@ of them was left out:
|
|
|
168
329
|
27 | | /api/users | 250 | 1m |
|
|
169
330
|
```
|
|
170
331
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
332
|
+
### Narrowing is the interface
|
|
333
|
+
|
|
334
|
+
Nobody finds the paragraph they want on the first ask. The second call is the
|
|
335
|
+
same question asked inside one part of the tree, one heading, or one kind of
|
|
336
|
+
content:
|
|
174
337
|
|
|
175
338
|
```sh
|
|
176
339
|
zen rag docs search --file "acme_4.2.*/api/**" "rate limit for the users route"
|
|
@@ -179,21 +342,29 @@ zen rag docs search --mode text "X-RateLimit-Remaining" # exact wording only
|
|
|
179
342
|
zen rag docs search --interactive # narrow by typing
|
|
180
343
|
```
|
|
181
344
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
covers whatever nests inside it. `--kind` takes `paragraph`, `list`, `table`,
|
|
187
|
-
`table_row`, `code`, `frontmatter` or `html`, for when the answer is a table
|
|
188
|
-
and not the prose around it. `-B/-A` widen each passage, `--max-lines` caps the
|
|
189
|
-
whole answer, `--exclude-id` moves on from what was already seen.
|
|
345
|
+
Two releases of the same file stay apart, because a document is known by its
|
|
346
|
+
path relative to the common root of everything indexed. Tables survive the trip
|
|
347
|
+
whole: a row that matches still arrives with its column names attached, so a
|
|
348
|
+
number is never quoted without the thing it measures.
|
|
190
349
|
|
|
191
|
-
|
|
192
|
-
the column names, and once per row, with the header row travelling alongside so
|
|
193
|
-
the columns are still named wherever a row lands. A row too wide to be one
|
|
194
|
-
chunk is cut into column groups, with the key column repeated in each.
|
|
350
|
+
### One wording per leg
|
|
195
351
|
|
|
196
|
-
|
|
352
|
+
A plain query is asked of two retrievers that read it differently: the full-text
|
|
353
|
+
index wants the words the documents themselves use, and the vector index wants
|
|
354
|
+
the question stated in full. One sentence rarely suits both. `--text-query` and
|
|
355
|
+
`--vector-query` say each outright, and what comes back is fused exactly as it
|
|
356
|
+
would have been:
|
|
357
|
+
|
|
358
|
+
```sh
|
|
359
|
+
zen rag docs search --text-query "Retry-After" \
|
|
360
|
+
--vector-query "how long to wait before trying the request again"
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
Either flag on its own runs that leg alone. They replace the positional text
|
|
364
|
+
rather than narrowing it, and they settle the mode by existing, so `--mode`
|
|
365
|
+
alongside a single wording is refused instead of quietly losing.
|
|
366
|
+
|
|
367
|
+
### Exact document inspection
|
|
197
368
|
|
|
198
369
|
```sh
|
|
199
370
|
zen rag docs list files # every document, and what it holds
|
|
@@ -204,90 +375,149 @@ zen rag docs show api/routing.md --section "Rate limits"
|
|
|
204
375
|
zen rag docs show api/routing.md --lines 40-80
|
|
205
376
|
```
|
|
206
377
|
|
|
207
|
-
`grep`
|
|
208
|
-
|
|
378
|
+
`grep` counts every match, not the top of a list, so unlike a search it can
|
|
379
|
+
answer whether a string appears at all.
|
|
380
|
+
|
|
381
|
+
<details>
|
|
382
|
+
<summary>Every flag</summary>
|
|
209
383
|
|
|
210
|
-
|
|
384
|
+
`zen rag docs --help` prints the same table.
|
|
211
385
|
|
|
212
|
-
|
|
213
|
-
`$ZEN_DOCS_DB` is used if it is set; without that, the nearest index to the
|
|
214
|
-
working directory is found and named on stderr as it is used.
|
|
386
|
+
**index**
|
|
215
387
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
388
|
+
```
|
|
389
|
+
<path...> Files, directories or globs. .md, .markdown, .txt, .text.
|
|
390
|
+
--embedding <ref> Which embedder makes the vectors. Omit it to be shown the choices.
|
|
391
|
+
-o, --out <dir> Where the index goes. Default docs-db, or $ZEN_DOCS_DB.
|
|
392
|
+
--batch <n> Texts per embedding request. Default: the model's own cap.
|
|
393
|
+
--dimensions <n> Narrower vectors, if the model allows it.
|
|
394
|
+
--chunk-tokens <n> Target chunk size. Default 384.
|
|
395
|
+
--no-cache Parse and embed everything again, ignoring what is already kept.
|
|
396
|
+
--cache-dir <dir> Keep the work somewhere other than the shared cache.
|
|
397
|
+
```
|
|
221
398
|
|
|
222
|
-
|
|
223
|
-
refused: the wrong index does not fail, it answers confidently about a
|
|
224
|
-
different API. Name one with `--dir`, or set the environment variable.
|
|
399
|
+
**search**
|
|
225
400
|
|
|
226
|
-
|
|
227
|
-
|
|
401
|
+
```
|
|
402
|
+
<text> What to look for. One question, not a list of terms.
|
|
403
|
+
-d, --dir <dir> Which index. Found from here if unset; see $ZEN_DOCS_DB.
|
|
404
|
+
--embedding <ref> Must be the one the index was built with.
|
|
405
|
+
-f, --file <pattern> Only these documents. Repeatable.
|
|
406
|
+
--exclude-file <pattern> Drop these documents. Repeatable.
|
|
407
|
+
-s, --section <name> Only under this heading, and what nests in it.
|
|
408
|
+
--kind <k> paragraph | list | table | table_row | code | frontmatter | html.
|
|
409
|
+
--mode <m> hybrid | vector | text. Default hybrid.
|
|
410
|
+
--exclude-id <id> Drop a passage already seen. Repeatable.
|
|
411
|
+
--limit <n> Passages kept. Default 8.
|
|
412
|
+
-B, --before <n> Extra lines quoted before each passage.
|
|
413
|
+
-A, --after <n> Extra lines quoted after each passage.
|
|
414
|
+
--max-lines <n> A ceiling on the whole answer. Default 400.
|
|
415
|
+
--no-numbers Quote the lines without their numbers.
|
|
416
|
+
--hits One line per passage instead of the text.
|
|
417
|
+
--interactive Prompt, search, narrow, search again. Needs a terminal.
|
|
418
|
+
--quiet No narration.
|
|
419
|
+
```
|
|
228
420
|
|
|
229
|
-
|
|
421
|
+
**list**, **grep** and **show**
|
|
230
422
|
|
|
231
423
|
```
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
424
|
+
list files Every document, with its size and what it holds.
|
|
425
|
+
list sections Every heading. --depth to stop at a level.
|
|
426
|
+
list tables Every table, with its columns and row count.
|
|
427
|
+
grep <pattern> Every matching line. --regex for a regex.
|
|
428
|
+
show <file> A document name, as list files prints it.
|
|
429
|
+
--file <pattern> Narrow to documents. Repeatable, as everywhere.
|
|
430
|
+
--section <name> Narrow to a heading and what nests in it.
|
|
431
|
+
--lines <from-to> show: just those lines, e.g. --lines 40-80.
|
|
432
|
+
--regex Read every pattern as a regex.
|
|
433
|
+
--case-sensitive Match the capitals too.
|
|
434
|
+
--no-numbers show: without the line-number gutter.
|
|
435
|
+
--limit <n> Keep at most n; the count still reports them all.
|
|
239
436
|
```
|
|
240
437
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
`--
|
|
245
|
-
|
|
438
|
+
A `--file` pattern is a glob when it has `*` or `?` and a substring otherwise,
|
|
439
|
+
matched against the document's name relative to what was indexed - so
|
|
440
|
+
`--file "acme_4.2*/api/**"` is one release and `--file routing` is a word.
|
|
441
|
+
`--section` takes a heading title and covers whatever nests inside it. `--kind`
|
|
442
|
+
is for when the answer is a table and not the prose around it.
|
|
246
443
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
444
|
+
</details>
|
|
445
|
+
|
|
446
|
+
## Which index gets read
|
|
447
|
+
|
|
448
|
+
Indexes are discovered from the working directory, so commands and agents use
|
|
449
|
+
the relevant project index without repeating its path. Use `-d, --dir` to choose
|
|
450
|
+
one explicitly; `$ZEN_SCHEMA_DB` and `$ZEN_DOCS_DB` set a project-wide default.
|
|
451
|
+
|
|
452
|
+
When more than one index could apply, the command asks for an explicit choice
|
|
453
|
+
rather than searching the wrong project.
|
|
454
|
+
|
|
455
|
+
<details>
|
|
456
|
+
<summary>Index discovery</summary>
|
|
457
|
+
|
|
458
|
+
Document and schema indexes are discovered independently, so both can sit in a
|
|
459
|
+
project tree. A new index defaults to `docs-db` or `schema-db`, but any directory
|
|
460
|
+
can be used with `--out`.
|
|
461
|
+
|
|
462
|
+
</details>
|
|
463
|
+
|
|
464
|
+
## Portable, source-faithful indexes
|
|
465
|
+
|
|
466
|
+
An index is a self-contained project asset. It can be moved, committed or
|
|
467
|
+
supplied to an agent environment without depending on the paths from which it
|
|
468
|
+
was built. Schema indexes preserve the connected API surface across external
|
|
469
|
+
references; document indexes preserve the source material used for retrieval.
|
|
470
|
+
|
|
471
|
+
For documents, search results are always grounded in the original text, with
|
|
472
|
+
source names and line ranges available for inspection and follow-up work.
|
|
473
|
+
|
|
474
|
+
<details>
|
|
475
|
+
<summary>Built for project workflows</summary>
|
|
476
|
+
|
|
477
|
+
Indexes report their contents and build progress, prevent conflicting builds,
|
|
478
|
+
and can be inspected with `stats`. Use `--no-sources` for schema indexes when
|
|
479
|
+
you do not need to retain the source material alongside the index.
|
|
480
|
+
|
|
481
|
+
</details>
|
|
482
|
+
|
|
483
|
+
## Configure semantic retrieval
|
|
484
|
+
|
|
485
|
+
Choose an embedding provider with `--embedding <ref>` when creating an index.
|
|
486
|
+
Run `index` without the flag to see the providers available in your environment.
|
|
257
487
|
|
|
258
488
|
```sh
|
|
259
|
-
|
|
260
|
-
zen rag schema
|
|
489
|
+
zen rag docs index --embedding openai:text-embedding-3-small ./docs
|
|
490
|
+
zen rag schema index --embedding openai:text-embedding-3-small ./specs
|
|
261
491
|
```
|
|
262
492
|
|
|
263
|
-
|
|
493
|
+
The selected provider stays associated with the index, ensuring search uses a
|
|
494
|
+
compatible representation of its content.
|
|
264
495
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
zen rag docs show <file> A document, a section of one, or a line range.
|
|
271
|
-
zen rag docs stats What is in an index, and what built it.
|
|
272
|
-
```
|
|
496
|
+
### Efficient rebuilds
|
|
497
|
+
|
|
498
|
+
Rebuild an index as content changes without repeating work for unchanged files
|
|
499
|
+
or passages. Interrupted builds resume, and shared content is reused across
|
|
500
|
+
indexes to keep large documentation and schema collections practical to update.
|
|
273
501
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
`-B/--before`, `-A/--after` and `--max-lines`, and moved along by
|
|
277
|
-
`--exclude-id`. `list` and `grep` share `--file`, `--section`, `--regex`,
|
|
278
|
-
`--case-sensitive` and `--limit`.
|
|
502
|
+
Use [`zen cache`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md)
|
|
503
|
+
to inspect and manage reusable indexing data when needed.
|
|
279
504
|
|
|
280
|
-
## From an
|
|
505
|
+
## From code - an index is a toolset
|
|
506
|
+
|
|
507
|
+
Open an index in process to expose the same retrieval capabilities as tools an
|
|
508
|
+
agent can call:
|
|
281
509
|
|
|
282
510
|
```ts
|
|
283
511
|
import { createEmbedder, loadProject } from '@zenera/neo';
|
|
284
|
-
import { SchemaIndex, schemaTools } from '@zenera/rag';
|
|
512
|
+
import { docs, SchemaIndex, schemaTools } from '@zenera/rag';
|
|
513
|
+
|
|
514
|
+
const embedder = createEmbedder('openai:text-embedding-3-small');
|
|
515
|
+
const api = await SchemaIndex.open('./schema-db', embedder);
|
|
516
|
+
const handbook = await docs.DocsIndex.open('./docs-db', embedder);
|
|
285
517
|
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
);
|
|
290
|
-
const project = await loadProject('./my-project', { tools: schemaTools(index) });
|
|
518
|
+
const project = await loadProject('./my-project', {
|
|
519
|
+
tools: [...schemaTools(api), ...docs.docsTools(handbook)],
|
|
520
|
+
});
|
|
291
521
|
```
|
|
292
522
|
|
|
293
523
|
Six tools in the group `schema`, selectable as `schema:*`:
|
|
@@ -296,163 +526,39 @@ Six tools in the group `schema`, selectable as `schema:*`:
|
|
|
296
526
|
| -------------------------- | ------------------------------------------------------- |
|
|
297
527
|
| `search_api` | the connected piece of the API that matches an intent |
|
|
298
528
|
| `describe_types` | named schemas as declarations that compile on their own |
|
|
299
|
-
| `find_types_with_property` | which types have a field of this name
|
|
529
|
+
| `find_types_with_property` | which types have a field of this name - no search |
|
|
300
530
|
| `list_api` | the shape of the API: methods, types or fields |
|
|
301
|
-
| `grep_api` | every literal occurrence of a string
|
|
531
|
+
| `grep_api` | every literal occurrence of a string - no search |
|
|
302
532
|
| `trace_api` | the operations that carry a given field or schema |
|
|
303
533
|
|
|
304
|
-
Only the first
|
|
305
|
-
told "no results" by a vector search has learned nothing: a ranking returns the
|
|
306
|
-
top of a list, so an empty answer and an absent thing look identical.
|
|
307
|
-
`find_types_with_property` is the one for the repair loop — when `tsc` says
|
|
308
|
-
`'password' does not exist in type 'PublicUserProfile'`, the model does not
|
|
309
|
-
need the word explained again, it needs the list of types that have one.
|
|
310
|
-
`grep_api` is the same instinct widened to the whole description, and
|
|
311
|
-
`trace_api` is the step after both: a field is of no use until the call that
|
|
312
|
-
carries it is known.
|
|
534
|
+
Only the first ranks. `find_types_with_property` is the one for the repair loop
|
|
313
535
|
|
|
314
|
-
|
|
536
|
+
- when `tsc` says `'password' does not exist in type 'PublicUserProfile'`, the
|
|
537
|
+
model does not need the word explained again, it needs the list of types that
|
|
538
|
+
have one. `grep_api` is the same instinct widened to the whole description, and
|
|
539
|
+
`trace_api` is the step after both.
|
|
315
540
|
|
|
316
|
-
|
|
317
|
-
import { docs } from '@zenera/rag';
|
|
318
|
-
|
|
319
|
-
const index = await docs.DocsIndex.open('./docs-db', embedder);
|
|
320
|
-
const project = await loadProject('./my-project', { tools: docs.docsTools(index) });
|
|
321
|
-
```
|
|
541
|
+
Four tools in the group `docs`, selectable as `docs:*`:
|
|
322
542
|
|
|
323
543
|
| Tool | For |
|
|
324
544
|
| ------------- | ------------------------------------------------------------ |
|
|
325
545
|
| `search_docs` | the passages that match, quoted with their line numbers |
|
|
326
|
-
| `list_docs` | the documents, their headings, or their tables
|
|
327
|
-
| `grep_docs` | every matching line, counted in full
|
|
546
|
+
| `list_docs` | the documents, their headings, or their tables - no search |
|
|
547
|
+
| `grep_docs` | every matching line, counted in full - no search |
|
|
328
548
|
| `read_docs` | a section or a line range, verbatim and with nothing omitted |
|
|
329
549
|
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
passage came from.
|
|
335
|
-
|
|
336
|
-
## What an index is
|
|
337
|
-
|
|
338
|
-
```
|
|
339
|
-
schema-db/
|
|
340
|
-
├── README.md what this index holds — a live progress report while it builds
|
|
341
|
-
├── manifest.json written last — its absence means "not indexed"
|
|
342
|
-
├── graph.json topology and light attributes, read whole
|
|
343
|
-
├── schemas.json the raw schemas, read the first time one is needed in full
|
|
344
|
-
├── operations.json likewise, for the OpenAPI subset
|
|
345
|
-
├── sources/ the documents themselves, bundled, exactly as indexed
|
|
346
|
-
└── lance/ one table: a row per node, one text column, one vector
|
|
347
|
-
```
|
|
348
|
-
|
|
349
|
-
The manifest records which embedder made the vectors, and a search with a
|
|
350
|
-
different one is refused rather than answered with noise.
|
|
351
|
-
|
|
352
|
-
A document index is the same idea with a different middle:
|
|
353
|
-
|
|
354
|
-
```
|
|
355
|
-
docs-db/
|
|
356
|
-
├── README.md what this index holds — a live progress report while it builds
|
|
357
|
-
├── manifest.json written last — its absence means "not indexed"
|
|
358
|
-
├── outline.json every heading and table, with the lines they cover
|
|
359
|
-
├── sources/ the documents themselves, verbatim — where the quotes come from
|
|
360
|
-
└── lance/ one table: a row per chunk, two texts, one vector
|
|
361
|
-
```
|
|
362
|
-
|
|
363
|
-
There the copies are not a record but the answer: a search returns line ranges
|
|
364
|
-
and the lines are read back out of `sources/`, so what is quoted is the document
|
|
365
|
-
rather than a reconstruction of it.
|
|
366
|
-
|
|
367
|
-
Indexing a large document is minutes of silence, so the directory says what is
|
|
368
|
-
happening to it. `README.md` appears first as a progress report — the documents,
|
|
369
|
-
the embedder, the step, how many entities have been embedded of how many, and
|
|
370
|
-
how long it has been going — rewritten at most every five seconds, and replaced
|
|
371
|
-
on completion by a description of what the index turned out to hold. A build
|
|
372
|
-
that dies leaves it saying so. While one runs, `.lock` names the process; a
|
|
373
|
-
second build of the same directory is refused, unless the lock is stale.
|
|
374
|
-
|
|
375
|
-
**An index is one portable thing.** Nothing in it names a path outside itself:
|
|
376
|
-
a document is known by a short name taken from its filename, and the document
|
|
377
|
-
itself is copied into `sources/` as it was indexed — bundled, so every external
|
|
378
|
-
`$ref` is already resolved and the copy stands alone. The directory can be moved,
|
|
379
|
-
committed or shipped whole and still says what it is made of. This matters
|
|
380
|
-
because a project's `assets/` is mounted at `/assets` inside an agent's sandbox,
|
|
381
|
-
so an index built here is read under a name this machine never sees. `--no-sources`
|
|
382
|
-
leaves the copies out, for an index that will never travel.
|
|
383
|
-
|
|
384
|
-
The copies are a record, not an input: rebuilding reads the files you name, not
|
|
385
|
-
the ones in `sources/`. A **document** index has no `--no-sources`, because
|
|
386
|
-
there the copies are what every quoted line is read from.
|
|
387
|
-
|
|
388
|
-
**Vectors are paid for once.** Embedding is the whole cost of a build — the
|
|
389
|
-
reading and the writing are milliseconds, the round trips are minutes, and they
|
|
390
|
-
are the only part anyone is billed for. Almost none of it is new work: editing
|
|
391
|
-
one paragraph re-embeds every other one unchanged, and a build killed at 90%
|
|
392
|
-
starts again from nothing. So the vectors are kept in this machine's shared
|
|
393
|
-
cache, `~/.zenera/neo/cache/vectors/`, keyed by a hash of the exact text that
|
|
394
|
-
produced them together with the model that made them. Re-indexing an unchanged
|
|
395
|
-
corpus embeds nothing; re-indexing after an edit pays for the chunks that
|
|
396
|
-
changed.
|
|
397
|
-
|
|
398
|
-
The store is the machine's rather than the index's, which is the point: the same
|
|
399
|
-
corpus indexed into a second directory costs nothing the second time, and two
|
|
400
|
-
projects that quote the same handbook pay for it once between them.
|
|
401
|
-
|
|
402
|
-
The key is the text, never the chunk's position — inserting a sentence at the
|
|
403
|
-
top of a document shifts every later chunk's ordinal without changing a word of
|
|
404
|
-
it. Two documents that share a paragraph share its vector. Vectors another model
|
|
405
|
-
made are not evicted, they are simply never asked for, because the model is part
|
|
406
|
-
of the key. It is a cache, so every failure in it is a miss and nothing more.
|
|
407
|
-
`--no-cache` ignores it; `--cache-dir <dir>` keeps the work somewhere else.
|
|
408
|
-
|
|
409
|
-
Nothing is ever evicted by a build: what a corpus stops referring to is still
|
|
410
|
-
work somebody paid for. Getting rid of it is [`zen cache`](../cli/README.md)'s
|
|
411
|
-
job — `zen cache ls` to see what is there, `zen cache prune --older-than 30d` to
|
|
412
|
-
drop what has gone unread.
|
|
413
|
-
|
|
414
|
-
**Parsing is spread across cores, and also cached.** With the vectors cached,
|
|
415
|
-
parsing is what is left: reading a document, cutting it into chunks, and
|
|
416
|
-
recording its shape. It is CPU-bound and every document is independent of every
|
|
417
|
-
other, so a **document** index does it on a pool of worker threads — one per
|
|
418
|
-
core, less the one running the build. The pool is skipped for a handful of
|
|
419
|
-
documents, where starting the threads costs more than it saves.
|
|
420
|
-
|
|
421
|
-
The result is cached the same way the vectors are, under
|
|
422
|
-
`~/.zenera/neo/cache/docs-parse/`. The key covers the file's bytes, the name
|
|
423
|
-
stamped on its chunks, and every setting that decides where a chunk ends, so
|
|
424
|
-
changing `--chunk-tokens` misses on all of it and editing one file misses on one
|
|
425
|
-
file. Both caches are governed by the same `--no-cache` and the same
|
|
426
|
-
`--cache-dir`, and both fall back to doing the work when anything about them
|
|
427
|
-
looks wrong.
|
|
428
|
-
|
|
429
|
-
## Notes
|
|
430
|
-
|
|
431
|
-
- Documents are **bundled, not dereferenced**: `#/components/schemas/User` is
|
|
432
|
-
an edge and `User` is a node id. Swagger 2.0, OpenAPI 3.0 and 3.1 are
|
|
433
|
-
converted to 2020-12 on the way in, `discriminator` included — it is what
|
|
434
|
-
turns a `oneOf` into a TypeScript tagged union the compiler can narrow.
|
|
435
|
-
- A **query parameter is a property**, like any field in a body. Nobody should
|
|
436
|
-
have to know in advance which one `page_size` is.
|
|
437
|
-
- Every type carries a **direction** — `input`, `output` or `both` — worked out
|
|
438
|
-
by propagating from the operations through composition, so a type used on both
|
|
439
|
-
sides is honestly both rather than whichever side was read last.
|
|
440
|
-
- Filters reaching the store are **closed enums only**. Exclusion lists are
|
|
441
|
-
applied in JavaScript afterwards, so nothing a model wrote ever reaches a SQL
|
|
442
|
-
predicate.
|
|
443
|
-
- A document chunk knows **exactly which lines** of the original it stands for,
|
|
444
|
-
headings and table headers included. That is what makes an answer quotable,
|
|
445
|
-
and what lets the next question be phrased in line numbers.
|
|
446
|
-
- Plain text is read as paragraphs and given **no invented headings**: a `.txt`
|
|
447
|
-
file has one section, which is the document.
|
|
550
|
+
`search_docs` is the way in when the question is vague; `grep_docs` is how "it
|
|
551
|
+
is not in here" can actually be concluded. Every answer carries line numbers and
|
|
552
|
+
`read_docs` takes them, which is the loop the subject exists for: find the
|
|
553
|
+
passage, read around it, then edit the file the passage came from.
|
|
448
554
|
|
|
449
555
|
## The rest of the family
|
|
450
556
|
|
|
451
557
|
| Package | What it is |
|
|
452
558
|
| ----------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
|
|
453
|
-
| [`@zenera/cli`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md) | `zen`
|
|
454
|
-
| [`@zenera/neo`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/neo/README.md) | the runtime
|
|
455
|
-
| [`@zenera/faker`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/faker/README.md) | `zen faker`
|
|
559
|
+
| [`@zenera/cli`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/cli/README.md) | `zen` - agent projects on the command line |
|
|
560
|
+
| [`@zenera/neo`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/neo/README.md) | the runtime - agents, models, tools, skills, memory |
|
|
561
|
+
| [`@zenera/faker`](https://github.com/andreyryabov/ZeneraNeo/blob/main/packages/faker/README.md) | `zen faker` - a mock API from the same kind of document |
|
|
456
562
|
|
|
457
563
|
## License
|
|
458
564
|
|
package/dist/command.js
CHANGED
|
@@ -16,8 +16,9 @@ import { command as schema } from "./schema/command.js";
|
|
|
16
16
|
// to mean one of them, and whichever was chosen would be wrong for the other
|
|
17
17
|
// forever.
|
|
18
18
|
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
19
|
+
// The outer frame settles `--help`, then tells an external command it saw it.
|
|
20
|
+
// That keeps help syntax uniform while still letting this dispatcher name the
|
|
21
|
+
// subject page.
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
const SUBJECTS = { schema, docs };
|
|
23
24
|
const USAGE = 'zen rag <subject> <command> [args...]';
|
|
@@ -28,15 +29,15 @@ export const command = {
|
|
|
28
29
|
'Subjects',
|
|
29
30
|
...table(Object.entries(SUBJECTS).map(([name, sub]) => [` ${name}`, dim(sub.summary)])),
|
|
30
31
|
'',
|
|
31
|
-
...Object.keys(SUBJECTS).map((name) => dim(` ${cyan(`zen rag
|
|
32
|
+
...Object.keys(SUBJECTS).map((name) => dim(` ${cyan(`zen rag ${name} --help`)} — its commands, flags and examples`)),
|
|
32
33
|
'',
|
|
33
34
|
dim(`Credentials come from the ${cyan('zen')} keyring — try ${cyan('zen key ls')}.`),
|
|
34
35
|
],
|
|
36
|
+
help(ctx) {
|
|
37
|
+
return help(ctx.args[0]);
|
|
38
|
+
},
|
|
35
39
|
async run(ctx) {
|
|
36
40
|
const [subject, ...rest] = ctx.args;
|
|
37
|
-
if (subject === 'help') {
|
|
38
|
-
return help(rest[0]);
|
|
39
|
-
}
|
|
40
41
|
const chosen = subject ? SUBJECTS[subject] : undefined;
|
|
41
42
|
if (!chosen) {
|
|
42
43
|
throw usageError(subject ? `unknown subject "${subject}"` : 'no subject given', `expected ${Object.keys(SUBJECTS).join(' or ')} — ${USAGE}`);
|
|
@@ -46,10 +47,16 @@ export const command = {
|
|
|
46
47
|
};
|
|
47
48
|
/** Laid out as `zen help <command>` lays out this one, so the two pages match. */
|
|
48
49
|
function help(subject) {
|
|
49
|
-
|
|
50
|
+
if (!subject) {
|
|
51
|
+
return page(command);
|
|
52
|
+
}
|
|
53
|
+
const chosen = SUBJECTS[subject];
|
|
50
54
|
if (!chosen) {
|
|
51
|
-
throw usageError(
|
|
55
|
+
throw usageError(`unknown subject "${subject}"`, `expected ${Object.keys(SUBJECTS).join(' or ')} — zen rag <subject> --help`);
|
|
52
56
|
}
|
|
57
|
+
return page(chosen);
|
|
58
|
+
}
|
|
59
|
+
function page(chosen) {
|
|
53
60
|
write(bold(chosen.usage));
|
|
54
61
|
write(`\n ${chosen.summary}`);
|
|
55
62
|
if (chosen.details?.length) {
|
package/dist/docs/command.js
CHANGED
|
@@ -14,7 +14,7 @@ import { DOC_EXTENSIONS } from "./load.js";
|
|
|
14
14
|
import { grepLines, listFiles, listSections, listTables, readRange, readSection, } from "./lookup.js";
|
|
15
15
|
import { MATCH_HEADERS, matchRows, renderAssembly } from "./render.js";
|
|
16
16
|
import { repl } from "./repl.js";
|
|
17
|
-
import { DEFAULT_LIMIT, DocsIndex, SEARCH_MODES, } from "./search.js";
|
|
17
|
+
import { DEFAULT_LIMIT, DocsIndex, legsOf, SEARCH_MODES, } from "./search.js";
|
|
18
18
|
const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = DOCS_INDEX;
|
|
19
19
|
// ---------------------------------------------------------------------------
|
|
20
20
|
// zen rag docs — a pile of documents, as something to search
|
|
@@ -41,7 +41,7 @@ const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = DOCS_INDEX;
|
|
|
41
41
|
// ---------------------------------------------------------------------------
|
|
42
42
|
const USAGE = 'zen rag docs <index|search|list|grep|show|stats> [args...]';
|
|
43
43
|
const INDEX_USAGE = 'zen rag docs index --embedding <ref> [--out <dir>] <path...>';
|
|
44
|
-
const SEARCH_USAGE = 'zen rag docs search [--dir <dir>] [query...]';
|
|
44
|
+
const SEARCH_USAGE = 'zen rag docs search [--dir <dir>] [query... | --text-query <text> --vector-query <text>]';
|
|
45
45
|
const LIST_USAGE = 'zen rag docs list <files|sections|tables> [--dir <dir>]';
|
|
46
46
|
const GREP_USAGE = 'zen rag docs grep <pattern> [--dir <dir>]';
|
|
47
47
|
const SHOW_USAGE = 'zen rag docs show <file> [--section <name>] [--lines <from-to>]';
|
|
@@ -51,12 +51,27 @@ export const command = {
|
|
|
51
51
|
details: [
|
|
52
52
|
'Commands',
|
|
53
53
|
...table([
|
|
54
|
-
[
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
[
|
|
59
|
-
|
|
54
|
+
[
|
|
55
|
+
' index <path...>',
|
|
56
|
+
dim('Build or refresh an index before searching changed documents.'),
|
|
57
|
+
],
|
|
58
|
+
[
|
|
59
|
+
' search [text]',
|
|
60
|
+
dim('Rank relevant passages with hybrid full-text and vector search.'),
|
|
61
|
+
],
|
|
62
|
+
[
|
|
63
|
+
' list <what>',
|
|
64
|
+
dim('Browse every indexed file, heading or table when you need an inventory.'),
|
|
65
|
+
],
|
|
66
|
+
[
|
|
67
|
+
' grep <pattern>',
|
|
68
|
+
dim('Find every literal or regex line match; use it to prove whether text exists.'),
|
|
69
|
+
],
|
|
70
|
+
[
|
|
71
|
+
' show <file>',
|
|
72
|
+
dim('Read a document, heading or line range verbatim after finding its location.'),
|
|
73
|
+
],
|
|
74
|
+
[' stats', dim('Inspect the index contents, size and embedding configuration.')],
|
|
60
75
|
]),
|
|
61
76
|
'',
|
|
62
77
|
'Index',
|
|
@@ -89,8 +104,14 @@ export const command = {
|
|
|
89
104
|
'Search',
|
|
90
105
|
...table([
|
|
91
106
|
[' <text>', dim('What to look for. One question, not a list of terms.')],
|
|
107
|
+
[
|
|
108
|
+
' --interactive',
|
|
109
|
+
dim('Keep a docs> prompt open to search again and refine file, section or kind.'),
|
|
110
|
+
],
|
|
92
111
|
[' -d, --dir <dir>', dim(`Which index. Found from here if unset; see ${DIR_ENV}.`)],
|
|
93
112
|
[' --embedding <ref>', dim('Must be the one the index was built with.')],
|
|
113
|
+
[' --text-query <text>', dim('Wording for the full-text leg, instead of <text>.')],
|
|
114
|
+
[' --vector-query <text>', dim('Wording for the vector leg, instead of <text>.')],
|
|
94
115
|
[' -f, --file <pattern>', dim('Only these documents. Repeatable.')],
|
|
95
116
|
[' --exclude-file <pattern>', dim('Drop these documents. Repeatable.')],
|
|
96
117
|
[' -s, --section <name>', dim('Only under this heading, and what nests in it.')],
|
|
@@ -106,10 +127,21 @@ export const command = {
|
|
|
106
127
|
],
|
|
107
128
|
[' --no-numbers', dim('Quote the lines without their numbers.')],
|
|
108
129
|
[' --hits', dim('One line per passage instead of the text.')],
|
|
109
|
-
[' --interactive', dim('Prompt, search, narrow, search again. Needs a terminal.')],
|
|
110
130
|
[' --quiet', dim('No narration.')],
|
|
111
131
|
]),
|
|
112
132
|
'',
|
|
133
|
+
dim(' Default hybrid fuses full-text matches with vector similarity. Use --mode text'),
|
|
134
|
+
dim(' or --mode vector to use just one of them.'),
|
|
135
|
+
'',
|
|
136
|
+
dim(' The two legs read a question differently: the full-text one wants the words the'),
|
|
137
|
+
dim(' documents use, the vector one wants the meaning. Say each outright with'),
|
|
138
|
+
dim(' --text-query and --vector-query, and what comes back is fused as usual. Either'),
|
|
139
|
+
dim(' one alone runs that leg alone, so they replace <text> and --mode rather than'),
|
|
140
|
+
dim(' joining them.'),
|
|
141
|
+
'',
|
|
142
|
+
dim(` ${cyan('zen rag docs search --text-query "retry after header" \\')}`),
|
|
143
|
+
dim(` ${cyan(' --vector-query "how long to wait before trying again"')}`),
|
|
144
|
+
'',
|
|
113
145
|
dim(' A --file pattern with * or ? is a glob over the whole document name,'),
|
|
114
146
|
dim(' otherwise a substring. Names are relative to what was indexed, so'),
|
|
115
147
|
dim(' --file "acme_4.2*/api/**" is one release and --file routing is a word.'),
|
|
@@ -282,6 +314,8 @@ async function search(args, ctx) {
|
|
|
282
314
|
const { values, positionals } = parse(args, {
|
|
283
315
|
dir: { type: 'string', short: 'd' },
|
|
284
316
|
embedding: { type: 'string' },
|
|
317
|
+
'text-query': { type: 'string' },
|
|
318
|
+
'vector-query': { type: 'string' },
|
|
285
319
|
file: { ...MANY, short: 'f' },
|
|
286
320
|
'exclude-file': MANY,
|
|
287
321
|
section: { ...MANY, short: 's' },
|
|
@@ -300,6 +334,8 @@ async function search(args, ctx) {
|
|
|
300
334
|
const text = positionals.join(' ').trim();
|
|
301
335
|
const query = {
|
|
302
336
|
query: text || undefined,
|
|
337
|
+
textQuery: values['text-query'],
|
|
338
|
+
vectorQuery: values['vector-query'],
|
|
303
339
|
mode: modeOf(values.mode),
|
|
304
340
|
files: values.file,
|
|
305
341
|
exclude_files: values['exclude-file'],
|
|
@@ -313,14 +349,26 @@ async function search(args, ctx) {
|
|
|
313
349
|
after: values.after ? count(values.after, '--after', 0) : undefined,
|
|
314
350
|
maxLines: values['max-lines'] ? count(values['max-lines'], '--max-lines') : undefined,
|
|
315
351
|
};
|
|
352
|
+
const perLeg = Boolean(values['text-query'] ?? values['vector-query']);
|
|
316
353
|
// Everything that can be wrong about the invocation is settled before a
|
|
317
354
|
// credential is asked for, so a typo is a usage error and not a login.
|
|
318
355
|
if (values.interactive && !isInteractive()) {
|
|
319
356
|
throw usageError('--interactive needs a terminal', SEARCH_USAGE);
|
|
320
357
|
}
|
|
321
|
-
|
|
358
|
+
for (const flag of ['text-query', 'vector-query']) {
|
|
359
|
+
if (values[flag] !== undefined && !values[flag].trim()) {
|
|
360
|
+
throw usageError(`--${flag} is empty`, SEARCH_USAGE);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (perLeg && values.interactive) {
|
|
364
|
+
// The prompt narrows one question by typing at it; there is no second
|
|
365
|
+
// line to type the other leg's wording on.
|
|
366
|
+
throw usageError('--interactive searches a single query', SEARCH_USAGE);
|
|
367
|
+
}
|
|
368
|
+
if (!values.interactive && !text && !perLeg) {
|
|
322
369
|
throw usageError('nothing to search for', SEARCH_USAGE);
|
|
323
370
|
}
|
|
371
|
+
await patterned(() => legsOf(query));
|
|
324
372
|
const dir = indexDir(ctx, values.dir);
|
|
325
373
|
const manifest = await readManifest(dir);
|
|
326
374
|
const ref = values.embedding ?? manifest.embedding.ref;
|
package/dist/docs/search.d.ts
CHANGED
|
@@ -5,6 +5,16 @@ export type SearchMode = 'hybrid' | 'vector' | 'text';
|
|
|
5
5
|
export declare const SEARCH_MODES: readonly SearchMode[];
|
|
6
6
|
export interface DocsQuery {
|
|
7
7
|
query?: string;
|
|
8
|
+
/**
|
|
9
|
+
* One wording per leg, for a caller that can write both. `query` asks the
|
|
10
|
+
* same sentence of two retrievers that read it differently: the full-text
|
|
11
|
+
* index wants the words the corpus uses, and the vector index wants the
|
|
12
|
+
* meaning. These say each half outright, and are fused exactly as `query`
|
|
13
|
+
* would have been. They replace `query` rather than narrowing it, so a
|
|
14
|
+
* request carries one account of what is wanted and not two.
|
|
15
|
+
*/
|
|
16
|
+
textQuery?: string;
|
|
17
|
+
vectorQuery?: string;
|
|
8
18
|
mode?: SearchMode;
|
|
9
19
|
/** patterns over the document name: a glob, a substring, or a regex */
|
|
10
20
|
files?: readonly string[];
|
|
@@ -89,4 +99,21 @@ export declare class DocsIndex {
|
|
|
89
99
|
* prefix of `doc/sec:10` and they are different sections.
|
|
90
100
|
*/
|
|
91
101
|
export declare const under: (path: string, sections: readonly string[]) => boolean;
|
|
102
|
+
/** What each retriever is asked, and the mode that describes the pair. */
|
|
103
|
+
export interface Legs {
|
|
104
|
+
text?: string;
|
|
105
|
+
vector?: string;
|
|
106
|
+
mode: SearchMode;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Which leg reads what, decided in one place because three surfaces ask it.
|
|
110
|
+
*
|
|
111
|
+
* A plain `query` keeps its old bargain with `mode`: hybrid runs both halves on
|
|
112
|
+
* the one sentence, and the other two shut a half off. A per-leg wording says
|
|
113
|
+
* the same thing by existing — one of them runs one half, both of them run both
|
|
114
|
+
* — so a `mode` alongside it is either a restatement or a contradiction, and
|
|
115
|
+
* neither is worth honouring. `hybrid` is let through for the pair alone, since
|
|
116
|
+
* that is the one spelling that agrees with what was asked.
|
|
117
|
+
*/
|
|
118
|
+
export declare function legsOf(query: DocsQuery): Legs;
|
|
92
119
|
//# sourceMappingURL=search.d.ts.map
|
package/dist/docs/search.js
CHANGED
|
@@ -102,8 +102,8 @@ export class DocsIndex {
|
|
|
102
102
|
this.#store.close();
|
|
103
103
|
}
|
|
104
104
|
async search(query, signal) {
|
|
105
|
-
const
|
|
106
|
-
const
|
|
105
|
+
const legs = legsOf(query);
|
|
106
|
+
const mode = legs.mode;
|
|
107
107
|
const limit = Math.max(1, query.limit ?? DEFAULT_LIMIT);
|
|
108
108
|
const files = this.resolveFiles(query.files, query.exclude_files);
|
|
109
109
|
const headings = this.resolveSections(query.section ?? [], files);
|
|
@@ -111,14 +111,14 @@ export class DocsIndex {
|
|
|
111
111
|
// Two ways to ask for nothing: a pattern no document answers to, and a
|
|
112
112
|
// section no document has. Both are told, not silently widened.
|
|
113
113
|
const impossible = files.length === 0 || ((query.section?.length ?? 0) > 0 && sections.length === 0);
|
|
114
|
-
if (impossible || !text) {
|
|
114
|
+
if (impossible || (!legs.text && !legs.vector)) {
|
|
115
115
|
return { matches: [], mode, files, sections, considered: 0 };
|
|
116
116
|
}
|
|
117
117
|
const filter = this.#filter(query, files, sections);
|
|
118
118
|
const excluded = new Set(query.exclude_ids ?? []);
|
|
119
119
|
const fetch = Math.min(limit * OVERFETCH, MAX_FETCH) + excluded.size;
|
|
120
|
-
const vector =
|
|
121
|
-
const lexical =
|
|
120
|
+
const vector = legs.vector ? await this.#nearest(legs.vector, filter, fetch, signal) : [];
|
|
121
|
+
const lexical = legs.text ? await this.#store.matching(legs.text, filter, fetch) : [];
|
|
122
122
|
const fused = fuse(vector, lexical).filter((match) => !excluded.has(match.id) &&
|
|
123
123
|
files.includes(match.path) &&
|
|
124
124
|
under(match.structurePath, sections));
|
|
@@ -167,6 +167,43 @@ const anyMatch = (patterns) => {
|
|
|
167
167
|
*/
|
|
168
168
|
export const under = (path, sections) => sections.length === 0 ||
|
|
169
169
|
sections.some((section) => path === section || path.startsWith(`${section}/`));
|
|
170
|
+
/**
|
|
171
|
+
* Which leg reads what, decided in one place because three surfaces ask it.
|
|
172
|
+
*
|
|
173
|
+
* A plain `query` keeps its old bargain with `mode`: hybrid runs both halves on
|
|
174
|
+
* the one sentence, and the other two shut a half off. A per-leg wording says
|
|
175
|
+
* the same thing by existing — one of them runs one half, both of them run both
|
|
176
|
+
* — so a `mode` alongside it is either a restatement or a contradiction, and
|
|
177
|
+
* neither is worth honouring. `hybrid` is let through for the pair alone, since
|
|
178
|
+
* that is the one spelling that agrees with what was asked.
|
|
179
|
+
*/
|
|
180
|
+
export function legsOf(query) {
|
|
181
|
+
const plain = (query.query ?? '').trim();
|
|
182
|
+
const text = (query.textQuery ?? '').trim();
|
|
183
|
+
const vector = (query.vectorQuery ?? '').trim();
|
|
184
|
+
if (!text && !vector) {
|
|
185
|
+
const mode = query.mode ?? 'hybrid';
|
|
186
|
+
return {
|
|
187
|
+
text: mode === 'vector' ? undefined : plain || undefined,
|
|
188
|
+
vector: mode === 'text' ? undefined : plain || undefined,
|
|
189
|
+
mode,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
if (plain) {
|
|
193
|
+
throw new PatternError('a per-leg wording replaces the plain query rather than narrowing it, so the two ways of asking cannot both be given');
|
|
194
|
+
}
|
|
195
|
+
const both = Boolean(text && vector);
|
|
196
|
+
if (query.mode !== undefined && !(both && query.mode === 'hybrid')) {
|
|
197
|
+
throw new PatternError(both
|
|
198
|
+
? 'a wording for each leg is already a hybrid search, so the mode cannot be anything else'
|
|
199
|
+
: `a ${text ? 'text' : 'vector'} wording on its own already says which leg runs, so a mode says nothing here`);
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
text: text || undefined,
|
|
203
|
+
vector: vector || undefined,
|
|
204
|
+
mode: both ? 'hybrid' : text ? 'text' : 'vector',
|
|
205
|
+
};
|
|
206
|
+
}
|
|
170
207
|
function kindsOf(kinds) {
|
|
171
208
|
if (!kinds || kinds.length === 0) {
|
|
172
209
|
return undefined;
|
package/dist/docs/tools.js
CHANGED
|
@@ -55,6 +55,18 @@ export function docsTools(index, options = {}) {
|
|
|
55
55
|
type: 'string',
|
|
56
56
|
description: 'What is wanted, as a sentence rather than keywords.',
|
|
57
57
|
},
|
|
58
|
+
text_query: {
|
|
59
|
+
type: 'string',
|
|
60
|
+
description: 'Instead of query: the wording for the full-text half alone — the ' +
|
|
61
|
+
'words the documents themselves would use. Pair it with vector_query ' +
|
|
62
|
+
'to search both halves, or send it alone for exact wording only.',
|
|
63
|
+
},
|
|
64
|
+
vector_query: {
|
|
65
|
+
type: 'string',
|
|
66
|
+
description: 'Instead of query: the wording for the vector half alone — the ' +
|
|
67
|
+
'question stated in full, for meaning rather than wording. Pair it ' +
|
|
68
|
+
'with text_query, or send it alone to search by meaning only.',
|
|
69
|
+
},
|
|
58
70
|
files: strings('Only documents whose name matches. A glob if it has * or ?, e.g. ' +
|
|
59
71
|
'"guides/**" or "acme_4.2*/api/**"; otherwise a substring.'),
|
|
60
72
|
exclude_files: strings('Documents to leave out, matched the same way.'),
|
|
@@ -84,12 +96,19 @@ export function docsTools(index, options = {}) {
|
|
|
84
96
|
description: `A ceiling on the whole answer. Default ${DEFAULT_MAX_LINES}.`,
|
|
85
97
|
},
|
|
86
98
|
},
|
|
87
|
-
required: ['query'],
|
|
88
99
|
additionalProperties: false,
|
|
89
100
|
},
|
|
90
101
|
execute: async (args) => {
|
|
102
|
+
if (!args.query?.trim() && !args.text_query?.trim() && !args.vector_query?.trim()) {
|
|
103
|
+
return {
|
|
104
|
+
error: 'nothing to search for',
|
|
105
|
+
hint: 'send `query`, or a `text_query` and `vector_query` for the two halves',
|
|
106
|
+
};
|
|
107
|
+
}
|
|
91
108
|
const query = {
|
|
92
109
|
query: args.query,
|
|
110
|
+
textQuery: args.text_query,
|
|
111
|
+
vectorQuery: args.vector_query,
|
|
93
112
|
files: args.files,
|
|
94
113
|
exclude_files: args.exclude_files,
|
|
95
114
|
section: args.section,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenera/rag",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.17",
|
|
4
4
|
"description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -61,8 +61,8 @@
|
|
|
61
61
|
"dependencies": {
|
|
62
62
|
"@apidevtools/swagger-parser": "^12.0.0",
|
|
63
63
|
"@lancedb/lancedb": "^0.38.0",
|
|
64
|
-
"@zenera/cli": "^1.1.
|
|
65
|
-
"@zenera/neo": "^1.1.
|
|
64
|
+
"@zenera/cli": "^1.1.17",
|
|
65
|
+
"@zenera/neo": "^1.1.17",
|
|
66
66
|
"graphology": "^0.26.0",
|
|
67
67
|
"remark-frontmatter": "^5.0.0",
|
|
68
68
|
"remark-gfm": "^4.0.1",
|