@zenera/rag 1.1.5 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -6
- package/dist/command.js +41 -674
- package/dist/common/embedder.d.ts +3 -0
- package/dist/common/embedder.js +64 -0
- package/dist/common/locate.d.ts +18 -0
- package/dist/common/locate.js +155 -0
- package/dist/common/manifest.d.ts +50 -0
- package/dist/common/manifest.js +62 -0
- package/dist/{schema → common}/match.d.ts +4 -0
- package/dist/{schema → common}/match.js +7 -0
- package/dist/common/progress.d.ts +57 -0
- package/dist/common/progress.js +155 -0
- package/dist/common/prose.d.ts +13 -0
- package/dist/common/prose.js +56 -0
- package/dist/index.d.ts +6 -3
- package/dist/index.js +6 -3
- package/dist/schema/build.js +6 -2
- package/dist/schema/command.d.ts +3 -0
- package/dist/schema/command.js +819 -0
- package/dist/schema/files.d.ts +5 -27
- package/dist/schema/files.js +9 -29
- package/dist/schema/lookup.d.ts +12 -2
- package/dist/schema/lookup.js +29 -2
- package/dist/{present.d.ts → schema/present.d.ts} +5 -5
- package/dist/{present.js → schema/present.js} +2 -2
- package/dist/{query.d.ts → schema/query.d.ts} +1 -1
- package/dist/schema/readme.d.ts +6 -0
- package/dist/schema/readme.js +122 -0
- package/dist/schema/render.d.ts +8 -0
- package/dist/schema/render.js +12 -2
- package/dist/{repl.d.ts → schema/repl.d.ts} +1 -1
- package/dist/schema/search.js +2 -1
- package/dist/schema/tools.d.ts +3 -1
- package/dist/schema/tools.js +152 -19
- package/dist/schema/trace.d.ts +52 -0
- package/dist/schema/trace.js +144 -0
- package/package.json +3 -3
- package/dist/schema/progress.d.ts +0 -26
- package/dist/schema/progress.js +0 -316
- /package/dist/{query.js → schema/query.js} +0 -0
- /package/dist/{repl.js → schema/repl.js} +0 -0
package/dist/command.js
CHANGED
|
@@ -1,694 +1,61 @@
|
|
|
1
|
-
import { bold,
|
|
2
|
-
import {
|
|
3
|
-
import { relative, resolve } from 'node:path';
|
|
4
|
-
import { isFormat, present } from "./present.js";
|
|
5
|
-
import { isEmpty, parseQuery, QueryError } from "./query.js";
|
|
6
|
-
import { repl } from "./repl.js";
|
|
7
|
-
import { buildIndex } from "./schema/build.js";
|
|
8
|
-
import { assertSameEmbedding, openIndex, readManifest, readSource, } from "./schema/files.js";
|
|
9
|
-
import { fields, grepNodes, listNodes, propertyCount } from "./schema/lookup.js";
|
|
10
|
-
import { isGlob, loose, matcher, PatternError, wildcard } from "./schema/match.js";
|
|
11
|
-
import { SchemaIndex } from "./schema/search.js";
|
|
12
|
-
import { select, stitch } from "./schema/subgraph.js";
|
|
1
|
+
import { bold, cyan, dim, table, usageError, write, } from '@zenera/cli/lib';
|
|
2
|
+
import { command as schema } from "./schema/command.js";
|
|
13
3
|
// ---------------------------------------------------------------------------
|
|
14
|
-
// zen rag —
|
|
4
|
+
// zen rag — retrieval, by subject
|
|
15
5
|
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
// required, and exit 0 when nothing matched — an empty answer is an answer, and
|
|
21
|
-
// a caller that has to tell "no results" from "the index is missing" by parsing
|
|
22
|
-
// stderr will get it wrong.
|
|
6
|
+
// A subject is a kind of corpus with its own index format, its own verbs and
|
|
7
|
+
// its own flags. They are not variations on one command: what `list` means to
|
|
8
|
+
// an API description is not what it would mean to a folder of notes, and one
|
|
9
|
+
// flag table covering both would be twice as long and half as true.
|
|
23
10
|
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
11
|
+
// So the only thing settled here is the subject word. Everything after it
|
|
12
|
+
// belongs to the subject, which is handed its own arguments with the word
|
|
13
|
+
// removed — the same contract the `zen` frame gives this command. There is
|
|
14
|
+
// deliberately no spelling that omits the subject: `zen rag search` would have
|
|
15
|
+
// to mean one of them, and whichever was chosen would be wrong for the other
|
|
16
|
+
// forever.
|
|
17
|
+
//
|
|
18
|
+
// `help <subject>` is a verb rather than a flag because `--help` never gets
|
|
19
|
+
// here: the frame lifts it out of the arguments and answers with this page.
|
|
29
20
|
// ---------------------------------------------------------------------------
|
|
30
|
-
const
|
|
31
|
-
const
|
|
32
|
-
const SEARCH_USAGE = 'zen rag schema search [--dir <dir>] [query...]';
|
|
33
|
-
const LIST_USAGE = 'zen rag schema list <methods|types|properties> [--dir <dir>]';
|
|
34
|
-
const GREP_USAGE = 'zen rag schema grep <pattern> [--dir <dir>]';
|
|
35
|
-
const SHOW_USAGE = 'zen rag schema show [id...] [--method <name>] [--type <name>]';
|
|
36
|
-
const DEFAULT_DIR = './schema-db';
|
|
21
|
+
const SUBJECTS = { schema };
|
|
22
|
+
const USAGE = 'zen rag <subject> <command> [args...]';
|
|
37
23
|
export const command = {
|
|
38
|
-
summary: '
|
|
24
|
+
summary: 'Retrieval over a corpus: index it, then ask it something.',
|
|
39
25
|
usage: USAGE,
|
|
40
26
|
details: [
|
|
41
|
-
'
|
|
42
|
-
...table([
|
|
43
|
-
[' index <spec...>', dim('Read the documents and write a searchable index.')],
|
|
44
|
-
[' search', dim('Ask it something. --interactive for a prompt.')],
|
|
45
|
-
[' list <what>', dim('Every method, type or property matching a pattern.')],
|
|
46
|
-
[' grep <pattern>', dim('Every literal match, ranked by nothing.')],
|
|
47
|
-
[' show [id...]', dim('Print named nodes, with no search in between.')],
|
|
48
|
-
[' stats', dim('What is in an index, and what built it.')],
|
|
49
|
-
]),
|
|
50
|
-
'',
|
|
51
|
-
'Index',
|
|
52
|
-
...table([
|
|
53
|
-
[
|
|
54
|
-
' --embedding <ref>',
|
|
55
|
-
dim('Which embedder makes the vectors. Omit it to be shown the choices.'),
|
|
56
|
-
],
|
|
57
|
-
[' -o, --out <dir>', dim(`Where the index goes. Default ${DEFAULT_DIR}.`)],
|
|
58
|
-
[
|
|
59
|
-
' --batch <n>',
|
|
60
|
-
dim('Texts per embedding request, and how often progress prints. Default 96.'),
|
|
61
|
-
],
|
|
62
|
-
[' --no-sources', dim('Do not keep a copy of each document in the index.')],
|
|
63
|
-
]),
|
|
64
|
-
'',
|
|
65
|
-
'Search terms (repeatable)',
|
|
66
|
-
...table([
|
|
67
|
-
[' <text>', dim('A bare phrase, the same as --all.')],
|
|
68
|
-
[' --all <q>', dim('Against everything, unfiltered.')],
|
|
69
|
-
[' --method <q>', dim('Operations.')],
|
|
70
|
-
[' --type <q>', dim('Schemas, on the side --direction names.')],
|
|
71
|
-
[' --input-type <q>', dim('Schemas a call accepts.')],
|
|
72
|
-
[' --output-type <q>', dim('Schemas a call returns.')],
|
|
73
|
-
[' --property <q>', dim('Fields and parameters, per --direction.')],
|
|
74
|
-
[' --input-property <q>', dim('Fields and parameters a call accepts.')],
|
|
75
|
-
[' --output-property <q>', dim('Fields a call returns.')],
|
|
76
|
-
[' --query <json|->', dim('A whole query object; - reads stdin.')],
|
|
77
|
-
]),
|
|
78
|
-
'',
|
|
79
|
-
'Search filters and shape',
|
|
80
|
-
...table([
|
|
81
|
-
[' -d, --dir <dir>', dim(`Which index. Default ${DEFAULT_DIR}.`)],
|
|
82
|
-
[' --embedding <ref>', dim('Must be the one the index was built with.')],
|
|
83
|
-
[' --direction <d>', dim('input | output | any. Default any.')],
|
|
84
|
-
[' --method-type <t>', dim('read_only | read_write | any. Default any.')],
|
|
85
|
-
[' --exclude-id <id>', dim('Drop a node. Repeatable, as are the three below.')],
|
|
86
|
-
[' --exclude-method <name>', dim('Drop an operation by name.')],
|
|
87
|
-
[' --exclude-type <name>', dim('Drop a schema by name.')],
|
|
88
|
-
[' --exclude-property <name>', dim('Drop a field by name.')],
|
|
89
|
-
[' --limit <n>', dim('Seeds kept per term. Default 5.')],
|
|
90
|
-
[' --max-hops <n>', dim('How far apart two hits may be. Default 3.')],
|
|
91
|
-
[' --max-nodes <n>', dim('Nodes per result. Default 200.')],
|
|
92
|
-
[' --format <f>', dim('text | mermaid | mermaid-flowchart | ts | openapi.')],
|
|
93
|
-
[' --no-docs', dim('Leave the descriptions out.')],
|
|
94
|
-
[' --interactive', dim('Prompt, search, refine. Needs a terminal.')],
|
|
95
|
-
[' --quiet', dim('No narration.')],
|
|
96
|
-
]),
|
|
97
|
-
'',
|
|
98
|
-
'Exact listing — no embedder, no credential',
|
|
99
|
-
...table([
|
|
100
|
-
[' list methods', dim('Operations. Filter with --path and --name.')],
|
|
101
|
-
[' list types', dim('Schemas. Filter with --name.')],
|
|
102
|
-
[' list properties', dim('Fields and parameters. Filter with --name.')],
|
|
103
|
-
[' grep <pattern>', dim('Substring over every node; --regex for a regex.')],
|
|
104
|
-
[' --case-sensitive', dim('grep: match the capitals too.')],
|
|
105
|
-
[' --kind <k>', dim('grep: method | type | property. Repeatable.')],
|
|
106
|
-
[' --ids-only', dim('grep: bare ids, to pipe into show.')],
|
|
107
|
-
[' --source <name>', dim('Only this document, as `stats` names it.')],
|
|
108
|
-
[' --limit <n>', dim('Keep at most n; the count still reports them all.')],
|
|
109
|
-
]),
|
|
110
|
-
'',
|
|
111
|
-
dim(' A pattern with * or ? is a glob over the whole name; otherwise it is'),
|
|
112
|
-
dim(' a substring, so --name password finds ResetPasswordPayload.'),
|
|
27
|
+
'Subjects',
|
|
28
|
+
...table(Object.entries(SUBJECTS).map(([name, sub]) => [` ${name}`, dim(sub.summary)])),
|
|
113
29
|
'',
|
|
114
|
-
|
|
115
|
-
...table([
|
|
116
|
-
[' <id...>', dim('Node ids, e.g. Type:User or Property:User.email.')],
|
|
117
|
-
[' --method <name>', dim('An operation by name. * to take more. Repeatable.')],
|
|
118
|
-
[' --type <name>', dim('A schema by name. * to take more. Repeatable.')],
|
|
119
|
-
[' --source <name>', dim('A whole document, as it was indexed.')],
|
|
120
|
-
[' --exact', dim('Only what was named, without the neighbours.')],
|
|
121
|
-
]),
|
|
30
|
+
...Object.keys(SUBJECTS).map((name) => dim(` ${cyan(`zen rag help ${name}`)} — its commands, flags and examples`)),
|
|
122
31
|
'',
|
|
123
32
|
dim(`Credentials come from the ${cyan('zen')} keyring — try ${cyan('zen key ls')}.`),
|
|
124
33
|
],
|
|
125
34
|
async run(ctx) {
|
|
126
|
-
const [
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const [name, ...tail] = group === 'schema' ? rest : ctx.args;
|
|
130
|
-
switch (name) {
|
|
131
|
-
case 'index':
|
|
132
|
-
return await index(tail, ctx);
|
|
133
|
-
case 'search':
|
|
134
|
-
return await search(tail, ctx);
|
|
135
|
-
case 'list':
|
|
136
|
-
return await list(tail, ctx);
|
|
137
|
-
case 'grep':
|
|
138
|
-
return await grep(tail, ctx);
|
|
139
|
-
case 'show':
|
|
140
|
-
return await show(tail, ctx);
|
|
141
|
-
case 'stats':
|
|
142
|
-
return await stats(tail, ctx);
|
|
143
|
-
default:
|
|
144
|
-
throw usageError(name ? `unknown command "${name}"` : 'no command given', USAGE);
|
|
35
|
+
const [subject, ...rest] = ctx.args;
|
|
36
|
+
if (subject === 'help') {
|
|
37
|
+
return help(rest[0]);
|
|
145
38
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const { values, positionals } = parse(args, {
|
|
150
|
-
out: { type: 'string', short: 'o' },
|
|
151
|
-
embedding: { type: 'string' },
|
|
152
|
-
batch: { type: 'string' },
|
|
153
|
-
'no-sources': { type: 'boolean' },
|
|
154
|
-
quiet: { type: 'boolean' },
|
|
155
|
-
}, INDEX_USAGE);
|
|
156
|
-
if (positionals.length === 0) {
|
|
157
|
-
throw usageError('no document given', INDEX_USAGE);
|
|
158
|
-
}
|
|
159
|
-
const out = resolve(ctx.cwd, values.out ?? DEFAULT_DIR);
|
|
160
|
-
const loud = !values.quiet && !ctx.json;
|
|
161
|
-
const chosen = await embedder(values.embedding);
|
|
162
|
-
const started = Date.now();
|
|
163
|
-
const { manifest } = await buildIndex({
|
|
164
|
-
files: positionals.map((file) => resolve(ctx.cwd, file)),
|
|
165
|
-
out,
|
|
166
|
-
embedder: chosen,
|
|
167
|
-
embeddingRef: values.embedding,
|
|
168
|
-
indexer: 'zenera-rag',
|
|
169
|
-
batch: values.batch ? count(values.batch, '--batch') : undefined,
|
|
170
|
-
sources: !values['no-sources'],
|
|
171
|
-
onRead: loud
|
|
172
|
-
? (summary) => {
|
|
173
|
-
printSources(summary.sources);
|
|
174
|
-
// The first batch can take a while and says nothing while it
|
|
175
|
-
// does; this is the line that makes that a wait, not a hang.
|
|
176
|
-
note(dim(` embedding ${summary.counts.entities} entities with ${chosen.id} …`));
|
|
177
|
-
}
|
|
178
|
-
: undefined,
|
|
179
|
-
onProgress: loud
|
|
180
|
-
? (done, total) => note(dim(` embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
|
|
181
|
-
: undefined,
|
|
182
|
-
});
|
|
183
|
-
if (ctx.json) {
|
|
184
|
-
json({ out, manifest });
|
|
185
|
-
return;
|
|
186
|
-
}
|
|
187
|
-
// stdout is the path and nothing else, so `DIR=$(zen rag schema index …)`
|
|
188
|
-
// works; what it means goes to stderr, where the narration lives.
|
|
189
|
-
note();
|
|
190
|
-
write(out);
|
|
191
|
-
note(` wrote ${bold(String(manifest.counts.entities))} entities to ${bold(out)}, ` +
|
|
192
|
-
`embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
|
|
193
|
-
const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
|
|
194
|
-
note(dim(` search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
|
|
195
|
-
}
|
|
196
|
-
const HEADERS = ['PATHS', 'OPERATIONS', 'SCHEMAS', 'FIELDS'];
|
|
197
|
-
function elapsed(since) {
|
|
198
|
-
const seconds = Math.round((Date.now() - since) / 1000);
|
|
199
|
-
return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
|
|
200
|
-
}
|
|
201
|
-
function printSources(sources) {
|
|
202
|
-
const rows = sources.map((s) => ({
|
|
203
|
-
name: s.file,
|
|
204
|
-
dialect: s.dialect,
|
|
205
|
-
cells: [s.paths, s.methods, s.types, s.properties],
|
|
206
|
-
}));
|
|
207
|
-
if (rows.length > 1) {
|
|
208
|
-
rows.push({
|
|
209
|
-
name: 'total',
|
|
210
|
-
dialect: '',
|
|
211
|
-
cells: HEADERS.map((_, i) => rows.reduce((n, r) => n + (r.cells[i] ?? 0), 0)),
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
|
-
// Numbers are padded before they are styled: a colour code has no width,
|
|
215
|
-
// and `table` cannot know that.
|
|
216
|
-
const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r.cells[i]).length)));
|
|
217
|
-
note();
|
|
218
|
-
notes(table([
|
|
219
|
-
[bold('SPEC'), bold('DIALECT'), ...HEADERS.map((h, i) => bold(h.padStart(widths[i])))],
|
|
220
|
-
...rows.map((r) => [
|
|
221
|
-
r.name === 'total' ? dim(r.name) : r.name,
|
|
222
|
-
dim(r.dialect),
|
|
223
|
-
...r.cells.map((c, i) => String(c).padStart(widths[i])),
|
|
224
|
-
]),
|
|
225
|
-
]).map((line) => ` ${line}`));
|
|
226
|
-
note();
|
|
227
|
-
}
|
|
228
|
-
const MANY = { type: 'string', multiple: true };
|
|
229
|
-
const SEARCH_OPTIONS = {
|
|
230
|
-
dir: { type: 'string', short: 'd' },
|
|
231
|
-
embedding: { type: 'string' },
|
|
232
|
-
all: MANY,
|
|
233
|
-
method: MANY,
|
|
234
|
-
type: MANY,
|
|
235
|
-
'input-type': MANY,
|
|
236
|
-
'output-type': MANY,
|
|
237
|
-
property: MANY,
|
|
238
|
-
'input-property': MANY,
|
|
239
|
-
'output-property': MANY,
|
|
240
|
-
query: { type: 'string' },
|
|
241
|
-
direction: { type: 'string' },
|
|
242
|
-
'method-type': { type: 'string' },
|
|
243
|
-
'exclude-id': MANY,
|
|
244
|
-
'exclude-method': MANY,
|
|
245
|
-
'exclude-type': MANY,
|
|
246
|
-
'exclude-property': MANY,
|
|
247
|
-
limit: { type: 'string' },
|
|
248
|
-
'max-hops': { type: 'string' },
|
|
249
|
-
'max-nodes': { type: 'string' },
|
|
250
|
-
format: { type: 'string' },
|
|
251
|
-
'no-docs': { type: 'boolean' },
|
|
252
|
-
'only-hits': { type: 'boolean' },
|
|
253
|
-
interactive: { type: 'boolean' },
|
|
254
|
-
quiet: { type: 'boolean' },
|
|
255
|
-
};
|
|
256
|
-
async function search(args, ctx) {
|
|
257
|
-
const { values, positionals } = parse(args, SEARCH_OPTIONS, SEARCH_USAGE);
|
|
258
|
-
const dir = resolve(ctx.cwd, values.dir ?? DEFAULT_DIR);
|
|
259
|
-
const format = formatOf(values.format);
|
|
260
|
-
const options = { docs: !values['no-docs'], onlyHits: values['only-hits'] };
|
|
261
|
-
const query = { ...(await fromStdin(values.query)), ...fromFlags(values, positionals) };
|
|
262
|
-
// Everything that can be wrong about the invocation is settled before a
|
|
263
|
-
// credential is asked for, so a typo is a usage error and not a login.
|
|
264
|
-
if (values.interactive && !isInteractive()) {
|
|
265
|
-
throw usageError('--interactive needs a terminal', SEARCH_USAGE);
|
|
266
|
-
}
|
|
267
|
-
if (!values.interactive && isEmpty(query)) {
|
|
268
|
-
throw usageError('no query given', SEARCH_USAGE);
|
|
269
|
-
}
|
|
270
|
-
const manifest = await readManifest(dir);
|
|
271
|
-
const ref = values.embedding ?? manifest.embedding.ref;
|
|
272
|
-
assertSameEmbedding(manifest, ref);
|
|
273
|
-
const index = await SchemaIndex.open(dir, await embedder(ref));
|
|
274
|
-
try {
|
|
275
|
-
if (values.interactive) {
|
|
276
|
-
await repl(index, query, { format, ...options });
|
|
277
|
-
return;
|
|
278
|
-
}
|
|
279
|
-
const result = await index.search(query);
|
|
280
|
-
if (ctx.json) {
|
|
281
|
-
json({
|
|
282
|
-
seeds: result.seeds,
|
|
283
|
-
empty: result.empty,
|
|
284
|
-
subgraphs: result.subgraphs,
|
|
285
|
-
rendered: await present(index, result.subgraphs, format, options),
|
|
286
|
-
});
|
|
287
|
-
return;
|
|
39
|
+
const chosen = subject ? SUBJECTS[subject] : undefined;
|
|
40
|
+
if (!chosen) {
|
|
41
|
+
throw usageError(subject ? `unknown subject "${subject}"` : 'no subject given', `expected ${Object.keys(SUBJECTS).join(' or ')} — ${USAGE}`);
|
|
288
42
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
write(text);
|
|
292
|
-
}
|
|
293
|
-
if (!values.quiet) {
|
|
294
|
-
note(dim(` ${result.seeds.length} seed(s) · ${result.subgraphs.length} result(s)${result.empty.length > 0 ? ` · nothing for: ${result.empty.join(', ')}` : ''}`));
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
finally {
|
|
298
|
-
index.close();
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
/** Flags win over `--query`: the more specific spelling is the later thought. */
|
|
302
|
-
function fromFlags(values, positionals = []) {
|
|
303
|
-
const query = {};
|
|
304
|
-
const put = (key, value) => {
|
|
305
|
-
if (value !== undefined && (!Array.isArray(value) || value.length > 0)) {
|
|
306
|
-
query[key] = value;
|
|
307
|
-
}
|
|
308
|
-
};
|
|
309
|
-
// A bare phrase is the unfiltered search; there is nothing else it could mean.
|
|
310
|
-
put('all', [...(values.all ?? []), ...positionals]);
|
|
311
|
-
put('methods', values.method);
|
|
312
|
-
put('types', values.type);
|
|
313
|
-
put('input_types', values['input-type']);
|
|
314
|
-
put('output_types', values['output-type']);
|
|
315
|
-
put('properties', values.property);
|
|
316
|
-
put('input_properties', values['input-property']);
|
|
317
|
-
put('output_properties', values['output-property']);
|
|
318
|
-
put('exclude_ids', values['exclude-id']);
|
|
319
|
-
put('exclude_methods', values['exclude-method']);
|
|
320
|
-
put('exclude_types', values['exclude-type']);
|
|
321
|
-
put('exclude_properties', values['exclude-property']);
|
|
322
|
-
put('direction', values.direction);
|
|
323
|
-
put('method_type', values['method-type']);
|
|
324
|
-
put('limit', values.limit && count(values.limit, '--limit'));
|
|
325
|
-
put('max_hops', values['max-hops'] && count(values['max-hops'], '--max-hops'));
|
|
326
|
-
put('max_nodes', values['max-nodes'] && count(values['max-nodes'], '--max-nodes'));
|
|
327
|
-
return check(query);
|
|
328
|
-
}
|
|
329
|
-
async function fromStdin(source) {
|
|
330
|
-
if (source === undefined) {
|
|
331
|
-
return {};
|
|
332
|
-
}
|
|
333
|
-
const text = source === '-' ? await readStdin() : source;
|
|
334
|
-
let parsed;
|
|
335
|
-
try {
|
|
336
|
-
parsed = JSON.parse(text);
|
|
337
|
-
}
|
|
338
|
-
catch (err) {
|
|
339
|
-
throw usageError(`--query is not JSON: ${err.message}`, SEARCH_USAGE);
|
|
340
|
-
}
|
|
341
|
-
return check(parsed);
|
|
342
|
-
}
|
|
343
|
-
async function readStdin() {
|
|
344
|
-
const chunks = [];
|
|
345
|
-
for await (const chunk of process.stdin) {
|
|
346
|
-
chunks.push(chunk);
|
|
347
|
-
}
|
|
348
|
-
return Buffer.concat(chunks).toString('utf8');
|
|
349
|
-
}
|
|
350
|
-
function check(value) {
|
|
351
|
-
try {
|
|
352
|
-
return parseQuery(value);
|
|
353
|
-
}
|
|
354
|
-
catch (err) {
|
|
355
|
-
if (err instanceof QueryError) {
|
|
356
|
-
throw usageError(err.message, SEARCH_USAGE);
|
|
357
|
-
}
|
|
358
|
-
throw err;
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
// ---------------------------------------------------------------------------
|
|
362
|
-
// list, grep
|
|
363
|
-
//
|
|
364
|
-
// The deterministic half. Neither takes an embedder, because neither ranks
|
|
365
|
-
// anything: `list` filters on the attributes a node already has and `grep`
|
|
366
|
-
// reads the same materialized string the index was built from. What comes back
|
|
367
|
-
// is every match, and where a limit cut the list the count still reports the
|
|
368
|
-
// total — being shown three of three hundred is only useful if you are told
|
|
369
|
-
// which of the two happened.
|
|
370
|
-
// ---------------------------------------------------------------------------
|
|
371
|
-
const SUBJECTS = {
|
|
372
|
-
methods: 'method',
|
|
373
|
-
types: 'type',
|
|
374
|
-
properties: 'property',
|
|
43
|
+
return await chosen.run({ ...ctx, args: rest });
|
|
44
|
+
},
|
|
375
45
|
};
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
source: { type: 'string' },
|
|
382
|
-
'method-type': { type: 'string' },
|
|
383
|
-
direction: { type: 'string' },
|
|
384
|
-
limit: { type: 'string' },
|
|
385
|
-
quiet: { type: 'boolean' },
|
|
386
|
-
}, LIST_USAGE);
|
|
387
|
-
const subject = positionals[0];
|
|
388
|
-
const kind = subject ? SUBJECTS[subject] : undefined;
|
|
389
|
-
if (!kind) {
|
|
390
|
-
throw usageError(subject ? `cannot list "${subject}"` : 'nothing named to list', `expected one of ${Object.keys(SUBJECTS).join(', ')}`);
|
|
391
|
-
}
|
|
392
|
-
if (positionals.length > 1) {
|
|
393
|
-
throw usageError('one subject at a time', LIST_USAGE);
|
|
394
|
-
}
|
|
395
|
-
const index = await openIndex(resolve(ctx.cwd, values.dir ?? DEFAULT_DIR));
|
|
396
|
-
const found = listNodes(index.graph, {
|
|
397
|
-
kind,
|
|
398
|
-
name: globs(values.name, '--name'),
|
|
399
|
-
path: globs(values.path, '--path'),
|
|
400
|
-
source: values.source,
|
|
401
|
-
methodType: oneOf(values['method-type'], ['read_only', 'read_write'], '--method-type'),
|
|
402
|
-
direction: oneOf(values.direction, ['input', 'output'], '--direction'),
|
|
403
|
-
limit: values.limit ? count(values.limit, '--limit') : undefined,
|
|
404
|
-
});
|
|
405
|
-
if (ctx.json) {
|
|
406
|
-
json({ found: found.found, truncated: found.truncated, rows: found.rows });
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
const lines = rowLines(index.graph, kind, found.rows);
|
|
410
|
-
if (lines.length > 0) {
|
|
411
|
-
write(lines.join('\n'));
|
|
412
|
-
}
|
|
413
|
-
if (!values.quiet) {
|
|
414
|
-
note(dim(` ${found.found} ${subject}${shown(found.found, found.rows.length)}`));
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
function rowLines(graph, kind, rows) {
|
|
418
|
-
if (kind === 'method') {
|
|
419
|
-
return table(rows.map((r) => [`${r.httpMethod} ${r.path}`, r.name, doc(r.doc)]));
|
|
46
|
+
/** Laid out as `zen help <command>` lays out this one, so the two pages match. */
|
|
47
|
+
function help(subject) {
|
|
48
|
+
const chosen = subject ? SUBJECTS[subject] : undefined;
|
|
49
|
+
if (!chosen) {
|
|
50
|
+
throw usageError(subject ? `unknown subject "${subject}"` : 'which subject', `expected ${Object.keys(SUBJECTS).join(' or ')} — zen rag help <subject>`);
|
|
420
51
|
}
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
]));
|
|
428
|
-
}
|
|
429
|
-
return table(rows.map((r) => [
|
|
430
|
-
`${r.parent ? `${r.parent}.` : ''}${r.name}${r.required ? '' : '?'}`,
|
|
431
|
-
`: ${r.signature || 'unknown'}`,
|
|
432
|
-
doc(r.doc),
|
|
433
|
-
]));
|
|
434
|
-
}
|
|
435
|
-
async function grep(args, ctx) {
|
|
436
|
-
const { values, positionals } = parse(args, {
|
|
437
|
-
dir: { type: 'string', short: 'd' },
|
|
438
|
-
regex: { type: 'boolean' },
|
|
439
|
-
'case-sensitive': { type: 'boolean' },
|
|
440
|
-
kind: MANY,
|
|
441
|
-
source: { type: 'string' },
|
|
442
|
-
limit: { type: 'string' },
|
|
443
|
-
'ids-only': { type: 'boolean' },
|
|
444
|
-
quiet: { type: 'boolean' },
|
|
445
|
-
}, GREP_USAGE);
|
|
446
|
-
if (positionals.length === 0) {
|
|
447
|
-
throw usageError('no pattern given', GREP_USAGE);
|
|
448
|
-
}
|
|
449
|
-
if (positionals.length > 1) {
|
|
450
|
-
throw usageError('one pattern at a time — quote it if it has spaces', GREP_USAGE);
|
|
451
|
-
}
|
|
452
|
-
const kinds = (values.kind ?? []).map((k) => oneOf(k, ['method', 'type', 'property'], '--kind'));
|
|
453
|
-
const index = await openIndex(resolve(ctx.cwd, values.dir ?? DEFAULT_DIR));
|
|
454
|
-
const result = pattern(() => grepNodes(index.graph, matcher(positionals[0], {
|
|
455
|
-
regex: values.regex,
|
|
456
|
-
caseSensitive: values['case-sensitive'],
|
|
457
|
-
}), {
|
|
458
|
-
kinds,
|
|
459
|
-
source: values.source,
|
|
460
|
-
limit: values.limit ? count(values.limit, '--limit') : undefined,
|
|
461
|
-
}));
|
|
462
|
-
if (ctx.json) {
|
|
463
|
-
json({
|
|
464
|
-
found: result.found,
|
|
465
|
-
truncated: result.truncated,
|
|
466
|
-
matches: result.matches.map((m) => ({ id: m.id, ...m.attributes, text: m.text })),
|
|
467
|
-
});
|
|
468
|
-
return;
|
|
469
|
-
}
|
|
470
|
-
if (result.matches.length > 0) {
|
|
471
|
-
const lines = values['ids-only']
|
|
472
|
-
? result.matches.map((m) => m.id)
|
|
473
|
-
: table(result.matches.map((m) => [m.id, dim(clip(m.text, 140))]));
|
|
474
|
-
write(lines.join('\n'));
|
|
475
|
-
}
|
|
476
|
-
if (!values.quiet && !values['ids-only']) {
|
|
477
|
-
note(dim(` ${result.found} match(es)${shown(result.found, result.matches.length)}`));
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
// ---------------------------------------------------------------------------
|
|
481
|
-
const shown = (found, kept) => (kept < found ? `, showing ${kept}` : '');
|
|
482
|
-
function globs(patterns, flag) {
|
|
483
|
-
if (!patterns || patterns.length === 0) {
|
|
484
|
-
return undefined;
|
|
485
|
-
}
|
|
486
|
-
return patterns.map((p) => pattern(() => loose(p), flag));
|
|
487
|
-
}
|
|
488
|
-
/** A bad pattern is a bad invocation, not a failure of the index. */
|
|
489
|
-
function pattern(run, flag) {
|
|
490
|
-
try {
|
|
491
|
-
return run();
|
|
492
|
-
}
|
|
493
|
-
catch (err) {
|
|
494
|
-
if (err instanceof PatternError) {
|
|
495
|
-
throw usageError(`${flag ? `${flag}: ` : ''}${err.message}`, USAGE);
|
|
52
|
+
write(bold(chosen.usage));
|
|
53
|
+
write(`\n ${chosen.summary}`);
|
|
54
|
+
if (chosen.details?.length) {
|
|
55
|
+
write('');
|
|
56
|
+
for (const line of chosen.details) {
|
|
57
|
+
write(line ? ` ${line}` : '');
|
|
496
58
|
}
|
|
497
|
-
throw err;
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
function oneOf(value, allowed, flag) {
|
|
501
|
-
if (value === undefined || value === 'any') {
|
|
502
|
-
return undefined;
|
|
503
|
-
}
|
|
504
|
-
if (!allowed.includes(value)) {
|
|
505
|
-
throw usageError(`${flag} cannot be "${value}"`, `expected ${allowed.join(' or ')}`);
|
|
506
|
-
}
|
|
507
|
-
return value;
|
|
508
|
-
}
|
|
509
|
-
const doc = (text) => (text ? dim(`— ${clip(text.replace(/\s+/g, ' '), 90)}`) : '');
|
|
510
|
-
const clip = (text, max) => text.length <= max ? text : `${text.slice(0, max - 1)}…`;
|
|
511
|
-
/**
|
|
512
|
-
* No embedder and no store: naming a node is a graph lookup, and asking for a
|
|
513
|
-
* credential to print something already on disk would be theatre.
|
|
514
|
-
*
|
|
515
|
-
* Ids are the precise way in, and `--method`/`--type` are the way in for
|
|
516
|
-
* someone who has a name rather than an id — which, with `--format openapi
|
|
517
|
-
* --exact`, is how a resolved slice of the document is got out.
|
|
518
|
-
*/
|
|
519
|
-
async function show(args, ctx) {
|
|
520
|
-
const { values, positionals } = parse(args, {
|
|
521
|
-
dir: { type: 'string', short: 'd' },
|
|
522
|
-
method: MANY,
|
|
523
|
-
type: MANY,
|
|
524
|
-
source: { type: 'string' },
|
|
525
|
-
exact: { type: 'boolean' },
|
|
526
|
-
format: { type: 'string' },
|
|
527
|
-
'max-nodes': { type: 'string' },
|
|
528
|
-
'no-docs': { type: 'boolean' },
|
|
529
|
-
quiet: { type: 'boolean' },
|
|
530
|
-
}, SHOW_USAGE);
|
|
531
|
-
const format = formatOf(values.format);
|
|
532
|
-
const dir = resolve(ctx.cwd, values.dir ?? DEFAULT_DIR);
|
|
533
|
-
// A whole document, verbatim: the copy kept at index time is the resolved
|
|
534
|
-
// original, and anything rebuilt from the graph would be a paraphrase.
|
|
535
|
-
if (values.source && format === 'openapi' && positionals.length === 0 && !named(values)) {
|
|
536
|
-
const document = await readSource(dir, values.source);
|
|
537
|
-
if (document) {
|
|
538
|
-
write(document);
|
|
539
|
-
return;
|
|
540
|
-
}
|
|
541
|
-
if (!values.quiet) {
|
|
542
|
-
note(dim(' this index kept no copy of the documents — rebuilding it from the graph'));
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
const index = await openIndex(dir);
|
|
546
|
-
const ids = resolveIds(index.graph, positionals, values);
|
|
547
|
-
const subgraphs = values.exact
|
|
548
|
-
? [select(index.graph, ids)]
|
|
549
|
-
: stitch(index.graph, ids.map((id) => ({ id, term: id, field: 'show', score: 1 })), {
|
|
550
|
-
maxNodes: values['max-nodes']
|
|
551
|
-
? count(values['max-nodes'], '--max-nodes')
|
|
552
|
-
: undefined,
|
|
553
|
-
});
|
|
554
|
-
const text = await present(index, subgraphs, format, { docs: !values['no-docs'] });
|
|
555
|
-
if (ctx.json) {
|
|
556
|
-
json({ ids, subgraphs, rendered: text });
|
|
557
|
-
}
|
|
558
|
-
else if (text) {
|
|
559
|
-
write(text);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
const named = (values) => Boolean(values.method?.length || values.type?.length);
|
|
563
|
-
/** Ids as given, plus whatever the name selectors resolve to. */
|
|
564
|
-
function resolveIds(graph, ids, values) {
|
|
565
|
-
if (ids.length === 0 && !named(values) && !values.source) {
|
|
566
|
-
throw usageError('no node named', SHOW_USAGE);
|
|
567
|
-
}
|
|
568
|
-
const missing = ids.filter((id) => !graph.hasNode(id));
|
|
569
|
-
if (missing.length > 0) {
|
|
570
|
-
throw new CliError(`no such node: ${missing.join(', ')}`, EXIT.failed, 'ids look like `Type:User` or `Property:User.email`');
|
|
571
|
-
}
|
|
572
|
-
const out = new Set(ids);
|
|
573
|
-
for (const kind of ['method', 'type']) {
|
|
574
|
-
for (const wanted of values[kind] ?? []) {
|
|
575
|
-
// Selecting, not searching: a bare name means that name. A star is
|
|
576
|
-
// the way to ask for more than one.
|
|
577
|
-
const match = isGlob(wanted)
|
|
578
|
-
? pattern(() => wildcard(wanted), `--${kind}`)
|
|
579
|
-
: (name) => name === wanted;
|
|
580
|
-
const rows = listNodes(graph, { kind, name: [match], source: values.source });
|
|
581
|
-
// A selector that matched nothing is a wrong answer, not an empty
|
|
582
|
-
// one: the caller named something they believe is there.
|
|
583
|
-
if (rows.found === 0) {
|
|
584
|
-
throw new CliError(`no ${kind} called ${wanted}`, EXIT.failed, `try: zen rag schema list ${kind}s --name "${wanted}"`);
|
|
585
|
-
}
|
|
586
|
-
for (const row of rows.rows) {
|
|
587
|
-
out.add(row.id);
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
// `--source` on its own means the whole document.
|
|
592
|
-
if (out.size === 0 && values.source) {
|
|
593
|
-
for (const kind of ['method', 'type']) {
|
|
594
|
-
for (const row of listNodes(graph, { kind, source: values.source }).rows) {
|
|
595
|
-
out.add(row.id);
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
if (out.size === 0) {
|
|
599
|
-
throw new CliError(`nothing in this index came from ${values.source}`, EXIT.failed);
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
return [...out];
|
|
603
|
-
}
|
|
604
|
-
async function stats(args, ctx) {
|
|
605
|
-
const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag schema stats [--dir <dir>]');
|
|
606
|
-
const dir = resolve(ctx.cwd, values.dir ?? DEFAULT_DIR);
|
|
607
|
-
const manifest = await readManifest(dir);
|
|
608
|
-
if (ctx.json) {
|
|
609
|
-
json(manifest);
|
|
610
|
-
return;
|
|
611
|
-
}
|
|
612
|
-
note(bold(dir));
|
|
613
|
-
notes(table([
|
|
614
|
-
[' built', manifest.createdAt],
|
|
615
|
-
[' by', manifest.indexer],
|
|
616
|
-
[' embedder', `${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`],
|
|
617
|
-
[
|
|
618
|
-
' indexes',
|
|
619
|
-
`fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
|
|
620
|
-
],
|
|
621
|
-
]));
|
|
622
|
-
printSources(manifest.sources);
|
|
623
|
-
notes(table([[' entities', String(manifest.counts.entities)]]).map(dim));
|
|
624
|
-
}
|
|
625
|
-
function notes(lines) {
|
|
626
|
-
for (const line of lines) {
|
|
627
|
-
note(line);
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
const yes = (value) => (value ? 'yes' : 'no');
|
|
631
|
-
// ---------------------------------------------------------------------------
|
|
632
|
-
/** The keyring is materialised here, and only here: `show` and `stats` read no vectors. */
|
|
633
|
-
async function embedder(ref) {
|
|
634
|
-
ensureHome();
|
|
635
|
-
const keys = await KeyStore.open();
|
|
636
|
-
// Asked before materialising, because materialising is what erases the
|
|
637
|
-
// difference between "the environment had it" and "the keyring supplied it".
|
|
638
|
-
const fromEnv = new Set(PROVIDERS.filter((p) => envNames(p).some((n) => process.env[n])));
|
|
639
|
-
keys.materialize();
|
|
640
|
-
if (!ref) {
|
|
641
|
-
throw choices(keys, fromEnv);
|
|
642
|
-
}
|
|
643
|
-
return createEmbedder(ref);
|
|
644
|
-
}
|
|
645
|
-
/**
|
|
646
|
-
* Well-known embedding models per provider, read off the CLI's catalog table so
|
|
647
|
-
* there is one list rather than two that drift. Any ref the registry can parse
|
|
648
|
-
* works; these are the ones worth typing. Anthropic has none because it
|
|
649
|
-
* publishes no embeddings API at all.
|
|
650
|
-
*/
|
|
651
|
-
const embeddingsOf = (provider) => CURATED[provider].filter((m) => m.roles.includes('embedding')).map((m) => m.id);
|
|
652
|
-
/** What could be passed, with the ones this machine can actually use first. */
|
|
653
|
-
function choices(keys, fromEnv) {
|
|
654
|
-
const rows = [];
|
|
655
|
-
const rest = [];
|
|
656
|
-
for (const provider of PROVIDERS) {
|
|
657
|
-
for (const model of embeddingsOf(provider)) {
|
|
658
|
-
const source = fromEnv.has(provider)
|
|
659
|
-
? 'environment'
|
|
660
|
-
: keys.active(provider)
|
|
661
|
-
? 'keyring'
|
|
662
|
-
: '';
|
|
663
|
-
const row = [` ${cyan(`${provider}:${model}`)}`, dim(source || form(provider).env)];
|
|
664
|
-
(source ? rows : rest).push(row);
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
note(bold('Embeddings'));
|
|
668
|
-
notes(table([...rows, ...rest]));
|
|
669
|
-
note('');
|
|
670
|
-
if (rows.length === 0) {
|
|
671
|
-
note(dim(' no provider on this machine has a credential — try: zen key add openai'));
|
|
672
|
-
note('');
|
|
673
|
-
}
|
|
674
|
-
// `pick` is the one that ends the question rather than restating it: it
|
|
675
|
-
// tries them and prints the first that answers.
|
|
676
|
-
return usageError('no embedder named', 'pass --embedding <ref>, or run: zen models pick --embedding');
|
|
677
|
-
}
|
|
678
|
-
function formatOf(value) {
|
|
679
|
-
if (value === undefined) {
|
|
680
|
-
return 'text';
|
|
681
|
-
}
|
|
682
|
-
if (!isFormat(value)) {
|
|
683
|
-
throw usageError(`unknown format "${value}"`, 'expected text, mermaid, mermaid-flowchart, ts or openapi');
|
|
684
|
-
}
|
|
685
|
-
return value;
|
|
686
|
-
}
|
|
687
|
-
function count(value, flag) {
|
|
688
|
-
const number = Number(value);
|
|
689
|
-
if (!Number.isInteger(number) || number < 1) {
|
|
690
|
-
throw usageError(`${flag} must be a whole number of at least 1`, USAGE);
|
|
691
59
|
}
|
|
692
|
-
return number;
|
|
693
60
|
}
|
|
694
61
|
//# sourceMappingURL=command.js.map
|