@zenera/rag 1.1.6 → 1.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +28 -2
  2. package/dist/command.js +41 -727
  3. package/dist/common/embedder.d.ts +3 -0
  4. package/dist/common/embedder.js +64 -0
  5. package/dist/common/locate.d.ts +18 -0
  6. package/dist/{schema → common}/locate.js +42 -28
  7. package/dist/common/manifest.d.ts +50 -0
  8. package/dist/common/manifest.js +62 -0
  9. package/dist/common/progress.d.ts +57 -0
  10. package/dist/common/progress.js +155 -0
  11. package/dist/common/prose.d.ts +13 -0
  12. package/dist/common/prose.js +56 -0
  13. package/dist/index.d.ts +6 -4
  14. package/dist/index.js +6 -4
  15. package/dist/schema/build.js +6 -2
  16. package/dist/schema/command.d.ts +3 -0
  17. package/dist/schema/command.js +819 -0
  18. package/dist/schema/files.d.ts +5 -27
  19. package/dist/schema/files.js +9 -29
  20. package/dist/schema/lookup.d.ts +1 -1
  21. package/dist/schema/lookup.js +1 -1
  22. package/dist/{present.d.ts → schema/present.d.ts} +5 -5
  23. package/dist/{present.js → schema/present.js} +2 -2
  24. package/dist/{query.d.ts → schema/query.d.ts} +1 -1
  25. package/dist/schema/readme.d.ts +6 -0
  26. package/dist/schema/readme.js +122 -0
  27. package/dist/{repl.d.ts → schema/repl.d.ts} +1 -1
  28. package/dist/schema/search.js +2 -1
  29. package/dist/schema/tools.d.ts +1 -1
  30. package/dist/schema/tools.js +97 -6
  31. package/dist/schema/trace.d.ts +52 -0
  32. package/dist/schema/trace.js +144 -0
  33. package/package.json +3 -3
  34. package/dist/schema/locate.d.ts +0 -20
  35. package/dist/schema/progress.d.ts +0 -26
  36. package/dist/schema/progress.js +0 -316
  37. /package/dist/{schema → common}/match.d.ts +0 -0
  38. /package/dist/{schema → common}/match.js +0 -0
  39. /package/dist/{query.js → schema/query.js} +0 -0
  40. /package/dist/{repl.js → schema/repl.js} +0 -0
package/dist/command.js CHANGED
@@ -1,747 +1,61 @@
1
- import { bold, CliError, CURATED, cyan, dim, ensureHome, envNames, EXIT, form, isInteractive, json, KeyStore, note, parse, PROVIDERS, table, usageError, write, } from '@zenera/cli/lib';
2
- import { createEmbedder } from '@zenera/neo';
3
- import { relative, resolve } from 'node:path';
4
- import { isFormat, present } from "./present.js";
5
- import { isEmpty, parseQuery, QueryError } from "./query.js";
6
- import { repl } from "./repl.js";
7
- import { buildIndex } from "./schema/build.js";
8
- import { assertSameEmbedding, openIndex, readManifest, readSource, } from "./schema/files.js";
9
- import { DEFAULT_DIR, DIR_ENV, locateIndex, outputDir } from "./schema/locate.js";
10
- import { fields, grepNodes, listNodes, propertyCount } from "./schema/lookup.js";
11
- import { isGlob, loose, matcher, PatternError, wildcard } from "./schema/match.js";
12
- import { sourceTag } from "./schema/render.js";
13
- import { SchemaIndex } from "./schema/search.js";
14
- import { select, stitch } from "./schema/subgraph.js";
1
+ import { bold, cyan, dim, table, usageError, write, } from '@zenera/cli/lib';
2
+ import { command as schema } from "./schema/command.js";
15
3
  // ---------------------------------------------------------------------------
16
- // zen rag — an api description, as something to search
4
+ // zen rag — retrieval, by subject
17
5
  //
18
- // `search` has two modes and neither is the afterthought. Interactively it is
19
- // a loop with prompts; non-interactively it is a tool, and that is the mode
20
- // that has to be exactly specified: every field settable from a flag, the whole
21
- // query settable as one JSON object, a stable `--json` shape, no terminal
22
- // required, and exit 0 when nothing matched — an empty answer is an answer, and
23
- // a caller that has to tell "no results" from "the index is missing" by parsing
24
- // stderr will get it wrong.
6
+ // A subject is a kind of corpus with its own index format, its own verbs and
7
+ // its own flags. They are not variations on one command: what `list` means to
8
+ // an API description is not what it would mean to a folder of notes, and one
9
+ // flag table covering both would be twice as long and half as true.
25
10
  //
26
- // `list`, `grep` and `show` are the other half, and they are deliberately not
27
- // searches. A ranking can only ever hand back the top of a list, so it cannot
28
- // answer "is there a field called `password` anywhere" the honest answer to
29
- // that question is every match or none, and these three give it without asking
30
- // a model or a credential for permission.
11
+ // So the only thing settled here is the subject word. Everything after it
12
+ // belongs to the subject, which is handed its own arguments with the word
13
+ // removed the same contract the `zen` frame gives this command. There is
14
+ // deliberately no spelling that omits the subject: `zen rag search` would have
15
+ // to mean one of them, and whichever was chosen would be wrong for the other
16
+ // forever.
17
+ //
18
+ // `help <subject>` is a verb rather than a flag because `--help` never gets
19
+ // here: the frame lifts it out of the arguments and answers with this page.
31
20
  // ---------------------------------------------------------------------------
32
- const USAGE = 'zen rag schema <index|search|list|grep|show|stats> [spec...]';
33
- const INDEX_USAGE = 'zen rag schema index --embedding <ref> [--out <dir>] <spec...>';
34
- const SEARCH_USAGE = 'zen rag schema search [--dir <dir>] [query...]';
35
- const LIST_USAGE = 'zen rag schema list <methods|types|properties> [--dir <dir>]';
36
- const GREP_USAGE = 'zen rag schema grep <pattern> [--dir <dir>]';
37
- const SHOW_USAGE = 'zen rag schema show [id...] [--method <name>] [--type <name>]';
21
+ const SUBJECTS = { schema };
22
+ const USAGE = 'zen rag <subject> <command> [args...]';
38
23
  export const command = {
39
- summary: 'Search an openapi/swagger document as a graph.',
24
+ summary: 'Retrieval over a corpus: index it, then ask it something.',
40
25
  usage: USAGE,
41
26
  details: [
42
- 'Commands',
43
- ...table([
44
- [' index <spec...>', dim('Read the documents and write a searchable index.')],
45
- [' search', dim('Ask it something. --interactive for a prompt.')],
46
- [' list <what>', dim('Every method, type or property matching a pattern.')],
47
- [' grep <pattern>', dim('Every literal match, ranked by nothing.')],
48
- [' show [id...]', dim('Print named nodes, with no search in between.')],
49
- [' stats', dim('What is in an index, and what built it.')],
50
- ]),
51
- '',
52
- 'Index',
53
- ...table([
54
- [
55
- ' --embedding <ref>',
56
- dim('Which embedder makes the vectors. Omit it to be shown the choices.'),
57
- ],
58
- [
59
- ' -o, --out <dir>',
60
- dim(`Where the index goes. Default ${DEFAULT_DIR}, or ${DIR_ENV}.`),
61
- ],
62
- [
63
- ' --batch <n>',
64
- dim('Texts per embedding request, and how often progress prints. Default 96.'),
65
- ],
66
- [' --no-sources', dim('Do not keep a copy of each document in the index.')],
67
- ]),
68
- '',
69
- 'Search terms (repeatable)',
70
- ...table([
71
- [' <text>', dim('A bare phrase, the same as --all.')],
72
- [' --all <q>', dim('Against everything, unfiltered.')],
73
- [' --method <q>', dim('Operations.')],
74
- [' --type <q>', dim('Schemas, on the side --direction names.')],
75
- [' --input-type <q>', dim('Schemas a call accepts.')],
76
- [' --output-type <q>', dim('Schemas a call returns.')],
77
- [' --property <q>', dim('Fields and parameters, per --direction.')],
78
- [' --input-property <q>', dim('Fields and parameters a call accepts.')],
79
- [' --output-property <q>', dim('Fields a call returns.')],
80
- [' --query <json|->', dim('A whole query object; - reads stdin.')],
81
- ]),
82
- '',
83
- 'Search filters and shape',
84
- ...table([
85
- [' -d, --dir <dir>', dim(`Which index. Found from here if unset; see ${DIR_ENV}.`)],
86
- [' --embedding <ref>', dim('Must be the one the index was built with.')],
87
- [' --direction <d>', dim('input | output | any. Default any.')],
88
- [' --method-type <t>', dim('read_only | read_write | any. Default any.')],
89
- [' --exclude-id <id>', dim('Drop a node. Repeatable, as are the three below.')],
90
- [' --exclude-method <name>', dim('Drop an operation by name.')],
91
- [' --exclude-type <name>', dim('Drop a schema by name.')],
92
- [' --exclude-property <name>', dim('Drop a field by name.')],
93
- [' --limit <n>', dim('Seeds kept per term. Default 5.')],
94
- [' --max-hops <n>', dim('How far apart two hits may be. Default 3.')],
95
- [' --max-nodes <n>', dim('Nodes per result. Default 200.')],
96
- [' --format <f>', dim('text | mermaid | mermaid-flowchart | ts | openapi.')],
97
- [' --show-source', dim('Name the document each operation and schema came from.')],
98
- [' --no-docs', dim('Leave the descriptions out.')],
99
- [' --interactive', dim('Prompt, search, refine. Needs a terminal.')],
100
- [' --quiet', dim('No narration.')],
101
- ]),
102
- '',
103
- 'Exact listing — no embedder, no credential',
104
- ...table([
105
- [' list methods', dim('Operations. Filter with --path and --name.')],
106
- [' list types', dim('Schemas. Filter with --name.')],
107
- [' list properties', dim('Fields and parameters. Filter with --name and --path.')],
108
- [' grep <pattern>', dim('Substring over every node; --regex for a regex.')],
109
- [' --regex', dim('Read every pattern as a regex, list and grep alike.')],
110
- [' --case-sensitive', dim('Match the capitals too.')],
111
- [' --kind <k>', dim('grep: method | type | property. Repeatable.')],
112
- [' --name <p>', dim('grep too: only nodes whose name matches. Repeatable.')],
113
- [' --path <p>', dim('grep too: only what sits on a matching route.')],
114
- [' --ids-only', dim('grep: bare ids, to pipe into show.')],
115
- [' --source <name>', dim('Only this document, as `stats` names it.')],
116
- [' --show-source', dim('Print which document each row came from.')],
117
- [' --limit <n>', dim('Keep at most n; the count still reports them all.')],
118
- ]),
119
- '',
120
- dim(' A pattern with * or ? is a glob over the whole name; otherwise it is'),
121
- dim(' a substring, so --name password finds ResetPasswordPayload. With'),
122
- dim(' --regex it is a regex either way, so --path "^/(users|teams)/" works.'),
27
+ 'Subjects',
28
+ ...table(Object.entries(SUBJECTS).map(([name, sub]) => [` ${name}`, dim(sub.summary)])),
123
29
  '',
124
- 'Show',
125
- ...table([
126
- [' <id...>', dim('Node ids, e.g. Type:User or Property:User.email.')],
127
- [' --method <name>', dim('An operation by name. * to take more. Repeatable.')],
128
- [' --type <name>', dim('A schema by name. * to take more. Repeatable.')],
129
- [' --source <name>', dim('A whole document, as it was indexed.')],
130
- [' --show-source', dim('Name the document each node came from.')],
131
- [' --exact', dim('Only what was named, without the neighbours.')],
132
- ]),
133
- '',
134
- dim(`Without --dir, the index is the nearest one at or above the working`),
135
- dim(`directory; ${cyan(DIR_ENV)} names it outright.`),
30
+ ...Object.keys(SUBJECTS).map((name) => dim(` ${cyan(`zen rag help ${name}`)} — its commands, flags and examples`)),
136
31
  '',
137
32
  dim(`Credentials come from the ${cyan('zen')} keyring — try ${cyan('zen key ls')}.`),
138
33
  ],
139
34
  async run(ctx) {
140
- const [group, ...rest] = ctx.args;
141
- // `schema` is the only subject so far; leaving it out is a courtesy,
142
- // not a second spelling to support forever.
143
- const [name, ...tail] = group === 'schema' ? rest : ctx.args;
144
- switch (name) {
145
- case 'index':
146
- return await index(tail, ctx);
147
- case 'search':
148
- return await search(tail, ctx);
149
- case 'list':
150
- return await list(tail, ctx);
151
- case 'grep':
152
- return await grep(tail, ctx);
153
- case 'show':
154
- return await show(tail, ctx);
155
- case 'stats':
156
- return await stats(tail, ctx);
157
- default:
158
- throw usageError(name ? `unknown command "${name}"` : 'no command given', USAGE);
35
+ const [subject, ...rest] = ctx.args;
36
+ if (subject === 'help') {
37
+ return help(rest[0]);
38
+ }
39
+ const chosen = subject ? SUBJECTS[subject] : undefined;
40
+ if (!chosen) {
41
+ throw usageError(subject ? `unknown subject "${subject}"` : 'no subject given', `expected ${Object.keys(SUBJECTS).join(' or ')} — ${USAGE}`);
159
42
  }
43
+ return await chosen.run({ ...ctx, args: rest });
160
44
  },
161
45
  };
162
- async function index(args, ctx) {
163
- const { values, positionals } = parse(args, {
164
- out: { type: 'string', short: 'o' },
165
- embedding: { type: 'string' },
166
- batch: { type: 'string' },
167
- 'no-sources': { type: 'boolean' },
168
- quiet: { type: 'boolean' },
169
- }, INDEX_USAGE);
170
- if (positionals.length === 0) {
171
- throw usageError('no document given', INDEX_USAGE);
172
- }
173
- const out = outputDir(ctx.cwd, values.out);
174
- const loud = !values.quiet && !ctx.json;
175
- const chosen = await embedder(values.embedding);
176
- const started = Date.now();
177
- const { manifest } = await buildIndex({
178
- files: positionals.map((file) => resolve(ctx.cwd, file)),
179
- out,
180
- embedder: chosen,
181
- embeddingRef: values.embedding,
182
- indexer: 'zenera-rag',
183
- batch: values.batch ? count(values.batch, '--batch') : undefined,
184
- sources: !values['no-sources'],
185
- onRead: loud
186
- ? (summary) => {
187
- printSources(summary.sources);
188
- // The first batch can take a while and says nothing while it
189
- // does; this is the line that makes that a wait, not a hang.
190
- note(dim(` embedding ${summary.counts.entities} entities with ${chosen.id} …`));
191
- }
192
- : undefined,
193
- onProgress: loud
194
- ? (done, total) => note(dim(` embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
195
- : undefined,
196
- });
197
- if (ctx.json) {
198
- json({ out, manifest });
199
- return;
200
- }
201
- // stdout is the path and nothing else, so `DIR=$(zen rag schema index …)`
202
- // works; what it means goes to stderr, where the narration lives.
203
- note();
204
- write(out);
205
- note(` wrote ${bold(String(manifest.counts.entities))} entities to ${bold(out)}, ` +
206
- `embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
207
- const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
208
- note(dim(` search it: ${cyan(`zen rag schema search${where} --all "what you are after"`)}`));
209
- }
210
- const HEADERS = ['PATHS', 'OPERATIONS', 'SCHEMAS', 'FIELDS'];
211
- function elapsed(since) {
212
- const seconds = Math.round((Date.now() - since) / 1000);
213
- return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
214
- }
215
- function printSources(sources) {
216
- const rows = sources.map((s) => ({
217
- name: s.file,
218
- dialect: s.dialect,
219
- cells: [s.paths, s.methods, s.types, s.properties],
220
- }));
221
- if (rows.length > 1) {
222
- rows.push({
223
- name: 'total',
224
- dialect: '',
225
- cells: HEADERS.map((_, i) => rows.reduce((n, r) => n + (r.cells[i] ?? 0), 0)),
226
- });
227
- }
228
- // Numbers are padded before they are styled: a colour code has no width,
229
- // and `table` cannot know that.
230
- const widths = HEADERS.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r.cells[i]).length)));
231
- note();
232
- notes(table([
233
- [bold('SPEC'), bold('DIALECT'), ...HEADERS.map((h, i) => bold(h.padStart(widths[i])))],
234
- ...rows.map((r) => [
235
- r.name === 'total' ? dim(r.name) : r.name,
236
- dim(r.dialect),
237
- ...r.cells.map((c, i) => String(c).padStart(widths[i])),
238
- ]),
239
- ]).map((line) => ` ${line}`));
240
- note();
241
- }
242
- const MANY = { type: 'string', multiple: true };
243
- const SEARCH_OPTIONS = {
244
- dir: { type: 'string', short: 'd' },
245
- embedding: { type: 'string' },
246
- all: MANY,
247
- method: MANY,
248
- type: MANY,
249
- 'input-type': MANY,
250
- 'output-type': MANY,
251
- property: MANY,
252
- 'input-property': MANY,
253
- 'output-property': MANY,
254
- query: { type: 'string' },
255
- direction: { type: 'string' },
256
- 'method-type': { type: 'string' },
257
- 'exclude-id': MANY,
258
- 'exclude-method': MANY,
259
- 'exclude-type': MANY,
260
- 'exclude-property': MANY,
261
- limit: { type: 'string' },
262
- 'max-hops': { type: 'string' },
263
- 'max-nodes': { type: 'string' },
264
- format: { type: 'string' },
265
- 'no-docs': { type: 'boolean' },
266
- 'only-hits': { type: 'boolean' },
267
- 'show-source': { type: 'boolean' },
268
- interactive: { type: 'boolean' },
269
- quiet: { type: 'boolean' },
270
- };
271
- async function search(args, ctx) {
272
- const { values, positionals } = parse(args, SEARCH_OPTIONS, SEARCH_USAGE);
273
- const dir = indexDir(ctx, values.dir);
274
- const format = formatOf(values.format);
275
- const options = {
276
- docs: !values['no-docs'],
277
- onlyHits: values['only-hits'],
278
- source: values['show-source'],
279
- };
280
- const query = { ...(await fromStdin(values.query)), ...fromFlags(values, positionals) };
281
- // Everything that can be wrong about the invocation is settled before a
282
- // credential is asked for, so a typo is a usage error and not a login.
283
- if (values.interactive && !isInteractive()) {
284
- throw usageError('--interactive needs a terminal', SEARCH_USAGE);
285
- }
286
- if (!values.interactive && isEmpty(query)) {
287
- throw usageError('no query given', SEARCH_USAGE);
288
- }
289
- const manifest = await readManifest(dir);
290
- const ref = values.embedding ?? manifest.embedding.ref;
291
- assertSameEmbedding(manifest, ref);
292
- const index = await SchemaIndex.open(dir, await embedder(ref));
293
- try {
294
- if (values.interactive) {
295
- await repl(index, query, { format, ...options });
296
- return;
297
- }
298
- const result = await index.search(query);
299
- if (ctx.json) {
300
- json({
301
- seeds: result.seeds,
302
- empty: result.empty,
303
- subgraphs: result.subgraphs,
304
- rendered: await present(index, result.subgraphs, format, options),
305
- });
306
- return;
307
- }
308
- const text = await present(index, result.subgraphs, format, options);
309
- if (text) {
310
- write(text);
311
- }
312
- if (!values.quiet) {
313
- note(dim(` ${result.seeds.length} seed(s) · ${result.subgraphs.length} result(s)${result.empty.length > 0 ? ` · nothing for: ${result.empty.join(', ')}` : ''}`));
314
- }
46
+ /** Laid out as `zen help <command>` lays out this one, so the two pages match. */
47
+ function help(subject) {
48
+ const chosen = subject ? SUBJECTS[subject] : undefined;
49
+ if (!chosen) {
50
+ throw usageError(subject ? `unknown subject "${subject}"` : 'which subject', `expected ${Object.keys(SUBJECTS).join(' or ')} — zen rag help <subject>`);
315
51
  }
316
- finally {
317
- index.close();
318
- }
319
- }
320
- /** Flags win over `--query`: the more specific spelling is the later thought. */
321
- function fromFlags(values, positionals = []) {
322
- const query = {};
323
- const put = (key, value) => {
324
- if (value !== undefined && (!Array.isArray(value) || value.length > 0)) {
325
- query[key] = value;
52
+ write(bold(chosen.usage));
53
+ write(`\n ${chosen.summary}`);
54
+ if (chosen.details?.length) {
55
+ write('');
56
+ for (const line of chosen.details) {
57
+ write(line ? ` ${line}` : '');
326
58
  }
327
- };
328
- // A bare phrase is the unfiltered search; there is nothing else it could mean.
329
- put('all', [...(values.all ?? []), ...positionals]);
330
- put('methods', values.method);
331
- put('types', values.type);
332
- put('input_types', values['input-type']);
333
- put('output_types', values['output-type']);
334
- put('properties', values.property);
335
- put('input_properties', values['input-property']);
336
- put('output_properties', values['output-property']);
337
- put('exclude_ids', values['exclude-id']);
338
- put('exclude_methods', values['exclude-method']);
339
- put('exclude_types', values['exclude-type']);
340
- put('exclude_properties', values['exclude-property']);
341
- put('direction', values.direction);
342
- put('method_type', values['method-type']);
343
- put('limit', values.limit && count(values.limit, '--limit'));
344
- put('max_hops', values['max-hops'] && count(values['max-hops'], '--max-hops'));
345
- put('max_nodes', values['max-nodes'] && count(values['max-nodes'], '--max-nodes'));
346
- return check(query);
347
- }
348
- async function fromStdin(source) {
349
- if (source === undefined) {
350
- return {};
351
- }
352
- const text = source === '-' ? await readStdin() : source;
353
- let parsed;
354
- try {
355
- parsed = JSON.parse(text);
356
- }
357
- catch (err) {
358
- throw usageError(`--query is not JSON: ${err.message}`, SEARCH_USAGE);
359
- }
360
- return check(parsed);
361
- }
362
- async function readStdin() {
363
- const chunks = [];
364
- for await (const chunk of process.stdin) {
365
- chunks.push(chunk);
366
- }
367
- return Buffer.concat(chunks).toString('utf8');
368
- }
369
- function check(value) {
370
- try {
371
- return parseQuery(value);
372
- }
373
- catch (err) {
374
- if (err instanceof QueryError) {
375
- throw usageError(err.message, SEARCH_USAGE);
376
- }
377
- throw err;
378
- }
379
- }
380
- // ---------------------------------------------------------------------------
381
- // list, grep
382
- //
383
- // The deterministic half. Neither takes an embedder, because neither ranks
384
- // anything: `list` filters on the attributes a node already has and `grep`
385
- // reads the same materialized string the index was built from. What comes back
386
- // is every match, and where a limit cut the list the count still reports the
387
- // total — being shown three of three hundred is only useful if you are told
388
- // which of the two happened.
389
- // ---------------------------------------------------------------------------
390
- const SUBJECTS = {
391
- methods: 'method',
392
- types: 'type',
393
- properties: 'property',
394
- };
395
- async function list(args, ctx) {
396
- const { values, positionals } = parse(args, {
397
- dir: { type: 'string', short: 'd' },
398
- name: MANY,
399
- path: MANY,
400
- source: { type: 'string' },
401
- 'method-type': { type: 'string' },
402
- direction: { type: 'string' },
403
- regex: { type: 'boolean' },
404
- 'case-sensitive': { type: 'boolean' },
405
- 'show-source': { type: 'boolean' },
406
- limit: { type: 'string' },
407
- quiet: { type: 'boolean' },
408
- }, LIST_USAGE);
409
- const subject = positionals[0];
410
- const kind = subject ? SUBJECTS[subject] : undefined;
411
- if (!kind) {
412
- throw usageError(subject ? `cannot list "${subject}"` : 'nothing named to list', `expected one of ${Object.keys(SUBJECTS).join(', ')}`);
413
- }
414
- if (positionals.length > 1) {
415
- throw usageError('one subject at a time', LIST_USAGE);
416
- }
417
- const how = { regex: values.regex, caseSensitive: values['case-sensitive'] };
418
- const index = await openIndex(indexDir(ctx, values.dir));
419
- const found = listNodes(index.graph, {
420
- kind,
421
- name: patterns(values.name, '--name', how),
422
- path: patterns(values.path, '--path', how),
423
- source: values.source,
424
- methodType: oneOf(values['method-type'], ['read_only', 'read_write'], '--method-type'),
425
- direction: oneOf(values.direction, ['input', 'output'], '--direction'),
426
- limit: values.limit ? count(values.limit, '--limit') : undefined,
427
- });
428
- if (ctx.json) {
429
- json({ found: found.found, truncated: found.truncated, rows: found.rows });
430
- return;
431
- }
432
- const lines = rowLines(index.graph, kind, found.rows, values['show-source']);
433
- if (lines.length > 0) {
434
- write(lines.join('\n'));
435
- }
436
- if (!values.quiet) {
437
- note(dim(` ${found.found} ${subject}${shown(found.found, found.rows.length)}`));
438
- }
439
- }
440
- function rowLines(graph, kind, rows, showSource = false) {
441
- const from = (r) => (showSource ? [dim(sourceTag(r.source))] : []);
442
- if (kind === 'method') {
443
- return table(rows.map((r) => [`${r.httpMethod} ${r.path}`, r.name, ...from(r), doc(r.doc)]));
444
- }
445
- if (kind === 'type') {
446
- return table(rows.map((r) => [
447
- r.name,
448
- dim(fields(propertyCount(graph, r.id))),
449
- dim(r.direction === 'none' ? '' : `(${r.direction})`),
450
- ...from(r),
451
- doc(r.doc),
452
- ]));
453
- }
454
- return table(rows.map((r) => [
455
- `${r.parent ? `${r.parent}.` : ''}${r.name}${r.required ? '' : '?'}`,
456
- `: ${r.signature || 'unknown'}`,
457
- ...from(r),
458
- doc(r.doc),
459
- ]));
460
- }
461
- async function grep(args, ctx) {
462
- const { values, positionals } = parse(args, {
463
- dir: { type: 'string', short: 'd' },
464
- regex: { type: 'boolean' },
465
- 'case-sensitive': { type: 'boolean' },
466
- kind: MANY,
467
- name: MANY,
468
- path: MANY,
469
- source: { type: 'string' },
470
- 'show-source': { type: 'boolean' },
471
- limit: { type: 'string' },
472
- 'ids-only': { type: 'boolean' },
473
- quiet: { type: 'boolean' },
474
- }, GREP_USAGE);
475
- if (positionals.length === 0) {
476
- throw usageError('no pattern given', GREP_USAGE);
477
- }
478
- if (positionals.length > 1) {
479
- throw usageError('one pattern at a time — quote it if it has spaces', GREP_USAGE);
480
- }
481
- const kinds = (values.kind ?? []).map((k) => oneOf(k, ['method', 'type', 'property'], '--kind'));
482
- // The pattern is read as the flags say; the constraints are always names,
483
- // so they stay globs-or-substrings even under --regex on the pattern.
484
- const how = { caseSensitive: values['case-sensitive'] };
485
- const index = await openIndex(indexDir(ctx, values.dir));
486
- const result = pattern(() => grepNodes(index.graph, matcher(positionals[0], {
487
- regex: values.regex,
488
- caseSensitive: values['case-sensitive'],
489
- }), {
490
- kinds,
491
- source: values.source,
492
- name: patterns(values.name, '--name', how),
493
- path: patterns(values.path, '--path', how),
494
- limit: values.limit ? count(values.limit, '--limit') : undefined,
495
- }));
496
- if (ctx.json) {
497
- json({
498
- found: result.found,
499
- truncated: result.truncated,
500
- matches: result.matches.map((m) => ({ id: m.id, ...m.attributes, text: m.text })),
501
- });
502
- return;
503
- }
504
- if (result.matches.length > 0) {
505
- const lines = values['ids-only']
506
- ? result.matches.map((m) => m.id)
507
- : table(result.matches.map((m) => [
508
- m.id,
509
- ...(values['show-source'] ? [dim(sourceTag(m.attributes.source))] : []),
510
- dim(clip(m.text, 140)),
511
- ]));
512
- write(lines.join('\n'));
513
- }
514
- if (!values.quiet && !values['ids-only']) {
515
- note(dim(` ${result.found} match(es)${shown(result.found, result.matches.length)}`));
516
- }
517
- }
518
- // ---------------------------------------------------------------------------
519
- const shown = (found, kept) => (kept < found ? `, showing ${kept}` : '');
520
- /**
521
- * Where the index is, said out loud when nobody named it. Finding one and not
522
- * saying which would make every answer here unattributable.
523
- */
524
- function indexDir(ctx, flag) {
525
- const { dir, from } = locateIndex(ctx.cwd, flag);
526
- if (from === 'found' && !ctx.json) {
527
- note(dim(` using ${relative(ctx.cwd, dir) || dir}`));
528
- }
529
- return dir;
530
- }
531
- function patterns(values, flag, options = {}) {
532
- if (!values || values.length === 0) {
533
- return undefined;
534
- }
535
- return values.map((p) => pattern(() => loose(p, options), flag));
536
- }
537
- /** A bad pattern is a bad invocation, not a failure of the index. */
538
- function pattern(run, flag) {
539
- try {
540
- return run();
541
- }
542
- catch (err) {
543
- if (err instanceof PatternError) {
544
- throw usageError(`${flag ? `${flag}: ` : ''}${err.message}`, USAGE);
545
- }
546
- throw err;
547
- }
548
- }
549
- function oneOf(value, allowed, flag) {
550
- if (value === undefined || value === 'any') {
551
- return undefined;
552
- }
553
- if (!allowed.includes(value)) {
554
- throw usageError(`${flag} cannot be "${value}"`, `expected ${allowed.join(' or ')}`);
555
- }
556
- return value;
557
- }
558
- const doc = (text) => (text ? dim(`— ${clip(text.replace(/\s+/g, ' '), 90)}`) : '');
559
- const clip = (text, max) => text.length <= max ? text : `${text.slice(0, max - 1)}…`;
560
- /**
561
- * No embedder and no store: naming a node is a graph lookup, and asking for a
562
- * credential to print something already on disk would be theatre.
563
- *
564
- * Ids are the precise way in, and `--method`/`--type` are the way in for
565
- * someone who has a name rather than an id — which, with `--format openapi
566
- * --exact`, is how a resolved slice of the document is got out.
567
- */
568
- async function show(args, ctx) {
569
- const { values, positionals } = parse(args, {
570
- dir: { type: 'string', short: 'd' },
571
- method: MANY,
572
- type: MANY,
573
- source: { type: 'string' },
574
- exact: { type: 'boolean' },
575
- format: { type: 'string' },
576
- 'max-nodes': { type: 'string' },
577
- 'no-docs': { type: 'boolean' },
578
- 'show-source': { type: 'boolean' },
579
- quiet: { type: 'boolean' },
580
- }, SHOW_USAGE);
581
- const format = formatOf(values.format);
582
- const dir = indexDir(ctx, values.dir);
583
- // A whole document, verbatim: the copy kept at index time is the resolved
584
- // original, and anything rebuilt from the graph would be a paraphrase.
585
- if (values.source && format === 'openapi' && positionals.length === 0 && !named(values)) {
586
- const document = await readSource(dir, values.source);
587
- if (document) {
588
- write(document);
589
- return;
590
- }
591
- if (!values.quiet) {
592
- note(dim(' this index kept no copy of the documents — rebuilding it from the graph'));
593
- }
594
- }
595
- const index = await openIndex(dir);
596
- const ids = resolveIds(index.graph, positionals, values);
597
- const subgraphs = values.exact
598
- ? [select(index.graph, ids)]
599
- : stitch(index.graph, ids.map((id) => ({ id, term: id, field: 'show', score: 1 })), {
600
- maxNodes: values['max-nodes']
601
- ? count(values['max-nodes'], '--max-nodes')
602
- : undefined,
603
- });
604
- const text = await present(index, subgraphs, format, {
605
- docs: !values['no-docs'],
606
- source: values['show-source'],
607
- });
608
- if (ctx.json) {
609
- json({ ids, subgraphs, rendered: text });
610
- }
611
- else if (text) {
612
- write(text);
613
- }
614
- }
615
- const named = (values) => Boolean(values.method?.length || values.type?.length);
616
- /** Ids as given, plus whatever the name selectors resolve to. */
617
- function resolveIds(graph, ids, values) {
618
- if (ids.length === 0 && !named(values) && !values.source) {
619
- throw usageError('no node named', SHOW_USAGE);
620
- }
621
- const missing = ids.filter((id) => !graph.hasNode(id));
622
- if (missing.length > 0) {
623
- throw new CliError(`no such node: ${missing.join(', ')}`, EXIT.failed, 'ids look like `Type:User` or `Property:User.email`');
624
- }
625
- const out = new Set(ids);
626
- for (const kind of ['method', 'type']) {
627
- for (const wanted of values[kind] ?? []) {
628
- // Selecting, not searching: a bare name means that name. A star is
629
- // the way to ask for more than one.
630
- const match = isGlob(wanted)
631
- ? pattern(() => wildcard(wanted), `--${kind}`)
632
- : (name) => name === wanted;
633
- const rows = listNodes(graph, { kind, name: [match], source: values.source });
634
- // A selector that matched nothing is a wrong answer, not an empty
635
- // one: the caller named something they believe is there.
636
- if (rows.found === 0) {
637
- throw new CliError(`no ${kind} called ${wanted}`, EXIT.failed, `try: zen rag schema list ${kind}s --name "${wanted}"`);
638
- }
639
- for (const row of rows.rows) {
640
- out.add(row.id);
641
- }
642
- }
643
- }
644
- // `--source` on its own means the whole document.
645
- if (out.size === 0 && values.source) {
646
- for (const kind of ['method', 'type']) {
647
- for (const row of listNodes(graph, { kind, source: values.source }).rows) {
648
- out.add(row.id);
649
- }
650
- }
651
- if (out.size === 0) {
652
- throw new CliError(`nothing in this index came from ${values.source}`, EXIT.failed);
653
- }
654
- }
655
- return [...out];
656
- }
657
- async function stats(args, ctx) {
658
- const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag schema stats [--dir <dir>]');
659
- const dir = indexDir(ctx, values.dir);
660
- const manifest = await readManifest(dir);
661
- if (ctx.json) {
662
- json(manifest);
663
- return;
664
- }
665
- note(bold(dir));
666
- notes(table([
667
- [' built', manifest.createdAt],
668
- [' by', manifest.indexer],
669
- [' embedder', `${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`],
670
- [
671
- ' indexes',
672
- `fts ${yes(manifest.indexes.fts)} · vector ${yes(manifest.indexes.vector)}`,
673
- ],
674
- ]));
675
- printSources(manifest.sources);
676
- notes(table([[' entities', String(manifest.counts.entities)]]).map(dim));
677
- }
678
- function notes(lines) {
679
- for (const line of lines) {
680
- note(line);
681
- }
682
- }
683
- const yes = (value) => (value ? 'yes' : 'no');
684
- // ---------------------------------------------------------------------------
685
- /** The keyring is materialised here, and only here: `show` and `stats` read no vectors. */
686
- async function embedder(ref) {
687
- ensureHome();
688
- const keys = await KeyStore.open();
689
- // Asked before materialising, because materialising is what erases the
690
- // difference between "the environment had it" and "the keyring supplied it".
691
- const fromEnv = new Set(PROVIDERS.filter((p) => envNames(p).some((n) => process.env[n])));
692
- keys.materialize();
693
- if (!ref) {
694
- throw choices(keys, fromEnv);
695
- }
696
- return createEmbedder(ref);
697
- }
698
- /**
699
- * Well-known embedding models per provider, read off the CLI's catalog table so
700
- * there is one list rather than two that drift. Any ref the registry can parse
701
- * works; these are the ones worth typing. Anthropic has none because it
702
- * publishes no embeddings API at all.
703
- */
704
- const embeddingsOf = (provider) => CURATED[provider].filter((m) => m.roles.includes('embedding')).map((m) => m.id);
705
- /** What could be passed, with the ones this machine can actually use first. */
706
- function choices(keys, fromEnv) {
707
- const rows = [];
708
- const rest = [];
709
- for (const provider of PROVIDERS) {
710
- for (const model of embeddingsOf(provider)) {
711
- const source = fromEnv.has(provider)
712
- ? 'environment'
713
- : keys.active(provider)
714
- ? 'keyring'
715
- : '';
716
- const row = [` ${cyan(`${provider}:${model}`)}`, dim(source || form(provider).env)];
717
- (source ? rows : rest).push(row);
718
- }
719
- }
720
- note(bold('Embeddings'));
721
- notes(table([...rows, ...rest]));
722
- note('');
723
- if (rows.length === 0) {
724
- note(dim(' no provider on this machine has a credential — try: zen key add openai'));
725
- note('');
726
- }
727
- // `pick` is the one that ends the question rather than restating it: it
728
- // tries them and prints the first that answers.
729
- return usageError('no embedder named', 'pass --embedding <ref>, or run: zen models pick --embedding');
730
- }
731
- function formatOf(value) {
732
- if (value === undefined) {
733
- return 'text';
734
- }
735
- if (!isFormat(value)) {
736
- throw usageError(`unknown format "${value}"`, 'expected text, mermaid, mermaid-flowchart, ts or openapi');
737
- }
738
- return value;
739
- }
740
- function count(value, flag) {
741
- const number = Number(value);
742
- if (!Number.isInteger(number) || number < 1) {
743
- throw usageError(`${flag} must be a whole number of at least 1`, USAGE);
744
59
  }
745
- return number;
746
60
  }
747
61
  //# sourceMappingURL=command.js.map