@zenera/rag 1.1.24 → 1.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/dist/common/progress.js +6 -48
- package/dist/docs/command.js +93 -12
- package/dist/docs/load.js +1 -1
- package/dist/docs/lookup.d.ts +54 -1
- package/dist/docs/lookup.js +120 -1
- package/dist/docs/render.d.ts +6 -0
- package/dist/docs/render.js +33 -0
- package/dist/docs/search.d.ts +1 -1
- package/dist/docs/search.js +1 -1
- package/dist/docs/targets.d.ts +7 -0
- package/dist/docs/targets.js +52 -0
- package/dist/docs/tools.js +69 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/present.d.ts +17 -0
- package/dist/present.js +40 -0
- package/dist/query.d.ts +7 -0
- package/dist/query.js +88 -0
- package/dist/repl.d.ts +8 -0
- package/dist/repl.js +119 -0
- package/dist/schema/command.js +1 -1
- package/dist/schema/locate.d.ts +20 -0
- package/dist/schema/locate.js +141 -0
- package/dist/schema/lookup.d.ts +1 -1
- package/dist/schema/lookup.js +1 -1
- package/dist/schema/match.d.ts +40 -0
- package/dist/schema/match.js +92 -0
- package/dist/schema/progress.d.ts +26 -0
- package/dist/schema/progress.js +316 -0
- package/dist/schema/tools.js +1 -2
- package/dist/schema/trace.d.ts +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -313,7 +313,7 @@ zen rag docs ready Can this index answer? Exit 0 if it can, 3 if i
|
|
|
313
313
|
zen rag docs search [text] Ask it something. --interactive for a prompt.
|
|
314
314
|
zen rag docs list <what> Every document, section or table. No ranking.
|
|
315
315
|
zen rag docs grep <pattern> Every matching line, with the section it sits in.
|
|
316
|
-
zen rag docs show <
|
|
316
|
+
zen rag docs show <target...> Documents, sections or line ranges, several in one call.
|
|
317
317
|
zen rag docs stats What is in an index, and what built it.
|
|
318
318
|
```
|
|
319
319
|
|
|
@@ -377,11 +377,19 @@ zen rag docs list tables # every table, with its columns
|
|
|
377
377
|
zen rag docs grep "Retry-After" # every matching line, and its section
|
|
378
378
|
zen rag docs show api/routing.md --section "Rate limits"
|
|
379
379
|
zen rag docs show api/routing.md --lines 40-80
|
|
380
|
+
zen rag docs show "api/routing.md:24-31" "api/errors.md#Retries"
|
|
380
381
|
```
|
|
381
382
|
|
|
382
383
|
`grep` counts every match, not the top of a list, so unlike a search it can
|
|
383
384
|
answer whether a string appears at all.
|
|
384
385
|
|
|
386
|
+
`show` takes any number of targets written the way a citation is -
|
|
387
|
+
`<document>:<a>-<b>`, `<document>:<a>`, `<document>#<heading>`, or a bare name -
|
|
388
|
+
and prints each under a header that is itself a target, so reading every
|
|
389
|
+
passage an answer cites costs one call. Overlapping ranges print once, a
|
|
390
|
+
missing target is reported in place, and `--max-lines` (default 400) is shared
|
|
391
|
+
between them, each cut block naming the target that continues it.
|
|
392
|
+
|
|
385
393
|
<details>
|
|
386
394
|
<summary>Every flag</summary>
|
|
387
395
|
|
|
@@ -559,12 +567,12 @@ Only the first ranks. `find_types_with_property` is the one for the repair loop
|
|
|
559
567
|
|
|
560
568
|
Four tools in the group `docs`, selectable as `docs:*`:
|
|
561
569
|
|
|
562
|
-
| Tool | For
|
|
563
|
-
| ------------- |
|
|
564
|
-
| `search_docs` | the passages that match, quoted with their line numbers
|
|
565
|
-
| `list_docs` | the documents, their headings, or their tables - no search
|
|
566
|
-
| `grep_docs` | every matching line, counted in full - no search
|
|
567
|
-
| `read_docs` | a section or a line range, verbatim
|
|
570
|
+
| Tool | For |
|
|
571
|
+
| ------------- | ---------------------------------------------------------------- |
|
|
572
|
+
| `search_docs` | the passages that match, quoted with their line numbers |
|
|
573
|
+
| `list_docs` | the documents, their headings, or their tables - no search |
|
|
574
|
+
| `grep_docs` | every matching line, counted in full - no search |
|
|
575
|
+
| `read_docs` | a section or a line range, verbatim; several at once via `reads` |
|
|
568
576
|
|
|
569
577
|
`search_docs` is the way in when the question is vague; `grep_docs` is how "it
|
|
570
578
|
is not in here" can actually be concluded. Every answer carries line numbers and
|
package/dist/common/progress.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { claimLock, ownLock } from '@zenera/neo';
|
|
3
|
+
import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { basename, join } from 'node:path';
|
|
5
5
|
// ---------------------------------------------------------------------------
|
|
6
6
|
// A build that says what it is doing
|
|
@@ -32,14 +32,12 @@ export const INTERVAL_MS = 5000;
|
|
|
32
32
|
export function beginBuild(plan) {
|
|
33
33
|
const documents = plan.files.map((file) => basename(file));
|
|
34
34
|
const started = Date.now();
|
|
35
|
-
const lock = {
|
|
36
|
-
pid: process.pid,
|
|
37
|
-
host: hostname(),
|
|
35
|
+
const lock = ownLock({
|
|
38
36
|
startedAt: new Date(started).toISOString(),
|
|
39
37
|
indexer: plan.indexer,
|
|
40
38
|
embedding: plan.embedding,
|
|
41
39
|
documents,
|
|
42
|
-
};
|
|
40
|
+
});
|
|
43
41
|
mkdirSync(plan.dir, { recursive: true });
|
|
44
42
|
claim(join(plan.dir, LOCK_FILE), lock);
|
|
45
43
|
let phase = Object.keys(plan.phases)[0];
|
|
@@ -140,48 +138,8 @@ export function beginBuild(plan) {
|
|
|
140
138
|
},
|
|
141
139
|
};
|
|
142
140
|
}
|
|
143
|
-
/**
|
|
144
|
-
* `wx` makes the create and the check one operation, so two builds racing for
|
|
145
|
-
* the same directory cannot both win. A lock whose process is gone is stale by
|
|
146
|
-
* definition and is taken rather than respected — a crashed build must not make
|
|
147
|
-
* a directory permanently unbuildable.
|
|
148
|
-
*/
|
|
141
|
+
/** Two builds writing one index would interleave their store writes. */
|
|
149
142
|
function claim(path, lock) {
|
|
150
|
-
|
|
151
|
-
try {
|
|
152
|
-
writeFileSync(path, body, { flag: 'wx' });
|
|
153
|
-
return;
|
|
154
|
-
}
|
|
155
|
-
catch (err) {
|
|
156
|
-
if (err.code !== 'EEXIST') {
|
|
157
|
-
throw err;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
const held = readLock(path);
|
|
161
|
-
if (held && held.host === hostname() && alive(held.pid)) {
|
|
162
|
-
throw new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`);
|
|
163
|
-
}
|
|
164
|
-
writeFileSync(path, body);
|
|
165
|
-
}
|
|
166
|
-
function readLock(path) {
|
|
167
|
-
try {
|
|
168
|
-
return JSON.parse(readFileSync(path, 'utf8'));
|
|
169
|
-
}
|
|
170
|
-
catch {
|
|
171
|
-
return undefined;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
/**
|
|
175
|
-
* `kill(pid, 0)` sends no signal and only asks whether the process exists.
|
|
176
|
-
* EPERM means it exists and belongs to someone else, which still counts.
|
|
177
|
-
*/
|
|
178
|
-
function alive(pid) {
|
|
179
|
-
try {
|
|
180
|
-
process.kill(pid, 0);
|
|
181
|
-
return true;
|
|
182
|
-
}
|
|
183
|
-
catch (err) {
|
|
184
|
-
return err.code === 'EPERM';
|
|
185
|
-
}
|
|
143
|
+
claimLock(path, lock, (held) => new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`));
|
|
186
144
|
}
|
|
187
145
|
//# sourceMappingURL=progress.js.map
|
package/dist/docs/command.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
|
|
2
|
+
import { PatternError } from '@zenera/neo';
|
|
2
3
|
import { existsSync } from 'node:fs';
|
|
3
4
|
import { join, relative, resolve } from 'node:path';
|
|
4
5
|
import { resolveEmbedder } from "../common/embedder.js";
|
|
5
6
|
import { locateIndex, outputDir } from "../common/locate.js";
|
|
6
7
|
import { assertSameEmbedding, inspectIndex } from "../common/manifest.js";
|
|
7
|
-
import { PatternError } from "../common/match.js";
|
|
8
8
|
import { INTERVAL_MS } from "../common/progress.js";
|
|
9
9
|
import { breakdown, grid, indexing } from "../common/prose.js";
|
|
10
10
|
import { announce, reportReady, stagingFor, swapIn } from "../common/restore.js";
|
|
@@ -13,11 +13,12 @@ import { buildIndex } from "./build.js";
|
|
|
13
13
|
import { CHUNK_KINDS } from "./chunk.js";
|
|
14
14
|
import { DOCS_INDEX, readManifest, SOURCES_DIR } from "./files.js";
|
|
15
15
|
import { DOC_EXTENSIONS } from "./load.js";
|
|
16
|
-
import { grepLines, listFiles, listSections, listTables, readRange, readSection, } from "./lookup.js";
|
|
17
|
-
import { MATCH_HEADERS, matchRows, renderAssembly } from "./render.js";
|
|
16
|
+
import { grepLines, isFailure, listFiles, listSections, listTables, readMany, readRange, readSection, spanOf, } from "./lookup.js";
|
|
17
|
+
import { MATCH_HEADERS, matchRows, renderAssembly, renderReads } from "./render.js";
|
|
18
18
|
import { repl } from "./repl.js";
|
|
19
19
|
import { DEFAULT_LIMIT, DocsIndex, legsOf, SEARCH_MODES, } from "./search.js";
|
|
20
20
|
import { ChunkStore } from "./store.js";
|
|
21
|
+
import { parseTarget, selects, TargetError } from "./targets.js";
|
|
21
22
|
const { defaultDir: DEFAULT_DIR, envName: DIR_ENV } = DOCS_INDEX;
|
|
22
23
|
// ---------------------------------------------------------------------------
|
|
23
24
|
// zen rag docs — a pile of documents, as something to search
|
|
@@ -49,7 +50,7 @@ const READY_USAGE = 'zen rag docs ready [--dir <dir>] [--quiet]';
|
|
|
49
50
|
const SEARCH_USAGE = 'zen rag docs search [--dir <dir>] [query... | --text-query <text> --vector-query <text>]';
|
|
50
51
|
const LIST_USAGE = 'zen rag docs list <files|sections|tables> [--dir <dir>]';
|
|
51
52
|
const GREP_USAGE = 'zen rag docs grep <pattern> [--dir <dir>]';
|
|
52
|
-
const SHOW_USAGE = 'zen rag docs show <file> [--section <name>] [--lines <from-to>]';
|
|
53
|
+
const SHOW_USAGE = 'zen rag docs show <file> [--section <name>] [--lines <from-to>] | <target>... [--max-lines <n>]';
|
|
53
54
|
export const command = {
|
|
54
55
|
summary: 'Search a pile of markdown and text documents.',
|
|
55
56
|
usage: USAGE,
|
|
@@ -78,8 +79,8 @@ export const command = {
|
|
|
78
79
|
dim('Find every literal or regex line match; use it to prove whether text exists.'),
|
|
79
80
|
],
|
|
80
81
|
[
|
|
81
|
-
' show <
|
|
82
|
-
dim('Read
|
|
82
|
+
' show <target...>',
|
|
83
|
+
dim('Read documents, headings or line ranges verbatim, several in one call.'),
|
|
83
84
|
],
|
|
84
85
|
[' stats', dim('Inspect the index contents, size and embedding configuration.')],
|
|
85
86
|
]),
|
|
@@ -197,9 +198,20 @@ export const command = {
|
|
|
197
198
|
[' <file>', dim('A document name, as `list files` prints it.')],
|
|
198
199
|
[' --section <name>', dim('Just that heading and what nests under it.')],
|
|
199
200
|
[' --lines <from-to>', dim('Just those lines, e.g. --lines 40-80.')],
|
|
201
|
+
[' <doc>:<a>-<b>', dim('Lines a to b. <doc>:<a> is one line.')],
|
|
202
|
+
[' <doc>#<heading>', dim('A heading and what nests under it.')],
|
|
203
|
+
[
|
|
204
|
+
' --max-lines <n>',
|
|
205
|
+
dim(`A ceiling shared by every target. Default ${DEFAULT_MAX_LINES}.`),
|
|
206
|
+
],
|
|
200
207
|
[' --no-numbers', dim('Without the line-number gutter.')],
|
|
201
208
|
]),
|
|
202
209
|
'',
|
|
210
|
+
dim(' Name every passage you will read in one call, as the search printed them:'),
|
|
211
|
+
dim(` ${cyan('zen rag docs show "api/routing.md:24-60" "api/limits.md#Rate limits"')}`),
|
|
212
|
+
dim(' Each prints under a header that is itself a target, in the order given;'),
|
|
213
|
+
dim(' overlapping ranges print once. --section and --lines take one document.'),
|
|
214
|
+
'',
|
|
203
215
|
dim(`Without --dir, the index is the nearest one at or above the working`),
|
|
204
216
|
dim(`directory; ${cyan(DIR_ENV)} names it outright.`),
|
|
205
217
|
'',
|
|
@@ -615,32 +627,56 @@ async function show(args, ctx) {
|
|
|
615
627
|
dir: { type: 'string', short: 'd' },
|
|
616
628
|
section: { type: 'string', short: 's' },
|
|
617
629
|
lines: { type: 'string' },
|
|
630
|
+
'max-lines': { type: 'string' },
|
|
618
631
|
'no-numbers': { type: 'boolean' },
|
|
619
632
|
quiet: { type: 'boolean' },
|
|
620
633
|
}, SHOW_USAGE);
|
|
621
|
-
|
|
622
|
-
if (!name) {
|
|
634
|
+
if (positionals.length === 0) {
|
|
623
635
|
throw usageError('no document named', SHOW_USAGE);
|
|
624
636
|
}
|
|
637
|
+
const flagged = values.section !== undefined || values.lines !== undefined;
|
|
638
|
+
if (flagged && positionals.length > 1) {
|
|
639
|
+
throw usageError('--section and --lines read one document', 'name each part instead: <document>:<a>-<b> or <document>#<heading>');
|
|
640
|
+
}
|
|
641
|
+
const span = values.lines ? range(values.lines) : undefined;
|
|
642
|
+
const maxLines = values['max-lines'] ? count(values['max-lines'], '--max-lines') : undefined;
|
|
625
643
|
const found = await open(ctx, values.dir);
|
|
626
644
|
try {
|
|
645
|
+
const names = new Set(found.manifest.sources.map((s) => s.name));
|
|
646
|
+
const several = !flagged && (positionals.length > 1 || targeted(() => selects(positionals[0], names)));
|
|
647
|
+
if (several) {
|
|
648
|
+
const reads = targeted(() => positionals.map((raw) => parseTarget(raw, names)));
|
|
649
|
+
await showMany(ctx, found, reads, values, maxLines ?? DEFAULT_MAX_LINES);
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
const name = positionals[0];
|
|
627
653
|
const file = found.resolveFiles([name])[0];
|
|
628
654
|
if (!file) {
|
|
629
655
|
throw new CliError(`no document called ${name}`, EXIT.failed, 'list them with `zen rag docs list files`');
|
|
630
656
|
}
|
|
631
|
-
const
|
|
632
|
-
const excerpt = values.section
|
|
657
|
+
const whole = values.section
|
|
633
658
|
? await readSection(found, file, values.section)
|
|
634
659
|
: await readRange(found, file, span?.[0] ?? 1, span?.[1] ?? Infinity);
|
|
660
|
+
// Uncapped unless asked: this form printed whole documents before the cap existed.
|
|
661
|
+
const cut = maxLines !== undefined && whole.lines.length > maxLines;
|
|
662
|
+
const excerpt = cut
|
|
663
|
+
? { ...whole, lines: whole.lines.slice(0, maxLines), end: whole.start + maxLines - 1 }
|
|
664
|
+
: whole;
|
|
635
665
|
if (ctx.json) {
|
|
636
|
-
json(excerpt);
|
|
666
|
+
json(cut ? { ...excerpt, truncated: true } : excerpt);
|
|
637
667
|
return;
|
|
638
668
|
}
|
|
639
|
-
const width = String(
|
|
669
|
+
const width = String(whole.end).length;
|
|
640
670
|
write(excerpt.lines
|
|
641
671
|
.map((line, at) => values['no-numbers']
|
|
642
672
|
? line
|
|
643
673
|
: `${dim(String(excerpt.start + at).padStart(width))} ${dim('|')} ${line}`)
|
|
674
|
+
.concat(cut
|
|
675
|
+
? [
|
|
676
|
+
dim(`... truncated at line ${excerpt.end} of ${whole.end} - read ` +
|
|
677
|
+
`${spanOf(file, excerpt.end + 1, whole.end)} to continue`),
|
|
678
|
+
]
|
|
679
|
+
: [])
|
|
644
680
|
.join('\n'));
|
|
645
681
|
if (!values.quiet) {
|
|
646
682
|
note(dim(` ${file} · lines ${excerpt.start}-${excerpt.end} of ${excerpt.total}`));
|
|
@@ -650,6 +686,51 @@ async function show(args, ctx) {
|
|
|
650
686
|
found.close();
|
|
651
687
|
}
|
|
652
688
|
}
|
|
689
|
+
async function showMany(ctx, found, reads, values, maxLines) {
|
|
690
|
+
const read = await patterned(() => readMany(found, reads, { maxLines }));
|
|
691
|
+
if (ctx.json) {
|
|
692
|
+
json({
|
|
693
|
+
results: read.results.map((result) => isFailure(result)
|
|
694
|
+
? result
|
|
695
|
+
: {
|
|
696
|
+
target: result.target,
|
|
697
|
+
file: result.file,
|
|
698
|
+
from: result.from,
|
|
699
|
+
to: result.to,
|
|
700
|
+
section: result.section ?? null,
|
|
701
|
+
text: result.lines.join('\n'),
|
|
702
|
+
truncated: result.truncated,
|
|
703
|
+
...(result.continue ? { continue: result.continue } : {}),
|
|
704
|
+
}),
|
|
705
|
+
printed: read.printed,
|
|
706
|
+
failed: read.failed,
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
else {
|
|
710
|
+
write(renderReads(read, { numbers: !values['no-numbers'] }));
|
|
711
|
+
if (!values.quiet) {
|
|
712
|
+
const lines = read.results.reduce((sum, r) => sum + (isFailure(r) ? 0 : r.lines.length), 0);
|
|
713
|
+
const cut = read.results.some((r) => !isFailure(r) && r.truncated);
|
|
714
|
+
note(dim(` ${read.printed} of ${read.results.length} target(s) · ${lines} line(s)` +
|
|
715
|
+
(cut ? ` · cut to --max-lines ${maxLines}` : '')));
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
if (read.printed === 0) {
|
|
719
|
+
throw new CliError('none of the targets could be read', EXIT.failed, '`zen rag docs list files` lists the document names');
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
/** A malformed target is a bad invocation, so nothing is printed for any of them. */
|
|
723
|
+
function targeted(run) {
|
|
724
|
+
try {
|
|
725
|
+
return run();
|
|
726
|
+
}
|
|
727
|
+
catch (err) {
|
|
728
|
+
if (err instanceof TargetError) {
|
|
729
|
+
throw usageError(err.message, SHOW_USAGE);
|
|
730
|
+
}
|
|
731
|
+
throw err;
|
|
732
|
+
}
|
|
733
|
+
}
|
|
653
734
|
async function stats(args, ctx) {
|
|
654
735
|
const { values } = parse(args, { dir: { type: 'string', short: 'd' } }, 'zen rag docs stats [--dir <dir>]');
|
|
655
736
|
const dir = indexDir(ctx, values.dir);
|
package/dist/docs/load.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
+
import { isGlob, wildcard } from '@zenera/neo';
|
|
2
3
|
import { createHash } from 'node:crypto';
|
|
3
4
|
import { readdir, readFile, stat } from 'node:fs/promises';
|
|
4
5
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
5
|
-
import { isGlob, wildcard } from "../common/match.js";
|
|
6
6
|
import { NO_PARSE_CACHE, openParseCache, parseKey } from "./parse-cache.js";
|
|
7
7
|
import { normalize } from "./parse.js";
|
|
8
8
|
import { parseAll } from "./pool.js";
|
package/dist/docs/lookup.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type MatchOptions } from '
|
|
1
|
+
import { type MatchOptions } from '@zenera/neo';
|
|
2
2
|
import type { DocsIndex } from './search.ts';
|
|
3
3
|
export interface Listing<T> {
|
|
4
4
|
found: number;
|
|
@@ -77,4 +77,57 @@ export interface Verbatim {
|
|
|
77
77
|
/** A named section, verbatim: heading line to the line before the next peer. */
|
|
78
78
|
export declare function readSection(index: DocsIndex, file: string, section: string): Promise<Verbatim>;
|
|
79
79
|
export declare function readRange(index: DocsIndex, file: string, from: number, to: number): Promise<Verbatim>;
|
|
80
|
+
export interface Read {
|
|
81
|
+
/** what the caller wrote, so a failure can be put back next to it */
|
|
82
|
+
target: string;
|
|
83
|
+
file: string;
|
|
84
|
+
/** a heading title, id or path; wins over from/to */
|
|
85
|
+
section?: string;
|
|
86
|
+
from?: number;
|
|
87
|
+
to?: number;
|
|
88
|
+
}
|
|
89
|
+
export interface ReadBlock {
|
|
90
|
+
/** what was printed, as a read that would print it again */
|
|
91
|
+
target: string;
|
|
92
|
+
file: string;
|
|
93
|
+
/** first line printed */
|
|
94
|
+
from: number;
|
|
95
|
+
/** last line printed; `from - 1` when the budget left none */
|
|
96
|
+
to: number;
|
|
97
|
+
/** last line that was asked for, after clamping to the document */
|
|
98
|
+
end: number;
|
|
99
|
+
total: number;
|
|
100
|
+
/** the heading, when the block is exactly one section */
|
|
101
|
+
section?: string;
|
|
102
|
+
/** the range asked for, when it ran past the end of the document */
|
|
103
|
+
asked?: string;
|
|
104
|
+
lines: string[];
|
|
105
|
+
truncated: boolean;
|
|
106
|
+
/** the read that prints the rest of a cut block */
|
|
107
|
+
continue?: string;
|
|
108
|
+
}
|
|
109
|
+
export interface ReadFailure {
|
|
110
|
+
target: string;
|
|
111
|
+
file: string;
|
|
112
|
+
error: string;
|
|
113
|
+
hint: string;
|
|
114
|
+
}
|
|
115
|
+
export type ReadResult = ReadBlock | ReadFailure;
|
|
116
|
+
export interface ReadMany {
|
|
117
|
+
results: ReadResult[];
|
|
118
|
+
printed: number;
|
|
119
|
+
failed: number;
|
|
120
|
+
}
|
|
121
|
+
export interface ReadHints {
|
|
122
|
+
document: (file: string) => string;
|
|
123
|
+
section: (file: string) => string;
|
|
124
|
+
range: (file: string, total: number) => string;
|
|
125
|
+
}
|
|
126
|
+
export interface ReadManyOptions {
|
|
127
|
+
maxLines?: number;
|
|
128
|
+
hints?: Partial<ReadHints>;
|
|
129
|
+
}
|
|
130
|
+
export declare const isFailure: <T extends object>(result: T | ReadFailure) => result is ReadFailure;
|
|
131
|
+
export declare const spanOf: (file: string, from: number, to: number) => string;
|
|
132
|
+
export declare function readMany(index: DocsIndex, reads: readonly Read[], options?: ReadManyOptions): Promise<ReadMany>;
|
|
80
133
|
//# sourceMappingURL=lookup.d.ts.map
|
package/dist/docs/lookup.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
-
import { loose } from
|
|
2
|
+
import { loose } from '@zenera/neo';
|
|
3
3
|
import { under } from "./search.js";
|
|
4
4
|
export const DEFAULT_ROWS = 50;
|
|
5
5
|
export const MAX_ROWS = 500;
|
|
@@ -120,6 +120,125 @@ export async function readRange(index, file, from, to) {
|
|
|
120
120
|
total: lines.length,
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
|
+
const CLI_HINTS = {
|
|
124
|
+
document: (file) => `\`zen rag docs list files --file "*${file.split('/').at(-1)}*"\` lists close names`,
|
|
125
|
+
section: (file) => `\`zen rag docs list sections --file "${file}"\` lists its headings`,
|
|
126
|
+
range: (file, total) => `read ${spanOf(file, 1, total)}`,
|
|
127
|
+
};
|
|
128
|
+
export const isFailure = (result) => 'error' in result;
|
|
129
|
+
export const spanOf = (file, from, to) => from === to ? `${file}:${from}` : `${file}:${from}-${to}`;
|
|
130
|
+
export async function readMany(index, reads, options = {}) {
|
|
131
|
+
const hints = { ...CLI_HINTS, ...options.hints };
|
|
132
|
+
const slots = [];
|
|
133
|
+
for (const read of reads) {
|
|
134
|
+
const span = await spanFor(index, read, hints);
|
|
135
|
+
if (isFailure(span)) {
|
|
136
|
+
slots.push(span);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// Everything it overlaps or touches folds into the first of them, so a
|
|
140
|
+
// line is printed once however many reads named it.
|
|
141
|
+
const touching = slots.filter((slot) => !isFailure(slot) &&
|
|
142
|
+
slot.file === span.file &&
|
|
143
|
+
span.from <= slot.end + 1 &&
|
|
144
|
+
span.end >= slot.from - 1);
|
|
145
|
+
if (touching.length === 0) {
|
|
146
|
+
slots.push(span);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const [first, ...rest] = touching;
|
|
150
|
+
const all = [first, ...rest, span];
|
|
151
|
+
const from = Math.min(...all.map((s) => s.from));
|
|
152
|
+
const end = Math.max(...all.map((s) => s.end));
|
|
153
|
+
const same = all.every((s) => s.from === from && s.end === end);
|
|
154
|
+
Object.assign(first, {
|
|
155
|
+
from,
|
|
156
|
+
end,
|
|
157
|
+
section: same && all.every((s) => s.section === first.section) ? first.section : undefined,
|
|
158
|
+
asked: same ? first.asked : undefined,
|
|
159
|
+
});
|
|
160
|
+
for (const gone of rest) {
|
|
161
|
+
slots.splice(slots.indexOf(gone), 1);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const spans = slots.filter((slot) => !isFailure(slot));
|
|
165
|
+
const grants = share(spans.map((s) => s.end - s.from + 1), options.maxLines ?? Infinity);
|
|
166
|
+
const results = [];
|
|
167
|
+
for (const slot of slots) {
|
|
168
|
+
if (isFailure(slot)) {
|
|
169
|
+
results.push(slot);
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const granted = grants[spans.indexOf(slot)];
|
|
173
|
+
const to = slot.from + granted - 1;
|
|
174
|
+
const lines = (await index.lines(slot.file)).slice(slot.from - 1, to);
|
|
175
|
+
const truncated = to < slot.end;
|
|
176
|
+
results.push({
|
|
177
|
+
target: spanOf(slot.file, slot.from, granted > 0 ? to : slot.end),
|
|
178
|
+
file: slot.file,
|
|
179
|
+
from: slot.from,
|
|
180
|
+
to,
|
|
181
|
+
end: slot.end,
|
|
182
|
+
total: slot.total,
|
|
183
|
+
...(slot.section ? { section: slot.section } : {}),
|
|
184
|
+
...(slot.asked ? { asked: slot.asked } : {}),
|
|
185
|
+
lines,
|
|
186
|
+
truncated,
|
|
187
|
+
...(truncated ? { continue: spanOf(slot.file, to + 1, slot.end) } : {}),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
const failed = results.filter(isFailure).length;
|
|
191
|
+
return { results, printed: results.length - failed, failed };
|
|
192
|
+
}
|
|
193
|
+
async function spanFor(index, read, hints) {
|
|
194
|
+
const fail = (error, hint) => ({
|
|
195
|
+
target: read.target,
|
|
196
|
+
file: read.file,
|
|
197
|
+
error,
|
|
198
|
+
hint,
|
|
199
|
+
});
|
|
200
|
+
const outline = index.file(read.file);
|
|
201
|
+
if (!outline) {
|
|
202
|
+
return fail('no document by that name', hints.document(read.file));
|
|
203
|
+
}
|
|
204
|
+
const total = (await index.lines(read.file)).length;
|
|
205
|
+
const base = { target: read.target, file: read.file, total };
|
|
206
|
+
if (read.section !== undefined) {
|
|
207
|
+
const found = index.resolveSections([read.section], [read.file]);
|
|
208
|
+
const wanted = read.section.toLowerCase();
|
|
209
|
+
const heading = found.find((h) => h.title.toLowerCase() === wanted) ?? found[0];
|
|
210
|
+
if (!heading) {
|
|
211
|
+
return fail(`no section called ${read.section}`, hints.section(read.file));
|
|
212
|
+
}
|
|
213
|
+
return { ...base, from: heading.line, end: heading.end, section: heading.title };
|
|
214
|
+
}
|
|
215
|
+
const from = read.from ?? 1;
|
|
216
|
+
const to = read.to ?? total;
|
|
217
|
+
if (from > total) {
|
|
218
|
+
return fail(`line ${from} is past the end; the document has ${total} lines`, hints.range(read.file, total));
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
...base,
|
|
222
|
+
from,
|
|
223
|
+
end: Math.min(to, total),
|
|
224
|
+
...(to > total && read.to !== undefined ? { asked: `${from}-${to}` } : {}),
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Divides a budget so no request can starve the rest: an even share each, and
|
|
229
|
+
* what a short one does not use is shared again among those still wanting.
|
|
230
|
+
*/
|
|
231
|
+
function share(sizes, budget) {
|
|
232
|
+
const grants = sizes.map(() => 0);
|
|
233
|
+
const order = sizes.map((_, at) => at).sort((a, b) => sizes[a] - sizes[b]);
|
|
234
|
+
let left = budget;
|
|
235
|
+
for (const [rank, at] of order.entries()) {
|
|
236
|
+
const fair = Math.floor(left / (order.length - rank));
|
|
237
|
+
grants[at] = Math.min(sizes[at], fair === Infinity ? sizes[at] : fair);
|
|
238
|
+
left -= grants[at];
|
|
239
|
+
}
|
|
240
|
+
return grants;
|
|
241
|
+
}
|
|
123
242
|
// ---------------------------------------------------------------------------
|
|
124
243
|
function cut(rows, limit) {
|
|
125
244
|
const take = Math.min(Math.max(1, limit ?? DEFAULT_ROWS), MAX_ROWS);
|
package/dist/docs/render.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Assembly, Excerpt } from './assemble.ts';
|
|
2
|
+
import { type ReadMany } from './lookup.ts';
|
|
2
3
|
import type { Match } from './search.ts';
|
|
3
4
|
export interface RenderOptions {
|
|
4
5
|
/** the line-number gutter; on unless something else is going to eat this */
|
|
@@ -7,6 +8,11 @@ export interface RenderOptions {
|
|
|
7
8
|
}
|
|
8
9
|
export declare function renderAssembly(assembly: Assembly, options?: RenderOptions): string;
|
|
9
10
|
export declare function renderExcerpt(file: Excerpt, options?: RenderOptions): string[];
|
|
11
|
+
/**
|
|
12
|
+
* Each read under a header that is itself a read, so a citation copied from
|
|
13
|
+
* the header, or the line that continues a cut block, pastes straight back.
|
|
14
|
+
*/
|
|
15
|
+
export declare function renderReads(read: ReadMany, options?: RenderOptions): string;
|
|
10
16
|
/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
|
|
11
17
|
export declare const matchRows: (matches: readonly Match[]) => string[][];
|
|
12
18
|
export declare const MATCH_HEADERS: readonly ["id", "kind", "lines", "score", "vec / txt", "heading"];
|
package/dist/docs/render.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { bold, dim } from '@zenera/cli/lib';
|
|
2
|
+
import { isFailure } from "./lookup.js";
|
|
2
3
|
export function renderAssembly(assembly, options = {}) {
|
|
3
4
|
const out = [];
|
|
4
5
|
for (const file of assembly.files) {
|
|
@@ -30,6 +31,38 @@ function renderPiece(piece, width, options, faint) {
|
|
|
30
31
|
: `${faint(String(piece.start + at).padStart(width))} ${faint('|')} ${line}`);
|
|
31
32
|
return [...lines, ''];
|
|
32
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* Each read under a header that is itself a read, so a citation copied from
|
|
36
|
+
* the header, or the line that continues a cut block, pastes straight back.
|
|
37
|
+
*/
|
|
38
|
+
export function renderReads(read, options = {}) {
|
|
39
|
+
const paint = options.colour === false ? (s) => s : undefined;
|
|
40
|
+
const strong = paint ?? bold;
|
|
41
|
+
const faint = paint ?? dim;
|
|
42
|
+
const blocks = read.results.map((result) => isFailure(result)
|
|
43
|
+
? [`## ${strong(result.target)}`, `error: ${result.error} - ${result.hint}`]
|
|
44
|
+
: renderBlock(result, options, strong, faint));
|
|
45
|
+
return blocks.map((lines) => lines.join('\n')).join('\n\n');
|
|
46
|
+
}
|
|
47
|
+
function renderBlock(block, options, strong, faint) {
|
|
48
|
+
const notes = [
|
|
49
|
+
block.section ? `(#${block.section})` : '',
|
|
50
|
+
block.asked ? `(asked ${block.asked}; the document ends at ${block.total})` : '',
|
|
51
|
+
].filter(Boolean);
|
|
52
|
+
const width = String(block.end).length;
|
|
53
|
+
const out = [`## ${strong(block.target)}${notes.length ? ` ${faint(notes.join(' '))}` : ''}`];
|
|
54
|
+
for (const [at, line] of block.lines.entries()) {
|
|
55
|
+
out.push(options.numbers === false
|
|
56
|
+
? line
|
|
57
|
+
: `${faint(String(block.from + at).padStart(width))} ${faint('|')} ${line}`);
|
|
58
|
+
}
|
|
59
|
+
if (block.continue) {
|
|
60
|
+
out.push(faint(block.lines.length > 0
|
|
61
|
+
? `... truncated at line ${block.to} of ${block.end} - read ${block.continue} to continue`
|
|
62
|
+
: `... the line budget ran out before this one - read ${block.continue}`));
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
33
66
|
/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
|
|
34
67
|
export const matchRows = (matches) => matches.map((m) => [
|
|
35
68
|
m.id,
|
package/dist/docs/search.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Embedder } from '@zenera/neo';
|
|
2
2
|
import { type FileOutline, type HeadingRecord, type Manifest, type OpenDocs, type Outline } from './files.ts';
|
|
3
3
|
import { ChunkStore } from './store.ts';
|
|
4
4
|
export type SearchMode = 'hybrid' | 'vector' | 'text';
|
package/dist/docs/search.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { loose, PatternError } from '@zenera/neo';
|
|
1
2
|
import { assertSameEmbedding } from "../common/manifest.js";
|
|
2
|
-
import { loose, PatternError } from "../common/match.js";
|
|
3
3
|
import { CHUNK_KINDS, parseLines } from "./chunk.js";
|
|
4
4
|
import { openIndex, readLines, } from "./files.js";
|
|
5
5
|
import { ChunkStore } from "./store.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Read } from './lookup.ts';
|
|
2
|
+
export declare class TargetError extends Error {
|
|
3
|
+
}
|
|
4
|
+
export declare function parseTarget(raw: string, names: ReadonlySet<string>): Read;
|
|
5
|
+
/** Does this argument name a part of a document the index holds? */
|
|
6
|
+
export declare function selects(raw: string, names: ReadonlySet<string>): boolean;
|
|
7
|
+
//# sourceMappingURL=targets.d.ts.map
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// A read, written as its citation
|
|
3
|
+
//
|
|
4
|
+
// `<document>:<a>-<b>`, `<document>:<a>`, `<document>#<heading>`, or a bare
|
|
5
|
+
// document name — the form an answer cites a passage in, so a citation pastes
|
|
6
|
+
// back unchanged.
|
|
7
|
+
//
|
|
8
|
+
// Splitting at the last `:` or `#` would be wrong both ways: a heading may hold
|
|
9
|
+
// a colon (`#Step 1: Install`) and a document name is any relative path. So the
|
|
10
|
+
// split is wherever the text before it is a document the index holds, longest
|
|
11
|
+
// first, and only a name nobody holds is split by guesswork.
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
export class TargetError extends Error {
|
|
14
|
+
}
|
|
15
|
+
export function parseTarget(raw, names) {
|
|
16
|
+
if (names.has(raw)) {
|
|
17
|
+
return { target: raw, file: raw };
|
|
18
|
+
}
|
|
19
|
+
for (let at = raw.length - 1; at > 0; at--) {
|
|
20
|
+
if ((raw[at] === ':' || raw[at] === '#') && names.has(raw.slice(0, at))) {
|
|
21
|
+
return selector(raw, raw.slice(0, at), raw[at], raw.slice(at + 1));
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// A name the index does not hold: a heading is likelier to carry a colon
|
|
25
|
+
// than a document name a hash, so the first `#` wins over the last `:`.
|
|
26
|
+
const hash = raw.indexOf('#');
|
|
27
|
+
const at = hash > 0 ? hash : raw.lastIndexOf(':');
|
|
28
|
+
return at > 0
|
|
29
|
+
? selector(raw, raw.slice(0, at), raw[at], raw.slice(at + 1))
|
|
30
|
+
: { target: raw, file: raw };
|
|
31
|
+
}
|
|
32
|
+
/** Does this argument name a part of a document the index holds? */
|
|
33
|
+
export function selects(raw, names) {
|
|
34
|
+
const read = parseTarget(raw, names);
|
|
35
|
+
return names.has(read.file) && read.file !== raw;
|
|
36
|
+
}
|
|
37
|
+
function selector(raw, file, mark, rest) {
|
|
38
|
+
if (mark === '#') {
|
|
39
|
+
if (!rest.trim()) {
|
|
40
|
+
throw new TargetError(`"${raw}" names no heading after the #`);
|
|
41
|
+
}
|
|
42
|
+
return { target: raw, file, section: rest };
|
|
43
|
+
}
|
|
44
|
+
const found = /^(\d+)(?:-(\d+))?$/.exec(rest);
|
|
45
|
+
const from = Number(found?.[1]);
|
|
46
|
+
const to = found?.[2] === undefined ? from : Number(found[2]);
|
|
47
|
+
if (!found || from < 1 || to < from) {
|
|
48
|
+
throw new TargetError(`"${raw}" is not a line range - expected <document>:<a>-<b>`);
|
|
49
|
+
}
|
|
50
|
+
return { target: raw, file, from, to };
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=targets.js.map
|