@zenera/rag 1.1.8 → 1.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -10
- package/dist/command.js +2 -1
- package/dist/docs/assemble.d.ts +52 -0
- package/dist/docs/assemble.js +127 -0
- package/dist/docs/build.d.ts +34 -0
- package/dist/docs/build.js +108 -0
- package/dist/docs/chunk.d.ts +73 -0
- package/dist/docs/chunk.js +586 -0
- package/dist/docs/command.d.ts +3 -0
- package/dist/docs/command.js +529 -0
- package/dist/docs/files.d.ts +94 -0
- package/dist/docs/files.js +80 -0
- package/dist/docs/index.d.ts +13 -0
- package/dist/docs/index.js +13 -0
- package/dist/docs/load.d.ts +28 -0
- package/dist/docs/load.js +212 -0
- package/dist/docs/lookup.d.ts +80 -0
- package/dist/docs/lookup.js +147 -0
- package/dist/docs/parse.d.ts +95 -0
- package/dist/docs/parse.js +372 -0
- package/dist/docs/readme.d.ts +6 -0
- package/dist/docs/readme.js +122 -0
- package/dist/docs/render.d.ts +13 -0
- package/dist/docs/render.js +46 -0
- package/dist/docs/repl.d.ts +7 -0
- package/dist/docs/repl.js +130 -0
- package/dist/docs/search.d.ts +92 -0
- package/dist/docs/search.js +251 -0
- package/dist/docs/store.d.ts +55 -0
- package/dist/docs/store.js +171 -0
- package/dist/docs/tools.d.ts +10 -0
- package/dist/docs/tools.js +300 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +3 -0
- package/dist/schema/command.js +3 -0
- package/dist/schema/query.js +1 -0
- package/dist/schema/search.d.ts +2 -0
- package/dist/schema/search.js +18 -2
- package/dist/schema/store.d.ts +4 -2
- package/dist/schema/store.js +16 -9
- package/dist/schema/tools.js +21 -2
- package/package.json +17 -4
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
|
+
import { MANIFEST_FILE, readHead } from "../common/manifest.js";
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// What a document index is, on disk
|
|
7
|
+
//
|
|
8
|
+
// A directory, written in an order that is the whole crash story: the manifest
|
|
9
|
+
// goes last, so a half-built index has no manifest and reads as "not indexed"
|
|
10
|
+
// rather than as a store that quietly lost half its documents.
|
|
11
|
+
//
|
|
12
|
+
// manifest.json what this index is and what built it
|
|
13
|
+
// outline.json the headings and tables of every document, read whole
|
|
14
|
+
// sources/ the documents themselves, verbatim
|
|
15
|
+
// lance/ the chunks: the search text, the vectors, the filters
|
|
16
|
+
//
|
|
17
|
+
// `sources/` is not a convenience here, the way it is for a schema index — it
|
|
18
|
+
// is where the answers come from. A search returns line ranges, and the lines
|
|
19
|
+
// are read back out of these copies, so what is quoted is the document and not
|
|
20
|
+
// a reconstruction of it. That also makes the index one portable thing: nothing
|
|
21
|
+
// in it names a path outside itself, so it can be moved, shipped, or mounted
|
|
22
|
+
// somewhere else in an agent's sandbox and still answer.
|
|
23
|
+
//
|
|
24
|
+
// It is also why there is no table of lines. The original design kept one row
|
|
25
|
+
// per physical line beside the chunks, carrying the text and its structure. With
|
|
26
|
+
// the document itself sitting in `sources/`, that table would be a second copy
|
|
27
|
+
// of the same bytes; the per-line structure it also held is derivable from the
|
|
28
|
+
// outline, since a section runs from its heading to the next heading at the same
|
|
29
|
+
// depth or shallower. One store, one copy, same answers.
|
|
30
|
+
//
|
|
31
|
+
// `outline.json` is deliberately thin — headings, tables, counts — because it is
|
|
32
|
+
// parsed whole every time anything is asked. It is what makes `list`, `show` and
|
|
33
|
+
// section scoping work with no store, no embedder and no credential.
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
export const INDEX_VERSION = 1;
|
|
36
|
+
/** How a document index is found, read, and refused. */
|
|
37
|
+
export const DOCS_INDEX = {
|
|
38
|
+
kind: 'docs',
|
|
39
|
+
version: INDEX_VERSION,
|
|
40
|
+
defaultDir: './docs-db',
|
|
41
|
+
envName: 'ZEN_DOCS_DB',
|
|
42
|
+
};
|
|
43
|
+
export const OUTLINE_FILE = 'outline.json';
|
|
44
|
+
export const LANCE_DIR = 'lance';
|
|
45
|
+
export const SOURCES_DIR = 'sources';
|
|
46
|
+
export const lancePath = (dir) => join(dir, LANCE_DIR);
|
|
47
|
+
export async function writeIndex(dir, index) {
|
|
48
|
+
await mkdir(dir, { recursive: true });
|
|
49
|
+
await writeFile(join(dir, OUTLINE_FILE), JSON.stringify(index.outline));
|
|
50
|
+
for (const [name, text] of Object.entries(index.documents)) {
|
|
51
|
+
const target = join(dir, SOURCES_DIR, name);
|
|
52
|
+
await mkdir(dirname(target), { recursive: true });
|
|
53
|
+
await writeFile(target, text);
|
|
54
|
+
}
|
|
55
|
+
await writeFile(join(dir, MANIFEST_FILE), `${JSON.stringify(index.manifest, null, 4)}\n`);
|
|
56
|
+
}
|
|
57
|
+
export async function openIndex(dir) {
|
|
58
|
+
const manifest = await readManifest(dir);
|
|
59
|
+
const outline = JSON.parse(await readFile(join(dir, OUTLINE_FILE), 'utf8'));
|
|
60
|
+
return { dir, manifest, outline };
|
|
61
|
+
}
|
|
62
|
+
export const readManifest = (dir) => readHead(dir, DOCS_INDEX);
|
|
63
|
+
/**
|
|
64
|
+
* A document, verbatim, as it was indexed. The name is looked up in the
|
|
65
|
+
* manifest rather than joined onto the directory, so nothing a caller types
|
|
66
|
+
* ever reaches the filesystem — and a name that is not in the index is told so
|
|
67
|
+
* instead of becoming a path that happens not to exist.
|
|
68
|
+
*/
|
|
69
|
+
export async function readSource(index, name) {
|
|
70
|
+
const record = index.manifest.sources.find((s) => s.name === name);
|
|
71
|
+
if (!record) {
|
|
72
|
+
throw new CliError(`${index.dir} holds no document called ${name}`, EXIT.failed, `list them with \`zen rag docs list files\``);
|
|
73
|
+
}
|
|
74
|
+
return await readFile(join(index.dir, record.path), 'utf8');
|
|
75
|
+
}
|
|
76
|
+
/** The same document, split the way every line number in the index counts it. */
|
|
77
|
+
export async function readLines(index, name) {
|
|
78
|
+
return (await readSource(index, name)).split('\n');
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=files.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export * from './assemble.ts';
|
|
2
|
+
export * from './build.ts';
|
|
3
|
+
export * from './chunk.ts';
|
|
4
|
+
export * from './files.ts';
|
|
5
|
+
export * from './load.ts';
|
|
6
|
+
export * from './lookup.ts';
|
|
7
|
+
export * from './parse.ts';
|
|
8
|
+
export * from './readme.ts';
|
|
9
|
+
export * from './render.ts';
|
|
10
|
+
export * from './search.ts';
|
|
11
|
+
export * from './store.ts';
|
|
12
|
+
export * from './tools.ts';
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export * from "./assemble.js";
|
|
2
|
+
export * from "./build.js";
|
|
3
|
+
export * from "./chunk.js";
|
|
4
|
+
export * from "./files.js";
|
|
5
|
+
export * from "./load.js";
|
|
6
|
+
export * from "./lookup.js";
|
|
7
|
+
export * from "./parse.js";
|
|
8
|
+
export * from "./readme.js";
|
|
9
|
+
export * from "./render.js";
|
|
10
|
+
export * from "./search.js";
|
|
11
|
+
export * from "./store.js";
|
|
12
|
+
export * from "./tools.js";
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type Chunk, type ChunkOptions } from './chunk.ts';
|
|
2
|
+
import type { FileOutline } from './files.ts';
|
|
3
|
+
import { type DocFormat, type ParsedDoc } from './parse.ts';
|
|
4
|
+
/** Markdown, and plain text read as paragraphs. Anything else is not a document. */
|
|
5
|
+
export declare const DOC_EXTENSIONS: readonly [".md", ".markdown", ".txt", ".text"];
|
|
6
|
+
export interface LoadedDoc {
|
|
7
|
+
name: string;
|
|
8
|
+
file: string;
|
|
9
|
+
sha256: string;
|
|
10
|
+
bytes: number;
|
|
11
|
+
format: DocFormat;
|
|
12
|
+
/** the document verbatim, CRLF normalized: what goes into `sources/` */
|
|
13
|
+
text: string;
|
|
14
|
+
parsed: ParsedDoc;
|
|
15
|
+
chunks: Chunk[];
|
|
16
|
+
outline: FileOutline;
|
|
17
|
+
}
|
|
18
|
+
export interface Corpus {
|
|
19
|
+
docs: LoadedDoc[];
|
|
20
|
+
/** files that were found and not read, with the reason */
|
|
21
|
+
skipped: {
|
|
22
|
+
name: string;
|
|
23
|
+
reason: string;
|
|
24
|
+
}[];
|
|
25
|
+
}
|
|
26
|
+
export declare function loadDocuments(inputs: readonly string[], cwd: string, options?: ChunkOptions): Promise<Corpus>;
|
|
27
|
+
export declare const formatOf: (path: string) => DocFormat;
|
|
28
|
+
//# sourceMappingURL=load.d.ts.map
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readdir, readFile, stat } from 'node:fs/promises';
|
|
4
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
5
|
+
import { isGlob, wildcard } from "../common/match.js";
|
|
6
|
+
import { chunkDocument } from "./chunk.js";
|
|
7
|
+
import { normalize, parseDocument } from "./parse.js";
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
// Finding the documents, and reading them
|
|
10
|
+
//
|
|
11
|
+
// What can be named is a file, a directory, or a pattern. A directory is walked,
|
|
12
|
+
// because "index my notes" is the question people actually have and asking them
|
|
13
|
+
// to enumerate it would be answering a different one. Hidden directories and
|
|
14
|
+
// `node_modules` are skipped: a document nobody can see is not one anybody meant
|
|
15
|
+
// to index.
|
|
16
|
+
//
|
|
17
|
+
// Every document is given a NAME, which is its path relative to the common root
|
|
18
|
+
// of everything that was named — so indexing two release trees keeps
|
|
19
|
+
// `nsx_4.1.0/api/routing.md` and `nsx_4.2.0/api/routing.md` apart, and a search
|
|
20
|
+
// can be narrowed to one of them with a pattern over exactly that string. The
|
|
21
|
+
// name is the identity: it is stamped on every chunk, it is what `sources/`
|
|
22
|
+
// files are called, and nothing anywhere records where the file was on the
|
|
23
|
+
// machine that built the index.
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
/** Markdown, and plain text read as paragraphs. Anything else is not a document. */
|
|
26
|
+
export const DOC_EXTENSIONS = ['.md', '.markdown', '.txt', '.text'];
|
|
27
|
+
/** A file larger than this is a data dump, not something anyone wrote. */
|
|
28
|
+
const MAX_BYTES = 16 * 1024 * 1024;
|
|
29
|
+
/** Enough for a documentation tree; far short of a filesystem. */
|
|
30
|
+
const MAX_FILES = 20_000;
|
|
31
|
+
export async function loadDocuments(inputs, cwd, options = {}) {
|
|
32
|
+
const found = await discover(inputs, cwd);
|
|
33
|
+
if (found.length === 0) {
|
|
34
|
+
throw new CliError('nothing to index', EXIT.invalid, `no ${DOC_EXTENSIONS.join(', ')} files were found under ${inputs.join(', ')}`);
|
|
35
|
+
}
|
|
36
|
+
const root = commonRoot(found);
|
|
37
|
+
const taken = new Set();
|
|
38
|
+
const docs = [];
|
|
39
|
+
const skipped = [];
|
|
40
|
+
for (const absolute of found) {
|
|
41
|
+
const name = distinct(nameOf(root, absolute), taken);
|
|
42
|
+
const info = await stat(absolute);
|
|
43
|
+
if (info.size > MAX_BYTES) {
|
|
44
|
+
skipped.push({ name, reason: `larger than ${MAX_BYTES / 1024 / 1024} MB` });
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const raw = await readFile(absolute);
|
|
48
|
+
const text = normalize(raw.toString('utf8'));
|
|
49
|
+
const format = formatOf(absolute);
|
|
50
|
+
const parsed = parseDocument(text, name, format);
|
|
51
|
+
const chunks = chunkDocument(parsed, options);
|
|
52
|
+
docs.push({
|
|
53
|
+
name,
|
|
54
|
+
file: basename(absolute),
|
|
55
|
+
sha256: createHash('sha256').update(raw).digest('hex'),
|
|
56
|
+
bytes: raw.byteLength,
|
|
57
|
+
format,
|
|
58
|
+
text,
|
|
59
|
+
parsed,
|
|
60
|
+
chunks,
|
|
61
|
+
outline: outlineOf(parsed, chunks.length),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return { docs, skipped };
|
|
65
|
+
}
|
|
66
|
+
export const formatOf = (path) => ['.txt', '.text'].includes(extname(path).toLowerCase()) ? 'text' : 'markdown';
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
// discovery
|
|
69
|
+
// ---------------------------------------------------------------------------
|
|
70
|
+
async function discover(inputs, cwd) {
|
|
71
|
+
const out = new Set();
|
|
72
|
+
for (const input of inputs) {
|
|
73
|
+
const path = resolve(cwd, input);
|
|
74
|
+
if (isGlob(input)) {
|
|
75
|
+
// The pattern is anchored at the deepest directory it names outright,
|
|
76
|
+
// so a walk of the whole tree is never needed to answer one.
|
|
77
|
+
const anchor = staticPrefix(path);
|
|
78
|
+
const match = wildcard(path, { caseSensitive: true });
|
|
79
|
+
for (const file of await walk(anchor)) {
|
|
80
|
+
if (match(file)) {
|
|
81
|
+
out.add(file);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const info = await stat(path).catch(() => undefined);
|
|
87
|
+
if (!info) {
|
|
88
|
+
throw new CliError(`no such file or directory: ${input}`, EXIT.invalid);
|
|
89
|
+
}
|
|
90
|
+
if (info.isDirectory()) {
|
|
91
|
+
for (const file of await walk(path)) {
|
|
92
|
+
out.add(file);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
// A file named outright is indexed whatever it is called: someone
|
|
97
|
+
// who types a name has already decided it is a document.
|
|
98
|
+
out.add(path);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return [...out].sort();
|
|
102
|
+
}
|
|
103
|
+
async function walk(root) {
|
|
104
|
+
const out = [];
|
|
105
|
+
const queue = [root];
|
|
106
|
+
while (queue.length > 0 && out.length < MAX_FILES) {
|
|
107
|
+
const dir = queue.shift();
|
|
108
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
109
|
+
for (const entry of entries) {
|
|
110
|
+
if (entry.name.startsWith('.') || entry.name === 'node_modules') {
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
const path = join(dir, entry.name);
|
|
114
|
+
if (entry.isDirectory()) {
|
|
115
|
+
queue.push(path);
|
|
116
|
+
}
|
|
117
|
+
else if (indexable(entry.name)) {
|
|
118
|
+
out.push(path);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
const indexable = (name) => DOC_EXTENSIONS.includes(extname(name).toLowerCase());
|
|
125
|
+
/** The part of a pattern before the first wildcard, which is a real directory. */
|
|
126
|
+
function staticPrefix(pattern) {
|
|
127
|
+
const parts = pattern.split(sep);
|
|
128
|
+
const at = parts.findIndex((part) => isGlob(part));
|
|
129
|
+
const head = (at === -1 ? parts : parts.slice(0, at)).join(sep);
|
|
130
|
+
return head && isAbsolute(head) ? head : dirname(pattern.split('*')[0] ?? pattern);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* The deepest directory holding every file, which is what names are taken
|
|
134
|
+
* relative to. One file is its own directory, so a single document is named by
|
|
135
|
+
* its basename rather than by an accident of where it was kept.
|
|
136
|
+
*/
|
|
137
|
+
function commonRoot(files) {
|
|
138
|
+
const parts = files.map((file) => dirname(file).split(sep));
|
|
139
|
+
const first = parts[0] ?? [];
|
|
140
|
+
let at = 0;
|
|
141
|
+
while (at < first.length && parts.every((p) => p[at] === first[at])) {
|
|
142
|
+
at++;
|
|
143
|
+
}
|
|
144
|
+
return first.slice(0, at).join(sep) || sep;
|
|
145
|
+
}
|
|
146
|
+
function nameOf(root, absolute) {
|
|
147
|
+
const rel = relative(root, absolute);
|
|
148
|
+
// A file outside the common root cannot happen by construction; if it ever
|
|
149
|
+
// did, a name climbing out of the index directory must not.
|
|
150
|
+
return !rel || rel.startsWith('..') || isAbsolute(rel)
|
|
151
|
+
? basename(absolute)
|
|
152
|
+
: rel.split(sep).join('/');
|
|
153
|
+
}
|
|
154
|
+
function distinct(name, taken) {
|
|
155
|
+
let candidate = name;
|
|
156
|
+
for (let n = 2; taken.has(candidate); n++) {
|
|
157
|
+
const dot = name.lastIndexOf('.');
|
|
158
|
+
candidate = dot === -1 ? `${name}_${n}` : `${name.slice(0, dot)}_${n}${name.slice(dot)}`;
|
|
159
|
+
}
|
|
160
|
+
taken.add(candidate);
|
|
161
|
+
return candidate;
|
|
162
|
+
}
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
// the outline
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
/**
|
|
167
|
+
* Headings and tables, with the line each ends on. That end is what makes the
|
|
168
|
+
* outline enough on its own: a section runs from its heading to the line before
|
|
169
|
+
* the next heading at the same depth or shallower, so scoping a search to a
|
|
170
|
+
* section, listing what is in one, or naming the sections a skipped range
|
|
171
|
+
* covered are all answerable without reading the document.
|
|
172
|
+
*/
|
|
173
|
+
function outlineOf(doc, chunks) {
|
|
174
|
+
const sections = doc.sections.filter((s) => s.line !== undefined);
|
|
175
|
+
const headings = sections.map((section, at) => {
|
|
176
|
+
const next = sections.findIndex((other, i) => i > at && other.level <= section.level);
|
|
177
|
+
const end = next === -1 ? doc.lines.length : sections[next].line - 1;
|
|
178
|
+
return {
|
|
179
|
+
line: section.line,
|
|
180
|
+
end,
|
|
181
|
+
level: section.level,
|
|
182
|
+
title: section.title,
|
|
183
|
+
id: section.id,
|
|
184
|
+
path: section.path,
|
|
185
|
+
};
|
|
186
|
+
});
|
|
187
|
+
const tables = doc.blocks
|
|
188
|
+
.filter((block) => block.table)
|
|
189
|
+
.map((block) => {
|
|
190
|
+
const table = block.table;
|
|
191
|
+
return {
|
|
192
|
+
id: block.id,
|
|
193
|
+
path: block.path,
|
|
194
|
+
section: block.section.path,
|
|
195
|
+
line: block.start,
|
|
196
|
+
end: block.end,
|
|
197
|
+
columns: table.columns,
|
|
198
|
+
rows: table.rows.length,
|
|
199
|
+
caption: table.caption,
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
return {
|
|
203
|
+
name: doc.name,
|
|
204
|
+
title: doc.title,
|
|
205
|
+
format: doc.format,
|
|
206
|
+
lines: doc.lines.length,
|
|
207
|
+
chunks,
|
|
208
|
+
headings,
|
|
209
|
+
tables,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
//# sourceMappingURL=load.js.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { type MatchOptions } from '../common/match.ts';
|
|
2
|
+
import type { DocsIndex } from './search.ts';
|
|
3
|
+
export interface Listing<T> {
|
|
4
|
+
found: number;
|
|
5
|
+
rows: T[];
|
|
6
|
+
truncated: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface ListOptions {
|
|
9
|
+
/** patterns over the document name */
|
|
10
|
+
files?: readonly string[];
|
|
11
|
+
exclude_files?: readonly string[];
|
|
12
|
+
limit?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare const DEFAULT_ROWS = 50;
|
|
15
|
+
export declare const MAX_ROWS = 500;
|
|
16
|
+
export interface FileRow {
|
|
17
|
+
name: string;
|
|
18
|
+
title: string;
|
|
19
|
+
format: string;
|
|
20
|
+
lines: number;
|
|
21
|
+
sections: number;
|
|
22
|
+
tables: number;
|
|
23
|
+
chunks: number;
|
|
24
|
+
}
|
|
25
|
+
export declare function listFiles(index: DocsIndex, options?: ListOptions): Listing<FileRow>;
|
|
26
|
+
export interface SectionRow {
|
|
27
|
+
file: string;
|
|
28
|
+
id: string;
|
|
29
|
+
path: string;
|
|
30
|
+
level: number;
|
|
31
|
+
title: string;
|
|
32
|
+
line: number;
|
|
33
|
+
end: number;
|
|
34
|
+
}
|
|
35
|
+
export interface SectionOptions extends ListOptions {
|
|
36
|
+
/** a heading title, a structure id, or a structure path */
|
|
37
|
+
section?: readonly string[];
|
|
38
|
+
/** deepest heading level to report; the default is everything */
|
|
39
|
+
depth?: number;
|
|
40
|
+
}
|
|
41
|
+
export declare function listSections(index: DocsIndex, options?: SectionOptions): Listing<SectionRow>;
|
|
42
|
+
export interface TableRow {
|
|
43
|
+
file: string;
|
|
44
|
+
id: string;
|
|
45
|
+
section: string;
|
|
46
|
+
caption: string;
|
|
47
|
+
columns: string;
|
|
48
|
+
rows: number;
|
|
49
|
+
line: number;
|
|
50
|
+
end: number;
|
|
51
|
+
}
|
|
52
|
+
export declare function listTables(index: DocsIndex, options?: SectionOptions): Listing<TableRow>;
|
|
53
|
+
export interface LineRow {
|
|
54
|
+
file: string;
|
|
55
|
+
line: number;
|
|
56
|
+
text: string;
|
|
57
|
+
/** the innermost heading the line sits under, so a hit has a place */
|
|
58
|
+
section: string;
|
|
59
|
+
}
|
|
60
|
+
export interface GrepOptions extends SectionOptions, MatchOptions {
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Lines matching a pattern. Substring by default, glob when it has wildcards,
|
|
64
|
+
* a regular expression when asked — the same rule every other pattern in this
|
|
65
|
+
* package follows, so nobody has to remember which flavour a flag takes.
|
|
66
|
+
*/
|
|
67
|
+
export declare function grepLines(index: DocsIndex, pattern: string, options?: GrepOptions): Promise<Listing<LineRow>>;
|
|
68
|
+
export interface Verbatim {
|
|
69
|
+
file: string;
|
|
70
|
+
title: string;
|
|
71
|
+
start: number;
|
|
72
|
+
end: number;
|
|
73
|
+
lines: string[];
|
|
74
|
+
/** the document's length, so a caller can tell what it did not get */
|
|
75
|
+
total: number;
|
|
76
|
+
}
|
|
77
|
+
/** A named section, verbatim: heading line to the line before the next peer. */
|
|
78
|
+
export declare function readSection(index: DocsIndex, file: string, section: string): Promise<Verbatim>;
|
|
79
|
+
export declare function readRange(index: DocsIndex, file: string, from: number, to: number): Promise<Verbatim>;
|
|
80
|
+
//# sourceMappingURL=lookup.d.ts.map
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { CliError, EXIT } from '@zenera/cli/lib';
|
|
2
|
+
import { loose } from "../common/match.js";
|
|
3
|
+
import { under } from "./search.js";
|
|
4
|
+
export const DEFAULT_ROWS = 50;
|
|
5
|
+
export const MAX_ROWS = 500;
|
|
6
|
+
export function listFiles(index, options = {}) {
|
|
7
|
+
const names = new Set(index.resolveFiles(options.files, options.exclude_files));
|
|
8
|
+
const rows = index.manifest.sources
|
|
9
|
+
.filter((source) => names.has(source.name))
|
|
10
|
+
.map((source) => ({
|
|
11
|
+
name: source.name,
|
|
12
|
+
title: source.title,
|
|
13
|
+
format: source.format,
|
|
14
|
+
lines: source.lines,
|
|
15
|
+
sections: source.sections,
|
|
16
|
+
tables: source.tables,
|
|
17
|
+
chunks: source.chunks,
|
|
18
|
+
}));
|
|
19
|
+
return cut(rows, options.limit);
|
|
20
|
+
}
|
|
21
|
+
export function listSections(index, options = {}) {
|
|
22
|
+
const files = index.resolveFiles(options.files, options.exclude_files);
|
|
23
|
+
const within = new Set(files);
|
|
24
|
+
const wanted = options.section?.length
|
|
25
|
+
? index.resolveSections(options.section, files).map((h) => h.path)
|
|
26
|
+
: [];
|
|
27
|
+
const rows = [];
|
|
28
|
+
for (const file of index.outline.files) {
|
|
29
|
+
if (!within.has(file.name)) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
for (const heading of file.headings) {
|
|
33
|
+
if (under(heading.path, wanted) && heading.level <= (options.depth ?? Infinity)) {
|
|
34
|
+
rows.push({ file: file.name, ...record(heading) });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return cut(rows, options.limit);
|
|
39
|
+
}
|
|
40
|
+
export function listTables(index, options = {}) {
|
|
41
|
+
const files = index.resolveFiles(options.files, options.exclude_files);
|
|
42
|
+
const within = new Set(files);
|
|
43
|
+
const wanted = options.section?.length
|
|
44
|
+
? index.resolveSections(options.section, files).map((h) => h.path)
|
|
45
|
+
: [];
|
|
46
|
+
const rows = [];
|
|
47
|
+
for (const file of index.outline.files) {
|
|
48
|
+
if (!within.has(file.name)) {
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
for (const table of file.tables) {
|
|
52
|
+
if (!under(table.path, wanted)) {
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
rows.push({
|
|
56
|
+
file: file.name,
|
|
57
|
+
id: table.id,
|
|
58
|
+
section: table.section,
|
|
59
|
+
caption: table.caption,
|
|
60
|
+
columns: table.columns.join(', '),
|
|
61
|
+
rows: table.rows,
|
|
62
|
+
line: table.line,
|
|
63
|
+
end: table.end,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return cut(rows, options.limit);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Lines matching a pattern. Substring by default, glob when it has wildcards,
|
|
71
|
+
* a regular expression when asked — the same rule every other pattern in this
|
|
72
|
+
* package follows, so nobody has to remember which flavour a flag takes.
|
|
73
|
+
*/
|
|
74
|
+
export async function grepLines(index, pattern, options = {}) {
|
|
75
|
+
const files = index.resolveFiles(options.files, options.exclude_files);
|
|
76
|
+
const wanted = options.section?.length
|
|
77
|
+
? index.resolveSections(options.section, files)
|
|
78
|
+
: undefined;
|
|
79
|
+
const match = loose(pattern, { regex: options.regex, caseSensitive: options.caseSensitive });
|
|
80
|
+
const rows = [];
|
|
81
|
+
for (const name of files) {
|
|
82
|
+
const spans = wanted?.filter((h) => index.file(name)?.headings.includes(h));
|
|
83
|
+
if (wanted && (!spans || spans.length === 0)) {
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const lines = await index.lines(name);
|
|
87
|
+
const headings = index.file(name)?.headings ?? [];
|
|
88
|
+
for (const [at, text] of lines.entries()) {
|
|
89
|
+
const line = at + 1;
|
|
90
|
+
if (spans && !spans.some((span) => line >= span.line && line <= span.end)) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (match(text)) {
|
|
94
|
+
rows.push({ file: name, line, text, section: enclosing(headings, line) });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return cut(rows, options.limit);
|
|
99
|
+
}
|
|
100
|
+
/** A named section, verbatim: heading line to the line before the next peer. */
|
|
101
|
+
export async function readSection(index, file, section) {
|
|
102
|
+
const found = index.resolveSections([section], [file]);
|
|
103
|
+
const heading = found[0];
|
|
104
|
+
if (!heading) {
|
|
105
|
+
throw new CliError(`${file} has no section called ${section}`, EXIT.failed, 'list them with `zen rag docs list sections --file <name>`');
|
|
106
|
+
}
|
|
107
|
+
return await readRange(index, file, heading.line, heading.end);
|
|
108
|
+
}
|
|
109
|
+
export async function readRange(index, file, from, to) {
|
|
110
|
+
const lines = await index.lines(file);
|
|
111
|
+
const outline = index.file(file);
|
|
112
|
+
const start = Math.max(1, from);
|
|
113
|
+
const end = Math.min(lines.length, to);
|
|
114
|
+
return {
|
|
115
|
+
file,
|
|
116
|
+
title: outline?.title ?? file,
|
|
117
|
+
start,
|
|
118
|
+
end,
|
|
119
|
+
lines: lines.slice(start - 1, end),
|
|
120
|
+
total: lines.length,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
// ---------------------------------------------------------------------------
|
|
124
|
+
function cut(rows, limit) {
|
|
125
|
+
const take = Math.min(Math.max(1, limit ?? DEFAULT_ROWS), MAX_ROWS);
|
|
126
|
+
return { found: rows.length, rows: rows.slice(0, take), truncated: rows.length > take };
|
|
127
|
+
}
|
|
128
|
+
const record = (heading) => ({
|
|
129
|
+
id: heading.id,
|
|
130
|
+
path: heading.path,
|
|
131
|
+
level: heading.level,
|
|
132
|
+
title: heading.title,
|
|
133
|
+
line: heading.line,
|
|
134
|
+
end: heading.end,
|
|
135
|
+
});
|
|
136
|
+
/** The last heading at or above this line, which is the section it is in. */
|
|
137
|
+
function enclosing(headings, line) {
|
|
138
|
+
let title = '';
|
|
139
|
+
for (const heading of headings) {
|
|
140
|
+
if (heading.line > line) {
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
title = heading.title;
|
|
144
|
+
}
|
|
145
|
+
return title;
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=lookup.js.map
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/** What a line, a block or a chunk is. The vocabulary is closed on purpose. */
|
|
2
|
+
export type StructureKind = 'heading' | 'paragraph' | 'blockquote' | 'list' | 'list_item' | 'table' | 'table_header' | 'table_row' | 'code' | 'frontmatter' | 'html' | 'hr';
|
|
3
|
+
/** The root of every path: a document is the structure everything else sits in. */
|
|
4
|
+
export declare const ROOT_SEGMENT = "doc";
|
|
5
|
+
export type DocFormat = 'markdown' | 'text';
|
|
6
|
+
/**
|
|
7
|
+
* A heading and everything under it, until a heading of the same depth or
|
|
8
|
+
* shallower. mdast has no such node — a heading there is a SIBLING of the
|
|
9
|
+
* paragraphs that follow it — so the nesting is built here.
|
|
10
|
+
*/
|
|
11
|
+
export interface Section {
|
|
12
|
+
id: string;
|
|
13
|
+
path: string;
|
|
14
|
+
title: string;
|
|
15
|
+
/** 0 for the document itself, otherwise the heading depth */
|
|
16
|
+
level: number;
|
|
17
|
+
/** the heading's own line; absent on the document root */
|
|
18
|
+
line: number | undefined;
|
|
19
|
+
parent: Section | undefined;
|
|
20
|
+
}
|
|
21
|
+
export interface TableRowBlock {
|
|
22
|
+
id: string;
|
|
23
|
+
path: string;
|
|
24
|
+
line: number;
|
|
25
|
+
cells: string[];
|
|
26
|
+
/** the row as prose, so it carries its own column names */
|
|
27
|
+
text: string;
|
|
28
|
+
}
|
|
29
|
+
export interface TableBlock {
|
|
30
|
+
columns: string[];
|
|
31
|
+
headerLine: number;
|
|
32
|
+
/** the alignment row; mdast emits no node for it, so it is derived */
|
|
33
|
+
separatorLine: number | undefined;
|
|
34
|
+
/** the paragraph before the table, which is the only caption one ever gets */
|
|
35
|
+
caption: string;
|
|
36
|
+
/** the column that identifies a row, repeated into every slice of it */
|
|
37
|
+
keyColumn: number;
|
|
38
|
+
rows: TableRowBlock[];
|
|
39
|
+
}
|
|
40
|
+
export interface ListItemBlock {
|
|
41
|
+
id: string;
|
|
42
|
+
path: string;
|
|
43
|
+
start: number;
|
|
44
|
+
end: number;
|
|
45
|
+
text: string;
|
|
46
|
+
}
|
|
47
|
+
export interface Block {
|
|
48
|
+
id: string;
|
|
49
|
+
path: string;
|
|
50
|
+
kind: StructureKind;
|
|
51
|
+
/** 1-based, inclusive */
|
|
52
|
+
start: number;
|
|
53
|
+
end: number;
|
|
54
|
+
section: Section;
|
|
55
|
+
/** normalized, never shown: this is what is embedded and indexed */
|
|
56
|
+
text: string;
|
|
57
|
+
items?: ListItemBlock[];
|
|
58
|
+
table?: TableBlock;
|
|
59
|
+
}
|
|
60
|
+
export interface ParsedDoc {
|
|
61
|
+
/** the document's name within the index — its path, and its identity */
|
|
62
|
+
name: string;
|
|
63
|
+
title: string;
|
|
64
|
+
format: DocFormat;
|
|
65
|
+
/** verbatim, CRLF normalized, 1-based when indexed as `lines[n - 1]` */
|
|
66
|
+
lines: string[];
|
|
67
|
+
sections: Section[];
|
|
68
|
+
blocks: Block[];
|
|
69
|
+
}
|
|
70
|
+
/** Every ancestor of a section, the document root first, itself last. */
|
|
71
|
+
export declare function ancestry(section: Section): Section[];
|
|
72
|
+
/** The breadcrumb, which prefixes every chunk of text under this section. */
|
|
73
|
+
export declare const headingPath: (section: Section) => string;
|
|
74
|
+
/** The heading lines a chunk under this section must be rendered with. */
|
|
75
|
+
export declare const headingLines: (section: Section) => number[];
|
|
76
|
+
/**
|
|
77
|
+
* Everything downstream indexes `lines` by number, and mdast positions refer to
|
|
78
|
+
* the exact string handed to the parser — so the normalization happens once,
|
|
79
|
+
* here, and the normalized text is the only version anything ever sees.
|
|
80
|
+
*/
|
|
81
|
+
export declare const normalize: (text: string) => string;
|
|
82
|
+
export declare function parseDocument(source: string, name: string, format: DocFormat): ParsedDoc;
|
|
83
|
+
/**
|
|
84
|
+
* A data row does not contain its own column names. Stripping the pipes leaves
|
|
85
|
+
* `V-200-30 3" WCC 98.2`, in which the word `Cv` appears nowhere, so a query
|
|
86
|
+
* naming a column could only ever match the header line and never a row — and
|
|
87
|
+
* hybrid search would silently become vector-only for every table in the
|
|
88
|
+
* corpus. Pairing each cell with its header is what fixes that, and it embeds
|
|
89
|
+
* better too, because `Cv: 45` is a sentence and `| 45 |` is not.
|
|
90
|
+
*/
|
|
91
|
+
export declare function rowText(columns: readonly string[], cells: readonly string[]): string;
|
|
92
|
+
export declare function cellText(column: string | undefined, cell: string): string;
|
|
93
|
+
/** Soft wraps inside a paragraph are wrapping, not meaning, so they rejoin. */
|
|
94
|
+
export declare const collapse: (text: string) => string;
|
|
95
|
+
//# sourceMappingURL=parse.d.ts.map
|