@zenera/rag 1.1.8 → 1.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +154 -10
  2. package/dist/command.js +2 -1
  3. package/dist/docs/assemble.d.ts +52 -0
  4. package/dist/docs/assemble.js +127 -0
  5. package/dist/docs/build.d.ts +34 -0
  6. package/dist/docs/build.js +108 -0
  7. package/dist/docs/chunk.d.ts +73 -0
  8. package/dist/docs/chunk.js +586 -0
  9. package/dist/docs/command.d.ts +3 -0
  10. package/dist/docs/command.js +529 -0
  11. package/dist/docs/files.d.ts +94 -0
  12. package/dist/docs/files.js +80 -0
  13. package/dist/docs/index.d.ts +13 -0
  14. package/dist/docs/index.js +13 -0
  15. package/dist/docs/load.d.ts +28 -0
  16. package/dist/docs/load.js +212 -0
  17. package/dist/docs/lookup.d.ts +80 -0
  18. package/dist/docs/lookup.js +147 -0
  19. package/dist/docs/parse.d.ts +95 -0
  20. package/dist/docs/parse.js +372 -0
  21. package/dist/docs/readme.d.ts +6 -0
  22. package/dist/docs/readme.js +122 -0
  23. package/dist/docs/render.d.ts +13 -0
  24. package/dist/docs/render.js +46 -0
  25. package/dist/docs/repl.d.ts +7 -0
  26. package/dist/docs/repl.js +130 -0
  27. package/dist/docs/search.d.ts +92 -0
  28. package/dist/docs/search.js +251 -0
  29. package/dist/docs/store.d.ts +55 -0
  30. package/dist/docs/store.js +171 -0
  31. package/dist/docs/tools.d.ts +10 -0
  32. package/dist/docs/tools.js +300 -0
  33. package/dist/index.d.ts +1 -0
  34. package/dist/index.js +3 -0
  35. package/dist/schema/command.js +3 -0
  36. package/dist/schema/query.js +1 -0
  37. package/dist/schema/search.d.ts +2 -0
  38. package/dist/schema/search.js +18 -2
  39. package/dist/schema/store.d.ts +4 -2
  40. package/dist/schema/store.js +16 -9
  41. package/dist/schema/tools.js +21 -2
  42. package/package.json +17 -4
@@ -0,0 +1,80 @@
1
+ import { CliError, EXIT } from '@zenera/cli/lib';
2
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
+ import { dirname, join } from 'node:path';
4
+ import { MANIFEST_FILE, readHead } from "../common/manifest.js";
5
+ // ---------------------------------------------------------------------------
6
+ // What a document index is, on disk
7
+ //
8
+ // A directory, written in an order that is the whole crash story: the manifest
9
+ // goes last, so a half-built index has no manifest and reads as "not indexed"
10
+ // rather than as a store that quietly lost half its documents.
11
+ //
12
+ // manifest.json what this index is and what built it
13
+ // outline.json the headings and tables of every document, read whole
14
+ // sources/ the documents themselves, verbatim
15
+ // lance/ the chunks: the search text, the vectors, the filters
16
+ //
17
+ // `sources/` is not a convenience here, the way it is for a schema index — it
18
+ // is where the answers come from. A search returns line ranges, and the lines
19
+ // are read back out of these copies, so what is quoted is the document and not
20
+ // a reconstruction of it. That also makes the index one portable thing: nothing
21
+ // in it names a path outside itself, so it can be moved, shipped, or mounted
22
+ // somewhere else in an agent's sandbox and still answer.
23
+ //
24
+ // It is also why there is no table of lines. The original design kept one row
25
+ // per physical line beside the chunks, carrying the text and its structure. With
26
+ // the document itself sitting in `sources/`, that table would be a second copy
27
+ // of the same bytes; the per-line structure it also held is derivable from the
28
+ // outline, since a section runs from its heading to the next heading at the same
29
+ // depth or shallower. One store, one copy, same answers.
30
+ //
31
+ // `outline.json` is deliberately thin — headings, tables, counts — because it is
32
+ // parsed whole every time anything is asked. It is what makes `list`, `show` and
33
+ // section scoping work with no store, no embedder and no credential.
34
+ // ---------------------------------------------------------------------------
35
+ export const INDEX_VERSION = 1;
36
+ /** How a document index is found, read, and refused. */
37
+ export const DOCS_INDEX = {
38
+ kind: 'docs',
39
+ version: INDEX_VERSION,
40
+ defaultDir: './docs-db',
41
+ envName: 'ZEN_DOCS_DB',
42
+ };
43
+ export const OUTLINE_FILE = 'outline.json';
44
+ export const LANCE_DIR = 'lance';
45
+ export const SOURCES_DIR = 'sources';
46
+ export const lancePath = (dir) => join(dir, LANCE_DIR);
47
+ export async function writeIndex(dir, index) {
48
+ await mkdir(dir, { recursive: true });
49
+ await writeFile(join(dir, OUTLINE_FILE), JSON.stringify(index.outline));
50
+ for (const [name, text] of Object.entries(index.documents)) {
51
+ const target = join(dir, SOURCES_DIR, name);
52
+ await mkdir(dirname(target), { recursive: true });
53
+ await writeFile(target, text);
54
+ }
55
+ await writeFile(join(dir, MANIFEST_FILE), `${JSON.stringify(index.manifest, null, 4)}\n`);
56
+ }
57
+ export async function openIndex(dir) {
58
+ const manifest = await readManifest(dir);
59
+ const outline = JSON.parse(await readFile(join(dir, OUTLINE_FILE), 'utf8'));
60
+ return { dir, manifest, outline };
61
+ }
62
+ export const readManifest = (dir) => readHead(dir, DOCS_INDEX);
63
+ /**
64
+ * A document, verbatim, as it was indexed. The name is looked up in the
65
+ * manifest rather than joined onto the directory, so nothing a caller types
66
+ * ever reaches the filesystem — and a name that is not in the index is told so
67
+ * instead of becoming a path that happens not to exist.
68
+ */
69
+ export async function readSource(index, name) {
70
+ const record = index.manifest.sources.find((s) => s.name === name);
71
+ if (!record) {
72
+ throw new CliError(`${index.dir} holds no document called ${name}`, EXIT.failed, `list them with \`zen rag docs list files\``);
73
+ }
74
+ return await readFile(join(index.dir, record.path), 'utf8');
75
+ }
76
+ /** The same document, split the way every line number in the index counts it. */
77
+ export async function readLines(index, name) {
78
+ return (await readSource(index, name)).split('\n');
79
+ }
80
+ //# sourceMappingURL=files.js.map
@@ -0,0 +1,13 @@
1
+ export * from './assemble.ts';
2
+ export * from './build.ts';
3
+ export * from './chunk.ts';
4
+ export * from './files.ts';
5
+ export * from './load.ts';
6
+ export * from './lookup.ts';
7
+ export * from './parse.ts';
8
+ export * from './readme.ts';
9
+ export * from './render.ts';
10
+ export * from './search.ts';
11
+ export * from './store.ts';
12
+ export * from './tools.ts';
13
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,13 @@
1
+ export * from "./assemble.js";
2
+ export * from "./build.js";
3
+ export * from "./chunk.js";
4
+ export * from "./files.js";
5
+ export * from "./load.js";
6
+ export * from "./lookup.js";
7
+ export * from "./parse.js";
8
+ export * from "./readme.js";
9
+ export * from "./render.js";
10
+ export * from "./search.js";
11
+ export * from "./store.js";
12
+ export * from "./tools.js";
13
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,28 @@
1
+ import { type Chunk, type ChunkOptions } from './chunk.ts';
2
+ import type { FileOutline } from './files.ts';
3
+ import { type DocFormat, type ParsedDoc } from './parse.ts';
4
+ /** Markdown, and plain text read as paragraphs. Anything else is not a document. */
5
+ export declare const DOC_EXTENSIONS: readonly [".md", ".markdown", ".txt", ".text"];
6
+ export interface LoadedDoc {
7
+ name: string;
8
+ file: string;
9
+ sha256: string;
10
+ bytes: number;
11
+ format: DocFormat;
12
+ /** the document verbatim, CRLF normalized: what goes into `sources/` */
13
+ text: string;
14
+ parsed: ParsedDoc;
15
+ chunks: Chunk[];
16
+ outline: FileOutline;
17
+ }
18
+ export interface Corpus {
19
+ docs: LoadedDoc[];
20
+ /** files that were found and not read, with the reason */
21
+ skipped: {
22
+ name: string;
23
+ reason: string;
24
+ }[];
25
+ }
26
+ export declare function loadDocuments(inputs: readonly string[], cwd: string, options?: ChunkOptions): Promise<Corpus>;
27
+ export declare const formatOf: (path: string) => DocFormat;
28
+ //# sourceMappingURL=load.d.ts.map
@@ -0,0 +1,212 @@
1
+ import { CliError, EXIT } from '@zenera/cli/lib';
2
+ import { createHash } from 'node:crypto';
3
+ import { readdir, readFile, stat } from 'node:fs/promises';
4
+ import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
5
+ import { isGlob, wildcard } from "../common/match.js";
6
+ import { chunkDocument } from "./chunk.js";
7
+ import { normalize, parseDocument } from "./parse.js";
8
+ // ---------------------------------------------------------------------------
9
+ // Finding the documents, and reading them
10
+ //
11
+ // What can be named is a file, a directory, or a pattern. A directory is walked,
12
+ // because "index my notes" is the question people actually have and asking them
13
+ // to enumerate it would be answering a different one. Hidden directories and
14
+ // `node_modules` are skipped: a document nobody can see is not one anybody meant
15
+ // to index.
16
+ //
17
+ // Every document is given a NAME, which is its path relative to the common root
18
+ // of everything that was named — so indexing two release trees keeps
19
+ // `nsx_4.1.0/api/routing.md` and `nsx_4.2.0/api/routing.md` apart, and a search
20
+ // can be narrowed to one of them with a pattern over exactly that string. The
21
+ // name is the identity: it is stamped on every chunk, it is what `sources/`
22
+ // files are called, and nothing anywhere records where the file was on the
23
+ // machine that built the index.
24
+ // ---------------------------------------------------------------------------
25
+ /** Markdown, and plain text read as paragraphs. Anything else is not a document. */
26
+ export const DOC_EXTENSIONS = ['.md', '.markdown', '.txt', '.text'];
27
+ /** A file larger than this is a data dump, not something anyone wrote. */
28
+ const MAX_BYTES = 16 * 1024 * 1024;
29
+ /** Enough for a documentation tree; far short of a filesystem. */
30
+ const MAX_FILES = 20_000;
31
+ export async function loadDocuments(inputs, cwd, options = {}) {
32
+ const found = await discover(inputs, cwd);
33
+ if (found.length === 0) {
34
+ throw new CliError('nothing to index', EXIT.invalid, `no ${DOC_EXTENSIONS.join(', ')} files were found under ${inputs.join(', ')}`);
35
+ }
36
+ const root = commonRoot(found);
37
+ const taken = new Set();
38
+ const docs = [];
39
+ const skipped = [];
40
+ for (const absolute of found) {
41
+ const name = distinct(nameOf(root, absolute), taken);
42
+ const info = await stat(absolute);
43
+ if (info.size > MAX_BYTES) {
44
+ skipped.push({ name, reason: `larger than ${MAX_BYTES / 1024 / 1024} MB` });
45
+ continue;
46
+ }
47
+ const raw = await readFile(absolute);
48
+ const text = normalize(raw.toString('utf8'));
49
+ const format = formatOf(absolute);
50
+ const parsed = parseDocument(text, name, format);
51
+ const chunks = chunkDocument(parsed, options);
52
+ docs.push({
53
+ name,
54
+ file: basename(absolute),
55
+ sha256: createHash('sha256').update(raw).digest('hex'),
56
+ bytes: raw.byteLength,
57
+ format,
58
+ text,
59
+ parsed,
60
+ chunks,
61
+ outline: outlineOf(parsed, chunks.length),
62
+ });
63
+ }
64
+ return { docs, skipped };
65
+ }
66
+ export const formatOf = (path) => ['.txt', '.text'].includes(extname(path).toLowerCase()) ? 'text' : 'markdown';
67
+ // ---------------------------------------------------------------------------
68
+ // discovery
69
+ // ---------------------------------------------------------------------------
70
+ async function discover(inputs, cwd) {
71
+ const out = new Set();
72
+ for (const input of inputs) {
73
+ const path = resolve(cwd, input);
74
+ if (isGlob(input)) {
75
+ // The pattern is anchored at the deepest directory it names outright,
76
+ // so a walk of the whole tree is never needed to answer one.
77
+ const anchor = staticPrefix(path);
78
+ const match = wildcard(path, { caseSensitive: true });
79
+ for (const file of await walk(anchor)) {
80
+ if (match(file)) {
81
+ out.add(file);
82
+ }
83
+ }
84
+ continue;
85
+ }
86
+ const info = await stat(path).catch(() => undefined);
87
+ if (!info) {
88
+ throw new CliError(`no such file or directory: ${input}`, EXIT.invalid);
89
+ }
90
+ if (info.isDirectory()) {
91
+ for (const file of await walk(path)) {
92
+ out.add(file);
93
+ }
94
+ }
95
+ else {
96
+ // A file named outright is indexed whatever it is called: someone
97
+ // who types a name has already decided it is a document.
98
+ out.add(path);
99
+ }
100
+ }
101
+ return [...out].sort();
102
+ }
103
+ async function walk(root) {
104
+ const out = [];
105
+ const queue = [root];
106
+ while (queue.length > 0 && out.length < MAX_FILES) {
107
+ const dir = queue.shift();
108
+ const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
109
+ for (const entry of entries) {
110
+ if (entry.name.startsWith('.') || entry.name === 'node_modules') {
111
+ continue;
112
+ }
113
+ const path = join(dir, entry.name);
114
+ if (entry.isDirectory()) {
115
+ queue.push(path);
116
+ }
117
+ else if (indexable(entry.name)) {
118
+ out.push(path);
119
+ }
120
+ }
121
+ }
122
+ return out;
123
+ }
124
+ const indexable = (name) => DOC_EXTENSIONS.includes(extname(name).toLowerCase());
125
+ /** The part of a pattern before the first wildcard, which is a real directory. */
126
+ function staticPrefix(pattern) {
127
+ const parts = pattern.split(sep);
128
+ const at = parts.findIndex((part) => isGlob(part));
129
+ const head = (at === -1 ? parts : parts.slice(0, at)).join(sep);
130
+ return head && isAbsolute(head) ? head : dirname(pattern.split('*')[0] ?? pattern);
131
+ }
132
+ /**
133
+ * The deepest directory holding every file, which is what names are taken
134
+ * relative to. One file is its own directory, so a single document is named by
135
+ * its basename rather than by an accident of where it was kept.
136
+ */
137
+ function commonRoot(files) {
138
+ const parts = files.map((file) => dirname(file).split(sep));
139
+ const first = parts[0] ?? [];
140
+ let at = 0;
141
+ while (at < first.length && parts.every((p) => p[at] === first[at])) {
142
+ at++;
143
+ }
144
+ return first.slice(0, at).join(sep) || sep;
145
+ }
146
+ function nameOf(root, absolute) {
147
+ const rel = relative(root, absolute);
148
+ // A file outside the common root cannot happen by construction; if it ever
149
+ // did, a name climbing out of the index directory must not.
150
+ return !rel || rel.startsWith('..') || isAbsolute(rel)
151
+ ? basename(absolute)
152
+ : rel.split(sep).join('/');
153
+ }
154
+ function distinct(name, taken) {
155
+ let candidate = name;
156
+ for (let n = 2; taken.has(candidate); n++) {
157
+ const dot = name.lastIndexOf('.');
158
+ candidate = dot === -1 ? `${name}_${n}` : `${name.slice(0, dot)}_${n}${name.slice(dot)}`;
159
+ }
160
+ taken.add(candidate);
161
+ return candidate;
162
+ }
163
+ // ---------------------------------------------------------------------------
164
+ // the outline
165
+ // ---------------------------------------------------------------------------
166
+ /**
167
+ * Headings and tables, with the line each ends on. That end is what makes the
168
+ * outline enough on its own: a section runs from its heading to the line before
169
+ * the next heading at the same depth or shallower, so scoping a search to a
170
+ * section, listing what is in one, or naming the sections a skipped range
171
+ * covered are all answerable without reading the document.
172
+ */
173
+ function outlineOf(doc, chunks) {
174
+ const sections = doc.sections.filter((s) => s.line !== undefined);
175
+ const headings = sections.map((section, at) => {
176
+ const next = sections.findIndex((other, i) => i > at && other.level <= section.level);
177
+ const end = next === -1 ? doc.lines.length : sections[next].line - 1;
178
+ return {
179
+ line: section.line,
180
+ end,
181
+ level: section.level,
182
+ title: section.title,
183
+ id: section.id,
184
+ path: section.path,
185
+ };
186
+ });
187
+ const tables = doc.blocks
188
+ .filter((block) => block.table)
189
+ .map((block) => {
190
+ const table = block.table;
191
+ return {
192
+ id: block.id,
193
+ path: block.path,
194
+ section: block.section.path,
195
+ line: block.start,
196
+ end: block.end,
197
+ columns: table.columns,
198
+ rows: table.rows.length,
199
+ caption: table.caption,
200
+ };
201
+ });
202
+ return {
203
+ name: doc.name,
204
+ title: doc.title,
205
+ format: doc.format,
206
+ lines: doc.lines.length,
207
+ chunks,
208
+ headings,
209
+ tables,
210
+ };
211
+ }
212
+ //# sourceMappingURL=load.js.map
@@ -0,0 +1,80 @@
1
+ import { type MatchOptions } from '../common/match.ts';
2
+ import type { DocsIndex } from './search.ts';
3
+ export interface Listing<T> {
4
+ found: number;
5
+ rows: T[];
6
+ truncated: boolean;
7
+ }
8
+ export interface ListOptions {
9
+ /** patterns over the document name */
10
+ files?: readonly string[];
11
+ exclude_files?: readonly string[];
12
+ limit?: number;
13
+ }
14
+ export declare const DEFAULT_ROWS = 50;
15
+ export declare const MAX_ROWS = 500;
16
+ export interface FileRow {
17
+ name: string;
18
+ title: string;
19
+ format: string;
20
+ lines: number;
21
+ sections: number;
22
+ tables: number;
23
+ chunks: number;
24
+ }
25
+ export declare function listFiles(index: DocsIndex, options?: ListOptions): Listing<FileRow>;
26
+ export interface SectionRow {
27
+ file: string;
28
+ id: string;
29
+ path: string;
30
+ level: number;
31
+ title: string;
32
+ line: number;
33
+ end: number;
34
+ }
35
+ export interface SectionOptions extends ListOptions {
36
+ /** a heading title, a structure id, or a structure path */
37
+ section?: readonly string[];
38
+ /** deepest heading level to report; the default is everything */
39
+ depth?: number;
40
+ }
41
+ export declare function listSections(index: DocsIndex, options?: SectionOptions): Listing<SectionRow>;
42
+ export interface TableRow {
43
+ file: string;
44
+ id: string;
45
+ section: string;
46
+ caption: string;
47
+ columns: string;
48
+ rows: number;
49
+ line: number;
50
+ end: number;
51
+ }
52
+ export declare function listTables(index: DocsIndex, options?: SectionOptions): Listing<TableRow>;
53
+ export interface LineRow {
54
+ file: string;
55
+ line: number;
56
+ text: string;
57
+ /** the innermost heading the line sits under, so a hit has a place */
58
+ section: string;
59
+ }
60
+ export interface GrepOptions extends SectionOptions, MatchOptions {
61
+ }
62
+ /**
63
+ * Lines matching a pattern. Substring by default, glob when it has wildcards,
64
+ * a regular expression when asked — the same rule every other pattern in this
65
+ * package follows, so nobody has to remember which flavour a flag takes.
66
+ */
67
+ export declare function grepLines(index: DocsIndex, pattern: string, options?: GrepOptions): Promise<Listing<LineRow>>;
68
+ export interface Verbatim {
69
+ file: string;
70
+ title: string;
71
+ start: number;
72
+ end: number;
73
+ lines: string[];
74
+ /** the document's length, so a caller can tell what it did not get */
75
+ total: number;
76
+ }
77
+ /** A named section, verbatim: heading line to the line before the next peer. */
78
+ export declare function readSection(index: DocsIndex, file: string, section: string): Promise<Verbatim>;
79
+ export declare function readRange(index: DocsIndex, file: string, from: number, to: number): Promise<Verbatim>;
80
+ //# sourceMappingURL=lookup.d.ts.map
@@ -0,0 +1,147 @@
1
+ import { CliError, EXIT } from '@zenera/cli/lib';
2
+ import { loose } from "../common/match.js";
3
+ import { under } from "./search.js";
4
+ export const DEFAULT_ROWS = 50;
5
+ export const MAX_ROWS = 500;
6
+ export function listFiles(index, options = {}) {
7
+ const names = new Set(index.resolveFiles(options.files, options.exclude_files));
8
+ const rows = index.manifest.sources
9
+ .filter((source) => names.has(source.name))
10
+ .map((source) => ({
11
+ name: source.name,
12
+ title: source.title,
13
+ format: source.format,
14
+ lines: source.lines,
15
+ sections: source.sections,
16
+ tables: source.tables,
17
+ chunks: source.chunks,
18
+ }));
19
+ return cut(rows, options.limit);
20
+ }
21
+ export function listSections(index, options = {}) {
22
+ const files = index.resolveFiles(options.files, options.exclude_files);
23
+ const within = new Set(files);
24
+ const wanted = options.section?.length
25
+ ? index.resolveSections(options.section, files).map((h) => h.path)
26
+ : [];
27
+ const rows = [];
28
+ for (const file of index.outline.files) {
29
+ if (!within.has(file.name)) {
30
+ continue;
31
+ }
32
+ for (const heading of file.headings) {
33
+ if (under(heading.path, wanted) && heading.level <= (options.depth ?? Infinity)) {
34
+ rows.push({ file: file.name, ...record(heading) });
35
+ }
36
+ }
37
+ }
38
+ return cut(rows, options.limit);
39
+ }
40
+ export function listTables(index, options = {}) {
41
+ const files = index.resolveFiles(options.files, options.exclude_files);
42
+ const within = new Set(files);
43
+ const wanted = options.section?.length
44
+ ? index.resolveSections(options.section, files).map((h) => h.path)
45
+ : [];
46
+ const rows = [];
47
+ for (const file of index.outline.files) {
48
+ if (!within.has(file.name)) {
49
+ continue;
50
+ }
51
+ for (const table of file.tables) {
52
+ if (!under(table.path, wanted)) {
53
+ continue;
54
+ }
55
+ rows.push({
56
+ file: file.name,
57
+ id: table.id,
58
+ section: table.section,
59
+ caption: table.caption,
60
+ columns: table.columns.join(', '),
61
+ rows: table.rows,
62
+ line: table.line,
63
+ end: table.end,
64
+ });
65
+ }
66
+ }
67
+ return cut(rows, options.limit);
68
+ }
69
+ /**
70
+ * Lines matching a pattern. Substring by default, glob when it has wildcards,
71
+ * a regular expression when asked — the same rule every other pattern in this
72
+ * package follows, so nobody has to remember which flavour a flag takes.
73
+ */
74
+ export async function grepLines(index, pattern, options = {}) {
75
+ const files = index.resolveFiles(options.files, options.exclude_files);
76
+ const wanted = options.section?.length
77
+ ? index.resolveSections(options.section, files)
78
+ : undefined;
79
+ const match = loose(pattern, { regex: options.regex, caseSensitive: options.caseSensitive });
80
+ const rows = [];
81
+ for (const name of files) {
82
+ const spans = wanted?.filter((h) => index.file(name)?.headings.includes(h));
83
+ if (wanted && (!spans || spans.length === 0)) {
84
+ continue;
85
+ }
86
+ const lines = await index.lines(name);
87
+ const headings = index.file(name)?.headings ?? [];
88
+ for (const [at, text] of lines.entries()) {
89
+ const line = at + 1;
90
+ if (spans && !spans.some((span) => line >= span.line && line <= span.end)) {
91
+ continue;
92
+ }
93
+ if (match(text)) {
94
+ rows.push({ file: name, line, text, section: enclosing(headings, line) });
95
+ }
96
+ }
97
+ }
98
+ return cut(rows, options.limit);
99
+ }
100
+ /** A named section, verbatim: heading line to the line before the next peer. */
101
+ export async function readSection(index, file, section) {
102
+ const found = index.resolveSections([section], [file]);
103
+ const heading = found[0];
104
+ if (!heading) {
105
+ throw new CliError(`${file} has no section called ${section}`, EXIT.failed, 'list them with `zen rag docs list sections --file <name>`');
106
+ }
107
+ return await readRange(index, file, heading.line, heading.end);
108
+ }
109
+ export async function readRange(index, file, from, to) {
110
+ const lines = await index.lines(file);
111
+ const outline = index.file(file);
112
+ const start = Math.max(1, from);
113
+ const end = Math.min(lines.length, to);
114
+ return {
115
+ file,
116
+ title: outline?.title ?? file,
117
+ start,
118
+ end,
119
+ lines: lines.slice(start - 1, end),
120
+ total: lines.length,
121
+ };
122
+ }
123
+ // ---------------------------------------------------------------------------
124
+ function cut(rows, limit) {
125
+ const take = Math.min(Math.max(1, limit ?? DEFAULT_ROWS), MAX_ROWS);
126
+ return { found: rows.length, rows: rows.slice(0, take), truncated: rows.length > take };
127
+ }
128
+ const record = (heading) => ({
129
+ id: heading.id,
130
+ path: heading.path,
131
+ level: heading.level,
132
+ title: heading.title,
133
+ line: heading.line,
134
+ end: heading.end,
135
+ });
136
+ /** The last heading at or above this line, which is the section it is in. */
137
+ function enclosing(headings, line) {
138
+ let title = '';
139
+ for (const heading of headings) {
140
+ if (heading.line > line) {
141
+ break;
142
+ }
143
+ title = heading.title;
144
+ }
145
+ return title;
146
+ }
147
+ //# sourceMappingURL=lookup.js.map
@@ -0,0 +1,95 @@
1
+ /** What a line, a block or a chunk is. The vocabulary is closed on purpose. */
2
+ export type StructureKind = 'heading' | 'paragraph' | 'blockquote' | 'list' | 'list_item' | 'table' | 'table_header' | 'table_row' | 'code' | 'frontmatter' | 'html' | 'hr';
3
+ /** The root of every path: a document is the structure everything else sits in. */
4
+ export declare const ROOT_SEGMENT = "doc";
5
+ export type DocFormat = 'markdown' | 'text';
6
+ /**
7
+ * A heading and everything under it, until a heading of the same depth or
8
+ * shallower. mdast has no such node — a heading there is a SIBLING of the
9
+ * paragraphs that follow it — so the nesting is built here.
10
+ */
11
+ export interface Section {
12
+ id: string;
13
+ path: string;
14
+ title: string;
15
+ /** 0 for the document itself, otherwise the heading depth */
16
+ level: number;
17
+ /** the heading's own line; absent on the document root */
18
+ line: number | undefined;
19
+ parent: Section | undefined;
20
+ }
21
+ export interface TableRowBlock {
22
+ id: string;
23
+ path: string;
24
+ line: number;
25
+ cells: string[];
26
+ /** the row as prose, so it carries its own column names */
27
+ text: string;
28
+ }
29
+ export interface TableBlock {
30
+ columns: string[];
31
+ headerLine: number;
32
+ /** the alignment row; mdast emits no node for it, so it is derived */
33
+ separatorLine: number | undefined;
34
+ /** the paragraph before the table, which is the only caption one ever gets */
35
+ caption: string;
36
+ /** the column that identifies a row, repeated into every slice of it */
37
+ keyColumn: number;
38
+ rows: TableRowBlock[];
39
+ }
40
+ export interface ListItemBlock {
41
+ id: string;
42
+ path: string;
43
+ start: number;
44
+ end: number;
45
+ text: string;
46
+ }
47
+ export interface Block {
48
+ id: string;
49
+ path: string;
50
+ kind: StructureKind;
51
+ /** 1-based, inclusive */
52
+ start: number;
53
+ end: number;
54
+ section: Section;
55
+ /** normalized, never shown: this is what is embedded and indexed */
56
+ text: string;
57
+ items?: ListItemBlock[];
58
+ table?: TableBlock;
59
+ }
60
+ export interface ParsedDoc {
61
+ /** the document's name within the index — its path, and its identity */
62
+ name: string;
63
+ title: string;
64
+ format: DocFormat;
65
+ /** verbatim, CRLF normalized, 1-based when indexed as `lines[n - 1]` */
66
+ lines: string[];
67
+ sections: Section[];
68
+ blocks: Block[];
69
+ }
70
+ /** Every ancestor of a section, the document root first, itself last. */
71
+ export declare function ancestry(section: Section): Section[];
72
+ /** The breadcrumb, which prefixes every chunk of text under this section. */
73
+ export declare const headingPath: (section: Section) => string;
74
+ /** The heading lines a chunk under this section must be rendered with. */
75
+ export declare const headingLines: (section: Section) => number[];
76
+ /**
77
+ * Everything downstream indexes `lines` by number, and mdast positions refer to
78
+ * the exact string handed to the parser — so the normalization happens once,
79
+ * here, and the normalized text is the only version anything ever sees.
80
+ */
81
+ export declare const normalize: (text: string) => string;
82
+ export declare function parseDocument(source: string, name: string, format: DocFormat): ParsedDoc;
83
+ /**
84
+ * A data row does not contain its own column names. Stripping the pipes leaves
85
+ * `V-200-30 3" WCC 98.2`, in which the word `Cv` appears nowhere, so a query
86
+ * naming a column could only ever match the header line and never a row — and
87
+ * hybrid search would silently become vector-only for every table in the
88
+ * corpus. Pairing each cell with its header is what fixes that, and it embeds
89
+ * better too, because `Cv: 45` is a sentence and `| 45 |` is not.
90
+ */
91
+ export declare function rowText(columns: readonly string[], cells: readonly string[]): string;
92
+ export declare function cellText(column: string | undefined, cell: string): string;
93
+ /** Soft wraps inside a paragraph are wrapping, not meaning, so they rejoin. */
94
+ export declare const collapse: (text: string) => string;
95
+ //# sourceMappingURL=parse.d.ts.map