@zenera/rag 1.1.8 → 1.1.10

package/dist/docs/parse.js

@@ -0,0 +1,372 @@
+import remarkFrontmatter from 'remark-frontmatter';
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+/** Path segments carry their kind, so heading ancestors stay recoverable. */
+const SEGMENT = {
+    heading: 'sec',
+    paragraph: 'para',
+    blockquote: 'quote',
+    list: 'list',
+    list_item: 'item',
+    table: 'table',
+    table_header: 'header',
+    table_row: 'row',
+    code: 'code',
+    frontmatter: 'fm',
+    html: 'html',
+    hr: 'hr',
+};
+/** The root of every path: a document is the structure everything else sits in. */
+export const ROOT_SEGMENT = 'doc';
+const processor = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkFrontmatter, ['yaml', 'toml']);
+// ---------------------------------------------------------------------------
+/** Every ancestor of a section, the document root first, itself last. */
+export function ancestry(section) {
+    const out = [];
+    for (let at = section; at; at = at.parent) {
+        out.unshift(at);
+    }
+    return out;
+}
+/** The breadcrumb, which prefixes every chunk of text under this section. */
+export const headingPath = (section) => ancestry(section)
+    .map((s) => s.title)
+    .filter(Boolean)
+    .join(' > ');
+/** The heading lines a chunk under this section must be rendered with. */
+export const headingLines = (section) => ancestry(section)
+    .map((s) => s.line)
+    .filter((line) => line !== undefined);
+/**
+ * Everything downstream indexes `lines` by number, and mdast positions refer to
+ * the exact string handed to the parser — so the normalization happens once,
+ * here, and the normalized text is the only version anything ever sees.
+ */
+export const normalize = (text) => text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n');
+export function parseDocument(source, name, format) {
+    const text = normalize(source);
+    const lines = text.split('\n');
+    return format === 'text' ? plain(text, lines, name) : markdown(text, lines, name);
+}
+function markdown(text, lines, name) {
+    const tree = processor.parse(text);
+    const root = {
+        id: ROOT_SEGMENT,
+        path: ROOT_SEGMENT,
+        title: stripExtension(name),
+        level: 0,
+        line: undefined,
+        parent: undefined,
+    };
+    const walk = { lines, counters: new Map(), sections: [root], blocks: [] };
+    let section = root;
+    let title = '';
+    // Only ever holds the paragraph immediately before the current position,
+    // which is the whole of what a markdown table gets for a caption.
+    let lead = '';
+    for (const node of tree.children) {
+        if (node.type === 'heading') {
+            const heading = node;
+            const line = lineOf(heading);
+            const text = collapse(inline(heading.children));
+            while (section.parent && section.level >= heading.depth) {
+                section = section.parent;
+            }
+            section = {
+                id: id(walk, 'heading'),
+                path: '',
+                title: text,
+                level: heading.depth,
+                line,
+                parent: section,
+            };
+            section.path = `${section.parent.path}/${section.id}`;
+            walk.sections.push(section);
+            if (!title && heading.depth <= 2) {
+                title = text;
+            }
+            lead = '';
+            continue;
+        }
+        const block = blockOf(node, section, walk, lead);
+        if (block) {
+            walk.blocks.push(block);
+            lead = block.kind === 'paragraph' ? block.text : '';
+        }
+    }
+    return {
+        name,
+        title: title || stripExtension(basenameOf(name)),
+        format: 'markdown',
+        lines,
+        sections: walk.sections,
+        blocks: walk.blocks,
+    };
+}
+function blockOf(node, section, walk, lead) {
+    const start = lineOf(node);
+    const end = endLineOf(node);
+    switch (node.type) {
+        case 'paragraph':
+            return make('paragraph', collapse(inline(node.children)));
+        case 'blockquote':
+            return make('blockquote', collapse(blocks(node.children)));
+        case 'code': {
+            const code = node;
+            return make('code', code.value);
+        }
+        case 'html': {
+            const stripped = collapse(tags(node.value));
+            return stripped.length >= 3 ? make('html', stripped) : undefined;
+        }
+        case 'yaml':
+            return make('frontmatter', String(node.value ?? ''));
+        case 'thematicBreak':
+            return make('hr', '');
+        case 'list':
+            return list(node, section, walk, start, end);
+        case 'table':
+            return table(node, section, walk, start, end, lead);
+        default:
+            // A `+++` block. remark-frontmatter puts it in the tree; mdast's own
+            // node union names only the `---` one.
+            return node.type === 'toml'
+                ? make('frontmatter', String(node.value ?? ''))
+                : undefined;
+    }
+    function make(kind, text) {
+        const own = id(walk, kind);
+        return { id: own, path: `${section.path}/${own}`, kind, start, end, section, text };
+    }
+}
+function list(node, section, walk, start, end) {
+    const own = id(walk, 'list');
+    const path = `${section.path}/${own}`;
+    const ordinal = node.start ?? 1;
+    const items = node.children.map((child, at) => {
+        const item = child;
+        const mine = id(walk, 'list_item');
+        const marker = node.ordered ? `${ordinal + at}.` : '-';
+        // The marker is kept: enumeration and step order are meaning, and a
+        // numbered step reads differently from a bullet.
+        return {
+            id: mine,
+            path: `${path}/${mine}`,
+            start: lineOf(item),
+            end: endLineOf(item),
+            text: `${marker} ${collapse(blocks(item.children))}`.trim(),
+        };
+    });
+    return {
+        id: own,
+        path,
+        kind: 'list',
+        start,
+        end,
+        section,
+        text: items.map((i) => i.text).join('\n'),
+        items,
+    };
+}
+function table(node, section, walk, start, end, caption) {
+    const own = id(walk, 'table');
+    const path = `${section.path}/${own}`;
+    const [header, ...body] = node.children;
+    const columns = header ? cellsOf(header) : [];
+    const headerLine = header ? lineOf(header) : start;
+    const firstRow = body[0] ? lineOf(body[0]) : undefined;
+    // mdast has no node for the alignment row — alignment lives on the table —
+    // so it is the line after the header, when there is room for it.
+    const separatorLine = firstRow === undefined || firstRow > headerLine + 1 ? headerLine + 1 : undefined;
+    const rows = body.map((row) => {
+        const mine = id(walk, 'table_row');
+        const cells = cellsOf(row);
+        return {
+            id: mine,
+            path: `${path}/${mine}`,
+            line: lineOf(row),
+            cells,
+            text: rowText(columns, cells),
+        };
+    });
+    return {
+        id: own,
+        path,
+        kind: 'table',
+        start,
+        end,
+        section,
+        text: rows.map((r) => r.text).join('\n'),
+        table: {
+            columns,
+            headerLine,
+            separatorLine,
+            caption,
+            keyColumn: keyColumnOf(columns, rows),
+            rows,
+        },
+    };
+}
+/**
+ * A data row does not contain its own column names. Stripping the pipes leaves
+ * `V-200-30 3" WCC 98.2`, in which the word `Cv` appears nowhere, so a query
+ * naming a column could only ever match the header line and never a row — and
+ * hybrid search would silently become vector-only for every table in the
+ * corpus. Pairing each cell with its header is what fixes that, and it embeds
+ * better too, because `Cv: 45` is a sentence and `| 45 |` is not.
+ */
+export function rowText(columns, cells) {
+    return cells
+        .map((cell, at) => cellText(columns[at], cell))
+        .filter(Boolean)
+        .join(' ');
+}
+export function cellText(column, cell) {
+    const value = cell.trim();
+    if (!value) {
+        return '';
+    }
+    return column ? `${column}: ${value}.` : `${value}.`;
+}
+/** Column 1, unless it is numeric — a slice with no name on it is unattributable. */
+function keyColumnOf(columns, rows) {
+    const numeric = (at) => rows.length > 0 &&
+        rows.every((row) => {
+            const cell = (row.cells[at] ?? '').trim();
+            return cell === '' || /^[-+]?[\d.,%\s]+$/.test(cell);
+        });
+    for (let at = 0; at < columns.length; at++) {
+        if (!numeric(at)) {
+            return at;
+        }
+    }
+    return 0;
+}
+const cellsOf = (row) => row.children.map((cell) => collapse(inline(cell.children)));
+// ---------------------------------------------------------------------------
+// plain text
+// ---------------------------------------------------------------------------
+/**
+ * A `.txt` file is read as paragraphs and nothing else. It would be easy to run
+ * it through the markdown parser and get headings for free, but they would be
+ * invented: a line beginning with `#` in a log or a licence is not a title, and
+ * an index that says it is would scope searches to sections nobody wrote.
+ */
+function plain(text, lines, name) {
+    const root = {
+        id: ROOT_SEGMENT,
+        path: ROOT_SEGMENT,
+        title: stripExtension(name),
+        level: 0,
+        line: undefined,
+        parent: undefined,
+    };
+    const blocks = [];
+    let at = 0;
+    let ordinal = 0;
+    while (at < lines.length) {
+        if ((lines[at] ?? '').trim() === '') {
+            at++;
+            continue;
+        }
+        const start = at;
+        while (at < lines.length && (lines[at] ?? '').trim() !== '') {
+            at++;
+        }
+        const own = `${SEGMENT.paragraph}:${++ordinal}`;
+        blocks.push({
+            id: own,
+            path: `${root.path}/${own}`,
+            kind: 'paragraph',
+            start: start + 1,
+            end: at,
+            section: root,
+            text: collapse(lines.slice(start, at).join(' ')),
+        });
+    }
+    const first = lines.find((line) => line.trim() !== '')?.trim() ?? '';
+    return {
+        name,
+        title: first.length > 0 && first.length <= 80 ? first : stripExtension(basenameOf(name)),
+        format: 'text',
+        lines,
+        sections: [root],
+        blocks,
+    };
+}
+// ---------------------------------------------------------------------------
+// serialization — walking the inline tree, never a regex over the markup
+// ---------------------------------------------------------------------------
+function inline(nodes) {
+    return nodes.map(one).join('');
+}
+function one(node) {
+    switch (node.type) {
+        case 'text':
+            return node.value;
+        // The value, not the markup: this is the node a regex stripper eats.
+        case 'inlineCode':
+            return node.value;
+        case 'image':
+        case 'imageReference':
+            // Distinguishable from a sentence on purpose.
+            return node.alt ? `image: ${node.alt}` : '';
+        case 'break':
+            return ' ';
+        // Inline html carries no words worth indexing, and its angle brackets
+        // would only ever match a query by accident.
+        case 'html':
+            return '';
+        case 'footnoteReference':
+            return '';
+        default:
+            return 'children' in node ? inline(node.children) : '';
+    }
+}
+/** Nested block content — a blockquote's paragraphs, a list item's children. */
+function blocks(nodes) {
+    return nodes
+        .map((node) => {
+        switch (node.type) {
+            case 'paragraph':
+            case 'heading':
+                return inline(node.children);
+            case 'code':
+                return node.value;
+            case 'blockquote':
+                return blocks(node.children);
+            case 'list':
+                return node.children
+                    .map((item) => blocks(item.children))
+                    .join(' ');
+            case 'table':
+                return node.children
+                    .slice(1)
+                    .map((row) => rowText(cellsOf(node.children[0]), cellsOf(row)))
+                    .join(' ');
+            default:
+                return '';
+        }
+    })
+        .filter(Boolean)
+        .join(' ');
+}
+/** Soft wraps inside a paragraph are wrapping, not meaning, so they rejoin. */
+export const collapse = (text) => text.replace(/\s+/g, ' ').trim();
+const tags = (html) => html.replace(/<[^>]*>/g, ' ');
+// ---------------------------------------------------------------------------
+function id(walk, kind) {
+    const segment = SEGMENT[kind];
+    const next = (walk.counters.get(segment) ?? 0) + 1;
+    walk.counters.set(segment, next);
+    return `${segment}:${next}`;
+}
+const lineOf = (node) => node.position?.start.line ?? 1;
+const endLineOf = (node) => node.position?.end.line ?? 1;
+const basenameOf = (name) => name.split('/').pop() ?? name;
+const stripExtension = (name) => name.replace(/\.[^./]+$/, '');
+//# sourceMappingURL=parse.js.map

package/dist/docs/readme.d.ts

@@ -0,0 +1,6 @@
+import { type Report } from '../common/progress.ts';
+import type { Counts, Manifest } from './files.ts';
+export type Phase = 'reading' | 'embedding' | 'writing';
+export declare const PHASES: Record<Phase, string>;
+export declare const DOCS_REPORT: Report<Counts, Manifest>;
+//# sourceMappingURL=readme.d.ts.map

package/dist/docs/readme.js

@@ -0,0 +1,122 @@
+import { basename } from 'node:path';
+import { INTERVAL_MS, } from "../common/progress.js";
+import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+export const PHASES = {
+    reading: 'reading the documents and cutting them into chunks',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+export const DOCS_REPORT = { building, complete, failed };
+function building(state) {
+    const rows = [
+        ['documents', summary(state.documents)],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(state.now - state.started)} ago)`,
+        ],
+        ['step', state.step],
+    ];
+    if (state.summary) {
+        rows.push(['found', counted(state.summary)]);
+    }
+    if (state.total > 0) {
+        const percent = Math.round((state.done / state.total) * 100);
+        rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
+    }
+    rows.push(['updated', new Date(state.now).toISOString()]);
+    return [
+        '# Document index — being built',
+        '',
+        'A searchable index of the documents named below, written by `zen rag docs index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(state) {
+    const { manifest } = state;
+    return [
+        `# Document index — ${basename(state.dir)}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(state.ms)}.`,
+        'Ask it a question and it answers with the passages that matched, quoted verbatim from',
+        'the copies kept in `sources/` — line numbers included, and with a marker wherever',
+        'something between two of them was left out.',
+        '',
+        '## What it covers',
+        '',
+        ...documentTable(manifest.sources),
+        '',
+        `${counted(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['outline.json', 'every heading and table, with the lines they cover'],
+            ['sources/', 'the documents themselves, verbatim: where the quotes come from'],
+            ['lance/', 'the chunks: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag docs search --dir . "what you are after"',
+        'zen rag docs search --dir . --file "guides/**" --kind table "pressure rating"',
+        'zen rag docs list files --dir .',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Document index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag docs index`.',
+        '',
+        ...fields([
+            ['documents', summary(state.documents)],
+            ['step', state.step],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+const HEADERS = ['document', 'format', 'lines', 'sections', 'tables', 'chunks'];
+const documentTable = (sources) => grid(HEADERS, sources.map((s) => [
+    s.name,
+    s.format,
+    String(s.lines),
+    String(s.sections),
+    String(s.tables),
+    String(s.chunks),
+]));
+function counted(counts) {
+    return (`${plural(counts.chunks, 'chunk')} over ${plural(counts.documents, 'document')}: ` +
+        `${counts.lines} lines, ${counts.sections} sections, ${counts.tables} tables`);
+}
+/** A corpus can be thousands of files; a README listing them all is a wall. */
+function summary(documents) {
+    const shown = documents.slice(0, 12).join(', ');
+    return documents.length > 12 ? `${shown}, and ${documents.length - 12} more` : shown;
+}
+//# sourceMappingURL=readme.js.map

package/dist/docs/render.d.ts

@@ -0,0 +1,13 @@
+import type { Assembly, Excerpt } from './assemble.ts';
+import type { Match } from './search.ts';
+export interface RenderOptions {
+    /** the line-number gutter; on unless something else is going to eat this */
+    numbers?: boolean;
+    colour?: boolean;
+}
+export declare function renderAssembly(assembly: Assembly, options?: RenderOptions): string;
+export declare function renderExcerpt(file: Excerpt, options?: RenderOptions): string[];
+/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
+export declare const matchRows: (matches: readonly Match[]) => string[][];
+export declare const MATCH_HEADERS: readonly ["id", "kind", "lines", "score", "vec / txt", "heading"];
+//# sourceMappingURL=render.d.ts.map

package/dist/docs/render.js

@@ -0,0 +1,46 @@
+import { bold, dim } from '@zenera/cli/lib';
+export function renderAssembly(assembly, options = {}) {
+    const out = [];
+    for (const file of assembly.files) {
+        out.push(...renderExcerpt(file, options), '');
+    }
+    if (assembly.truncated) {
+        out.push('(cut short by the line budget — raise it with --max-lines)');
+    }
+    return out.join('\n').trimEnd();
+}
+export function renderExcerpt(file, options = {}) {
+    const paint = options.colour === false ? (s) => s : undefined;
+    const strong = paint ?? bold;
+    const faint = paint ?? dim;
+    const width = String(file.lines).length;
+    return [
+        `## ${strong(file.path)} ${faint(`— ${file.shown} of ${file.lines} lines`)}`,
+        '',
+        ...file.pieces.flatMap((piece) => renderPiece(piece, width, options, faint)),
+    ];
+}
+function renderPiece(piece, width, options, faint) {
+    if (piece.type === 'omission') {
+        const named = piece.sections.length > 0 ? ` (${piece.sections.join(', ')})` : '';
+        return [faint(`... ${piece.count} lines omitted${named} ...`), ''];
+    }
+    const lines = piece.lines.map((line, at) => options.numbers === false
+        ? line
+        : `${faint(String(piece.start + at).padStart(width))} ${faint('|')} ${line}`);
+    return [...lines, ''];
+}
+/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
+export const matchRows = (matches) => matches.map((m) => [
+    m.id,
+    m.kind,
+    `${m.bodyStart}-${m.bodyEnd}`,
+    m.score.toFixed(4),
+    // Fusion ranks; these say whether the rank was worth anything.
+    [m.relevance.vector?.toFixed(2), m.relevance.text?.toFixed(1)]
+        .map((v) => v ?? '·')
+        .join(' / '),
+    m.headings.split('\n')[0] ?? '',
+]);
+export const MATCH_HEADERS = ['id', 'kind', 'lines', 'score', 'vec / txt', 'heading'];
+//# sourceMappingURL=render.js.map

package/dist/docs/repl.d.ts

@@ -0,0 +1,7 @@
+import { type AssembleOptions } from './assemble.ts';
+import { type DocsIndex, type DocsQuery } from './search.ts';
+export interface ReplSettings extends AssembleOptions {
+    quiet?: boolean;
+}
+export declare function repl(index: DocsIndex, initial: DocsQuery, settings?: ReplSettings): Promise<void>;
+//# sourceMappingURL=repl.d.ts.map