@zenera/rag 1.1.9 → 1.1.11

package/dist/docs/parse-cache.js

@@ -0,0 +1,60 @@
+import { Cache, cacheKey } from '@zenera/cli/lib';
+import { createHash } from 'node:crypto';
+// ---------------------------------------------------------------------------
+// Not parsing the same document twice
+//
+// Once vectors are cached, parsing is what a rebuild of an unchanged corpus
+// spends all of its time on — the threads make it several times faster, and
+// this makes it nothing at all. A document that has not changed produces
+// exactly the chunks it produced last time, so the chunks are what is kept.
+//
+// The key is the file's bytes together with everything that decides how they
+// are cut: change the document, the chunk settings or this module, and the
+// entry is a miss rather than a wrong answer. It sits in the machine's shared
+// cache next to the vectors, so a corpus indexed into two directories is read
+// once, and it follows the same rule everything there does — every error is a
+// miss and nothing else.
+// ---------------------------------------------------------------------------
+export const PARSE_KIND = 'docs-parse';
+/** Bumped when chunking changes shape, which invalidates every entry. */
+export const PARSE_VERSION = 1;
+export const NO_PARSE_CACHE = {
+    get: () => undefined,
+    put: () => { },
+    commit: () => { },
+    hits: 0,
+};
+/**
+ * What the chunks depend on, and nothing else. `tokenCount` is a function and
+ * cannot be hashed; a caller that supplies one gets no cache rather than a key
+ * that quietly ignores it.
+ */
+export function parseKey(bytes, name, options) {
+    if (typeof options.tokenCount === 'function') {
+        return undefined;
+    }
+    return cacheKey(PARSE_VERSION, name, options.chunkTokens, options.minChunkTokens, options.maxChunkTokens, options.tableSliceTokens, createHash('sha256').update(bytes).digest('hex'));
+}
+export function openParseCache(dir) {
+    return new StoredParses(new Cache(PARSE_KIND, { dir }));
+}
+class StoredParses {
+    #store;
+    constructor(store) {
+        this.#store = store;
+    }
+    get hits() {
+        return this.#store.hits;
+    }
+    get(key) {
+        const found = this.#store.get(key);
+        return found?.chunks && found.outline ? found : undefined;
+    }
+    put(key, parsed) {
+        this.#store.put(key, { chunks: parsed.chunks, outline: parsed.outline });
+    }
+    commit() {
+        this.#store.commit();
+    }
+}
+//# sourceMappingURL=parse-cache.js.map

package/dist/docs/parse-worker.d.ts

@@ -0,0 +1,13 @@
+export interface ParseJob {
+    at: number;
+    name: string;
+    text: string;
+    format: 'markdown' | 'text';
+}
+export interface ParseDone {
+    at: number;
+    chunks?: unknown;
+    outline?: unknown;
+    error?: string;
+}
+//# sourceMappingURL=parse-worker.d.ts.map

package/dist/docs/parse-worker.js

@@ -0,0 +1,21 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { chunkDocument } from "./chunk.js";
+import { outlineOf } from "./outline.js";
+import { parseDocument } from "./parse.js";
+const options = (workerData ?? {});
+parentPort?.on('message', (job) => {
+    try {
+        const parsed = parseDocument(job.text, job.name, job.format);
+        const chunks = chunkDocument(parsed, options);
+        parentPort.postMessage({ at: job.at, chunks, outline: outlineOf(parsed, chunks.length) });
+    }
+    catch (err) {
+        // Sent back rather than thrown: an uncaught throw here kills the thread,
+        // and the pool would lose the other documents queued behind this one.
+        parentPort.postMessage({
+            at: job.at,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+});
+//# sourceMappingURL=parse-worker.js.map

package/dist/docs/parse.d.ts

@@ -0,0 +1,95 @@
+/** What a line, a block or a chunk is. The vocabulary is closed on purpose. */
+export type StructureKind = 'heading' | 'paragraph' | 'blockquote' | 'list' | 'list_item' | 'table' | 'table_header' | 'table_row' | 'code' | 'frontmatter' | 'html' | 'hr';
+/** The root of every path: a document is the structure everything else sits in. */
+export declare const ROOT_SEGMENT = "doc";
+export type DocFormat = 'markdown' | 'text';
+/**
+ * A heading and everything under it, until a heading of the same depth or
+ * shallower. mdast has no such node — a heading there is a SIBLING of the
+ * paragraphs that follow it — so the nesting is built here.
+ */
+export interface Section {
+    id: string;
+    path: string;
+    title: string;
+    /** 0 for the document itself, otherwise the heading depth */
+    level: number;
+    /** the heading's own line; absent on the document root */
+    line: number | undefined;
+    parent: Section | undefined;
+}
+export interface TableRowBlock {
+    id: string;
+    path: string;
+    line: number;
+    cells: string[];
+    /** the row as prose, so it carries its own column names */
+    text: string;
+}
+export interface TableBlock {
+    columns: string[];
+    headerLine: number;
+    /** the alignment row; mdast emits no node for it, so it is derived */
+    separatorLine: number | undefined;
+    /** the paragraph before the table, which is the only caption one ever gets */
+    caption: string;
+    /** the column that identifies a row, repeated into every slice of it */
+    keyColumn: number;
+    rows: TableRowBlock[];
+}
+export interface ListItemBlock {
+    id: string;
+    path: string;
+    start: number;
+    end: number;
+    text: string;
+}
+export interface Block {
+    id: string;
+    path: string;
+    kind: StructureKind;
+    /** 1-based, inclusive */
+    start: number;
+    end: number;
+    section: Section;
+    /** normalized, never shown: this is what is embedded and indexed */
+    text: string;
+    items?: ListItemBlock[];
+    table?: TableBlock;
+}
+export interface ParsedDoc {
+    /** the document's name within the index — its path, and its identity */
+    name: string;
+    title: string;
+    format: DocFormat;
+    /** verbatim, CRLF normalized, 1-based when indexed as `lines[n - 1]` */
+    lines: string[];
+    sections: Section[];
+    blocks: Block[];
+}
+/** Every ancestor of a section, the document root first, itself last. */
+export declare function ancestry(section: Section): Section[];
+/** The breadcrumb, which prefixes every chunk of text under this section. */
+export declare const headingPath: (section: Section) => string;
+/** The heading lines a chunk under this section must be rendered with. */
+export declare const headingLines: (section: Section) => number[];
+/**
+ * Everything downstream indexes `lines` by number, and mdast positions refer to
+ * the exact string handed to the parser — so the normalization happens once,
+ * here, and the normalized text is the only version anything ever sees.
+ */
+export declare const normalize: (text: string) => string;
+export declare function parseDocument(source: string, name: string, format: DocFormat): ParsedDoc;
+/**
+ * A data row does not contain its own column names. Stripping the pipes leaves
+ * `V-200-30 3" WCC 98.2`, in which the word `Cv` appears nowhere, so a query
+ * naming a column could only ever match the header line and never a row — and
+ * hybrid search would silently become vector-only for every table in the
+ * corpus. Pairing each cell with its header is what fixes that, and it embeds
+ * better too, because `Cv: 45` is a sentence and `| 45 |` is not.
+ */
+export declare function rowText(columns: readonly string[], cells: readonly string[]): string;
+export declare function cellText(column: string | undefined, cell: string): string;
+/** Soft wraps inside a paragraph are wrapping, not meaning, so they rejoin. */
+export declare const collapse: (text: string) => string;
+//# sourceMappingURL=parse.d.ts.map

package/dist/docs/parse.js

@@ -0,0 +1,372 @@
+import remarkFrontmatter from 'remark-frontmatter';
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+/** Path segments carry their kind, so heading ancestors stay recoverable. */
+const SEGMENT = {
+    heading: 'sec',
+    paragraph: 'para',
+    blockquote: 'quote',
+    list: 'list',
+    list_item: 'item',
+    table: 'table',
+    table_header: 'header',
+    table_row: 'row',
+    code: 'code',
+    frontmatter: 'fm',
+    html: 'html',
+    hr: 'hr',
+};
+/** The root of every path: a document is the structure everything else sits in. */
+export const ROOT_SEGMENT = 'doc';
+const processor = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkFrontmatter, ['yaml', 'toml']);
+// ---------------------------------------------------------------------------
+/** Every ancestor of a section, the document root first, itself last. */
+export function ancestry(section) {
+    const out = [];
+    for (let at = section; at; at = at.parent) {
+        out.unshift(at);
+    }
+    return out;
+}
+/** The breadcrumb, which prefixes every chunk of text under this section. */
+export const headingPath = (section) => ancestry(section)
+    .map((s) => s.title)
+    .filter(Boolean)
+    .join(' > ');
+/** The heading lines a chunk under this section must be rendered with. */
+export const headingLines = (section) => ancestry(section)
+    .map((s) => s.line)
+    .filter((line) => line !== undefined);
+/**
+ * Everything downstream indexes `lines` by number, and mdast positions refer to
+ * the exact string handed to the parser — so the normalization happens once,
+ * here, and the normalized text is the only version anything ever sees.
+ */
+export const normalize = (text) => text.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n');
+export function parseDocument(source, name, format) {
+    const text = normalize(source);
+    const lines = text.split('\n');
+    return format === 'text' ? plain(text, lines, name) : markdown(text, lines, name);
+}
+function markdown(text, lines, name) {
+    const tree = processor.parse(text);
+    const root = {
+        id: ROOT_SEGMENT,
+        path: ROOT_SEGMENT,
+        title: stripExtension(name),
+        level: 0,
+        line: undefined,
+        parent: undefined,
+    };
+    const walk = { lines, counters: new Map(), sections: [root], blocks: [] };
+    let section = root;
+    let title = '';
+    // Only ever holds the paragraph immediately before the current position,
+    // which is the whole of what a markdown table gets for a caption.
+    let lead = '';
+    for (const node of tree.children) {
+        if (node.type === 'heading') {
+            const heading = node;
+            const line = lineOf(heading);
+            const text = collapse(inline(heading.children));
+            while (section.parent && section.level >= heading.depth) {
+                section = section.parent;
+            }
+            section = {
+                id: id(walk, 'heading'),
+                path: '',
+                title: text,
+                level: heading.depth,
+                line,
+                parent: section,
+            };
+            section.path = `${section.parent.path}/${section.id}`;
+            walk.sections.push(section);
+            if (!title && heading.depth <= 2) {
+                title = text;
+            }
+            lead = '';
+            continue;
+        }
+        const block = blockOf(node, section, walk, lead);
+        if (block) {
+            walk.blocks.push(block);
+            lead = block.kind === 'paragraph' ? block.text : '';
+        }
+    }
+    return {
+        name,
+        title: title || stripExtension(basenameOf(name)),
+        format: 'markdown',
+        lines,
+        sections: walk.sections,
+        blocks: walk.blocks,
+    };
+}
+function blockOf(node, section, walk, lead) {
+    const start = lineOf(node);
+    const end = endLineOf(node);
+    switch (node.type) {
+        case 'paragraph':
+            return make('paragraph', collapse(inline(node.children)));
+        case 'blockquote':
+            return make('blockquote', collapse(blocks(node.children)));
+        case 'code': {
+            const code = node;
+            return make('code', code.value);
+        }
+        case 'html': {
+            const stripped = collapse(tags(node.value));
+            return stripped.length >= 3 ? make('html', stripped) : undefined;
+        }
+        case 'yaml':
+            return make('frontmatter', String(node.value ?? ''));
+        case 'thematicBreak':
+            return make('hr', '');
+        case 'list':
+            return list(node, section, walk, start, end);
+        case 'table':
+            return table(node, section, walk, start, end, lead);
+        default:
+            // A `+++` block. remark-frontmatter puts it in the tree; mdast's own
+            // node union names only the `---` one.
+            return node.type === 'toml'
+                ? make('frontmatter', String(node.value ?? ''))
+                : undefined;
+    }
+    function make(kind, text) {
+        const own = id(walk, kind);
+        return { id: own, path: `${section.path}/${own}`, kind, start, end, section, text };
+    }
+}
+function list(node, section, walk, start, end) {
+    const own = id(walk, 'list');
+    const path = `${section.path}/${own}`;
+    const ordinal = node.start ?? 1;
+    const items = node.children.map((child, at) => {
+        const item = child;
+        const mine = id(walk, 'list_item');
+        const marker = node.ordered ? `${ordinal + at}.` : '-';
+        // The marker is kept: enumeration and step order are meaning, and a
+        // numbered step reads differently from a bullet.
+        return {
+            id: mine,
+            path: `${path}/${mine}`,
+            start: lineOf(item),
+            end: endLineOf(item),
+            text: `${marker} ${collapse(blocks(item.children))}`.trim(),
+        };
+    });
+    return {
+        id: own,
+        path,
+        kind: 'list',
+        start,
+        end,
+        section,
+        text: items.map((i) => i.text).join('\n'),
+        items,
+    };
+}
+function table(node, section, walk, start, end, caption) {
+    const own = id(walk, 'table');
+    const path = `${section.path}/${own}`;
+    const [header, ...body] = node.children;
+    const columns = header ? cellsOf(header) : [];
+    const headerLine = header ? lineOf(header) : start;
+    const firstRow = body[0] ? lineOf(body[0]) : undefined;
+    // mdast has no node for the alignment row — alignment lives on the table —
+    // so it is the line after the header, when there is room for it.
+    const separatorLine = firstRow === undefined || firstRow > headerLine + 1 ? headerLine + 1 : undefined;
+    const rows = body.map((row) => {
+        const mine = id(walk, 'table_row');
+        const cells = cellsOf(row);
+        return {
+            id: mine,
+            path: `${path}/${mine}`,
+            line: lineOf(row),
+            cells,
+            text: rowText(columns, cells),
+        };
+    });
+    return {
+        id: own,
+        path,
+        kind: 'table',
+        start,
+        end,
+        section,
+        text: rows.map((r) => r.text).join('\n'),
+        table: {
+            columns,
+            headerLine,
+            separatorLine,
+            caption,
+            keyColumn: keyColumnOf(columns, rows),
+            rows,
+        },
+    };
+}
+/**
+ * A data row does not contain its own column names. Stripping the pipes leaves
+ * `V-200-30 3" WCC 98.2`, in which the word `Cv` appears nowhere, so a query
+ * naming a column could only ever match the header line and never a row — and
+ * hybrid search would silently become vector-only for every table in the
+ * corpus. Pairing each cell with its header is what fixes that, and it embeds
+ * better too, because `Cv: 45` is a sentence and `| 45 |` is not.
+ */
+export function rowText(columns, cells) {
+    return cells
+        .map((cell, at) => cellText(columns[at], cell))
+        .filter(Boolean)
+        .join(' ');
+}
+export function cellText(column, cell) {
+    const value = cell.trim();
+    if (!value) {
+        return '';
+    }
+    return column ? `${column}: ${value}.` : `${value}.`;
+}
+/** Column 1, unless it is numeric — a slice with no name on it is unattributable. */
+function keyColumnOf(columns, rows) {
+    const numeric = (at) => rows.length > 0 &&
+        rows.every((row) => {
+            const cell = (row.cells[at] ?? '').trim();
+            return cell === '' || /^[-+]?[\d.,%\s]+$/.test(cell);
+        });
+    for (let at = 0; at < columns.length; at++) {
+        if (!numeric(at)) {
+            return at;
+        }
+    }
+    return 0;
+}
+const cellsOf = (row) => row.children.map((cell) => collapse(inline(cell.children)));
+// ---------------------------------------------------------------------------
+// plain text
+// ---------------------------------------------------------------------------
+/**
+ * A `.txt` file is read as paragraphs and nothing else. It would be easy to run
+ * it through the markdown parser and get headings for free, but they would be
+ * invented: a line beginning with `#` in a log or a licence is not a title, and
+ * an index that says it is would scope searches to sections nobody wrote.
+ */
+function plain(text, lines, name) {
+    const root = {
+        id: ROOT_SEGMENT,
+        path: ROOT_SEGMENT,
+        title: stripExtension(name),
+        level: 0,
+        line: undefined,
+        parent: undefined,
+    };
+    const blocks = [];
+    let at = 0;
+    let ordinal = 0;
+    while (at < lines.length) {
+        if ((lines[at] ?? '').trim() === '') {
+            at++;
+            continue;
+        }
+        const start = at;
+        while (at < lines.length && (lines[at] ?? '').trim() !== '') {
+            at++;
+        }
+        const own = `${SEGMENT.paragraph}:${++ordinal}`;
+        blocks.push({
+            id: own,
+            path: `${root.path}/${own}`,
+            kind: 'paragraph',
+            start: start + 1,
+            end: at,
+            section: root,
+            text: collapse(lines.slice(start, at).join(' ')),
+        });
+    }
+    const first = lines.find((line) => line.trim() !== '')?.trim() ?? '';
+    return {
+        name,
+        title: first.length > 0 && first.length <= 80 ? first : stripExtension(basenameOf(name)),
+        format: 'text',
+        lines,
+        sections: [root],
+        blocks,
+    };
+}
+// ---------------------------------------------------------------------------
+// serialization — walking the inline tree, never a regex over the markup
+// ---------------------------------------------------------------------------
+function inline(nodes) {
+    return nodes.map(one).join('');
+}
+function one(node) {
+    switch (node.type) {
+        case 'text':
+            return node.value;
+        // The value, not the markup: this is the node a regex stripper eats.
+        case 'inlineCode':
+            return node.value;
+        case 'image':
+        case 'imageReference':
+            // Distinguishable from a sentence on purpose.
+            return node.alt ? `image: ${node.alt}` : '';
+        case 'break':
+            return ' ';
+        // Inline html carries no words worth indexing, and its angle brackets
+        // would only ever match a query by accident.
+        case 'html':
+            return '';
+        case 'footnoteReference':
+            return '';
+        default:
+            return 'children' in node ? inline(node.children) : '';
+    }
+}
+/** Nested block content — a blockquote's paragraphs, a list item's children. */
+function blocks(nodes) {
+    return nodes
+        .map((node) => {
+        switch (node.type) {
+            case 'paragraph':
+            case 'heading':
+                return inline(node.children);
+            case 'code':
+                return node.value;
+            case 'blockquote':
+                return blocks(node.children);
+            case 'list':
+                return node.children
+                    .map((item) => blocks(item.children))
+                    .join(' ');
+            case 'table':
+                return node.children
+                    .slice(1)
+                    .map((row) => rowText(cellsOf(node.children[0]), cellsOf(row)))
+                    .join(' ');
+            default:
+                return '';
+        }
+    })
+        .filter(Boolean)
+        .join(' ');
+}
+/** Soft wraps inside a paragraph are wrapping, not meaning, so they rejoin. */
+export const collapse = (text) => text.replace(/\s+/g, ' ').trim();
+const tags = (html) => html.replace(/<[^>]*>/g, ' ');
+// ---------------------------------------------------------------------------
+function id(walk, kind) {
+    const segment = SEGMENT[kind];
+    const next = (walk.counters.get(segment) ?? 0) + 1;
+    walk.counters.set(segment, next);
+    return `${segment}:${next}`;
+}
+const lineOf = (node) => node.position?.start.line ?? 1;
+const endLineOf = (node) => node.position?.end.line ?? 1;
+const basenameOf = (name) => name.split('/').pop() ?? name;
+const stripExtension = (name) => name.replace(/\.[^./]+$/, '');
+//# sourceMappingURL=parse.js.map

package/dist/docs/pool.d.ts

@@ -0,0 +1,27 @@
+import { type Chunk, type ChunkOptions } from './chunk.ts';
+import type { FileOutline } from './files.ts';
+export interface ParseInput {
+    name: string;
+    text: string;
+    format: 'markdown' | 'text';
+}
+export interface Parsed {
+    chunks: Chunk[];
+    outline: FileOutline;
+}
+/** One short of the machine, so the thread doing everything else keeps a core. */
+export declare const poolSize: () => number;
+export interface ParseAllOptions {
+    chunk?: ChunkOptions;
+    /** how many threads to allow; 1 keeps everything here */
+    workers?: number;
+    onProgress?: (done: number, total: number, pending: readonly string[]) => void;
+}
+export declare function parseAll(inputs: readonly ParseInput[], options?: ParseAllOptions): Promise<Parsed[]>;
+/**
+ * The threaded path on its own. `parseAll` falls back to this thread when the
+ * workers cannot run, which is right for a build and useless for a test: a
+ * worker that never starts would look exactly like a fast one. Tests call this.
+ */
+export declare function parseThreaded(inputs: readonly ParseInput[], chunk: ChunkOptions, size: number, onProgress?: (done: number, total: number, pending: readonly string[]) => void): Promise<Parsed[]>;
+//# sourceMappingURL=pool.d.ts.map