@zenera/rag 1.1.9 → 1.1.11

package/dist/docs/pool.js

@@ -0,0 +1,133 @@
+import { availableParallelism } from 'node:os';
+import { Worker } from 'node:worker_threads';
+import { chunkDocument } from "./chunk.js";
+import { outlineOf } from "./outline.js";
+import { parseDocument } from "./parse.js";
+// ---------------------------------------------------------------------------
+// Parsing several documents at once
+//
+// Remark is synchronous and holds the thread for as long as a document takes,
+// so a corpus is parsed strictly one file at a time no matter how many cores
+// are idle. A pool fixes that and nothing else: the work is pure, the inputs are
+// independent, and the answers are placed by index, so the result is the same
+// list in the same order that one thread would have produced.
+//
+// It is not always the faster choice. Starting a thread costs more than parsing
+// a small document, so a handful of files are done here; and a caller that
+// supplied its own `tokenCount` cannot be helped at all, because a function is
+// not something `postMessage` can clone. Every one of those paths runs the same
+// three functions in this thread instead, which is why the fallback is safe
+// rather than merely convenient.
+// ---------------------------------------------------------------------------
+/** Below this, starting threads costs more than the parsing they would save. */
+const MIN_DOCUMENTS = 8;
+/** More than this and the threads contend for memory bandwidth, not for work. */
+const MAX_WORKERS = 8;
+/** One short of the machine, so the thread doing everything else keeps a core. */
+export const poolSize = () => Math.max(1, Math.min(MAX_WORKERS, availableParallelism() - 1));
+const EXTENSION = import.meta.url.endsWith('.ts') ? '.ts' : '.js';
+// Built from `import.meta.url` because the extension changes on the way to
+// `dist/`, and `rewriteRelativeImportExtensions` does not touch string literals.
+const WORKER = new URL(`./parse-worker${EXTENSION}`, import.meta.url);
+/** How many documents run between yields, so a long parse still narrates. */
+const YIELD_EVERY = 32;
+export async function parseAll(inputs, options = {}) {
+    const chunk = options.chunk ?? {};
+    const workers = Math.min(options.workers ?? poolSize(), inputs.length);
+    const out = new Array(inputs.length);
+    if (workers > 1 && inputs.length >= MIN_DOCUMENTS && typeof chunk.tokenCount !== 'function') {
+        try {
+            await threaded(out, inputs, chunk, workers, options.onProgress);
+        }
+        catch {
+            // A thread that would not start, or died. Whatever it did finish is
+            // in `out` already and is not parsed again; `here` fills the rest.
+        }
+    }
+    return here(out, inputs, chunk, options.onProgress);
+}
+async function here(out, inputs, chunk, onProgress) {
+    let done = out.reduce((n, parsed) => (parsed ? n + 1 : n), 0);
+    for (const [at, input] of inputs.entries()) {
+        if (out[at]) {
+            continue;
+        }
+        const parsed = parseDocument(input.text, input.name, input.format);
+        const chunks = chunkDocument(parsed, chunk);
+        out[at] = { chunks, outline: outlineOf(parsed, chunks.length) };
+        onProgress?.(++done, inputs.length, [input.name]);
+        if (done % YIELD_EVERY === 0) {
+            // Remark is synchronous, so a corpus parsed here would hold the loop
+            // for minutes and freeze the very report saying it is still working.
+            await new Promise((resume) => setImmediate(resume));
+        }
+    }
+    return out;
+}
+/**
+ * The threaded path on its own. `parseAll` falls back to this thread when the
+ * workers cannot run, which is right for a build and useless for a test: a
+ * worker that never starts would look exactly like a fast one. Tests call this.
+ */
+export async function parseThreaded(inputs, chunk, size, onProgress) {
+    const out = new Array(inputs.length);
+    await threaded(out, inputs, chunk, size, onProgress);
+    return out;
+}
+async function threaded(out, inputs, chunk, size, onProgress) {
+    const workers = Array.from({ length: size }, () => new Worker(WORKER, { workerData: chunk }));
+    // What each thread is on, so a corpus with two pathological documents in it
+    // can name them instead of looking stalled a hair short of the total.
+    const busy = new Map();
+    let next = 0;
+    let done = 0;
+    const pump = async (worker) => {
+        while (next < inputs.length) {
+            const at = next++;
+            busy.set(worker, inputs[at].name);
+            out[at] = await ask(worker, { at, ...inputs[at] });
+            busy.delete(worker);
+            onProgress?.(++done, inputs.length, [...busy.values()]);
+        }
+    };
+    try {
+        await Promise.all(workers.map(pump));
+    }
+    finally {
+        await Promise.all(workers.map((worker) => worker.terminate()));
+    }
+}
+function ask(worker, job) {
+    return new Promise((resolve, reject) => {
+        const done = () => {
+            worker.off('message', onMessage);
+            worker.off('error', onError);
+            worker.off('exit', onExit);
+        };
+        const onMessage = (reply) => {
+            if (reply.at !== job.at) {
+                return;
+            }
+            done();
+            if (reply.error !== undefined) {
+                reject(new Error(`${job.name}: ${reply.error}`));
+            }
+            else {
+                resolve({ chunks: reply.chunks, outline: reply.outline });
+            }
+        };
+        const onError = (err) => {
+            done();
+            reject(err);
+        };
+        const onExit = (code) => {
+            done();
+            reject(new Error(`parse worker stopped with code ${code}`));
+        };
+        worker.on('message', onMessage);
+        worker.on('error', onError);
+        worker.on('exit', onExit);
+        worker.postMessage(job);
+    });
+}
+//# sourceMappingURL=pool.js.map

package/dist/docs/readme.d.ts

@@ -0,0 +1,6 @@
+import { type Report } from '../common/progress.ts';
+import type { Counts, Manifest } from './files.ts';
+export type Phase = 'reading' | 'embedding' | 'writing';
+export declare const PHASES: Record<Phase, string>;
+export declare const DOCS_REPORT: Report<Counts, Manifest>;
+//# sourceMappingURL=readme.d.ts.map

package/dist/docs/readme.js

@@ -0,0 +1,142 @@
+import { basename } from 'node:path';
+import { INTERVAL_MS, } from "../common/progress.js";
+import { breakdown, duration, fields, grid, message, plural, searched } from "../common/prose.js";
+export const PHASES = {
+    reading: 'reading the documents and cutting them into chunks',
+    embedding: 'embedding',
+    writing: 'writing the store',
+};
+/** What the running count is counting, which is not the same in every phase. */
+const COUNTING = {
+    reading: 'parsed',
+    embedding: 'embedded',
+};
+export const DOCS_REPORT = { building, complete, failed };
+function building(state) {
+    const rows = [
+        ['documents', summary(state.documents)],
+        ['embedding', state.embedding],
+        [
+            'started',
+            `${new Date(state.started).toISOString()} (${duration(state.now - state.started)} ago)`,
+        ],
+        ['step', state.step],
+    ];
+    if (state.summary) {
+        rows.push(['found', counted(state.summary)]);
+    }
+    if (state.total > 0) {
+        // Rounded down: 13619 of 13621 is not 100%, and a report that says it is
+        // turns two slow documents into a build that looks hung.
+        const percent = Math.floor((state.done / state.total) * 100);
+        rows.push([
+            COUNTING[state.phase] ?? 'done',
+            `${state.done} of ${state.total} · ${percent}%`,
+        ]);
+    }
+    if (state.pending.length > 0) {
+        rows.push(['still on', waiting(state.pending)]);
+    }
+    rows.push(['timing', breakdown(state.timings)]);
+    rows.push(['updated', new Date(state.now).toISOString()]);
+    return [
+        '# Document index — being built',
+        '',
+        'A searchable index of the documents named below, written by `zen rag docs index`.',
+        '**It is incomplete. Nothing should read it yet.**',
+        '',
+        ...fields(rows),
+        '',
+        `These lines are refreshed at most every ${INTERVAL_MS / 1000} seconds while the build runs, and`,
+        'the whole file is replaced by a description of the index when it finishes. If it still says',
+        '"being built" and `.lock` names no living process, the build died part way.',
+        '',
+    ].join('\n');
+}
+function complete(state) {
+    const { manifest } = state;
+    return [
+        `# Document index — ${basename(state.dir)}`,
+        '',
+        `A searchable index of ${plural(manifest.sources.length, 'document')}, built with`,
+        `${manifest.embedding.ref} (${manifest.embedding.dimensions}d) in ${duration(state.ms)}.`,
+        'Ask it a question and it answers with the passages that matched, quoted verbatim from',
+        'the copies kept in `sources/` — line numbers included, and with a marker wherever',
+        'something between two of them was left out.',
+        '',
+        '## What it covers',
+        '',
+        ...documentTable(manifest.sources),
+        '',
+        `${counted(manifest.counts)},`,
+        `${searched(manifest.indexes)}.`,
+        '',
+        `Time: ${breakdown(state.timings)}.`,
+        '',
+        '## Files',
+        '',
+        ...fields([
+            ['manifest.json', 'what this index is and what built it — read this first'],
+            ['outline.json', 'every heading and table, with the lines they cover'],
+            ['sources/', 'the documents themselves, verbatim: where the quotes come from'],
+            ['lance/', 'the chunks: the search text, the vectors, the filter columns'],
+        ]),
+        '',
+        '## Asking it something',
+        '',
+        'From this directory:',
+        '',
+        '```',
+        'zen rag docs search --dir . "what you are after"',
+        'zen rag docs search --dir . --file "guides/**" --kind table "pressure rating"',
+        'zen rag docs list files --dir .',
+        '```',
+        '',
+        `Built by ${manifest.indexer} on ${manifest.createdAt}.`,
+        '',
+    ].join('\n');
+}
+function failed(state) {
+    return [
+        '# Document index — failed',
+        '',
+        'This index was not finished and what is here is incomplete. Nothing should read it;',
+        'build it again with `zen rag docs index`.',
+        '',
+        ...fields([
+            ['documents', summary(state.documents)],
+            ['step', state.step],
+            ['reason', message(state.reason)],
+            ['started', new Date(state.started).toISOString()],
+            ['timing', breakdown(state.timings)],
+            [
+                'failed',
+                `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,
+            ],
+        ]),
+        '',
+    ].join('\n');
+}
+// ---------------------------------------------------------------------------
+/** Names are only worth reading once there are few enough to act on. */
+const MOST_NAMES = 3;
+const waiting = (pending) => pending.length <= MOST_NAMES ? pending.join(', ') : `${pending.length} documents`;
+const HEADERS = ['document', 'format', 'lines', 'sections', 'tables', 'chunks'];
+const documentTable = (sources) => grid(HEADERS, sources.map((s) => [
+    s.name,
+    s.format,
+    String(s.lines),
+    String(s.sections),
+    String(s.tables),
+    String(s.chunks),
+]));
+function counted(counts) {
+    return (`${plural(counts.chunks, 'chunk')} over ${plural(counts.documents, 'document')}: ` +
+        `${counts.lines} lines, ${counts.sections} sections, ${counts.tables} tables`);
+}
+/** A corpus can be thousands of files; a README listing them all is a wall. */
+function summary(documents) {
+    const shown = documents.slice(0, 12).join(', ');
+    return documents.length > 12 ? `${shown}, and ${documents.length - 12} more` : shown;
+}
+//# sourceMappingURL=readme.js.map

package/dist/docs/render.d.ts

@@ -0,0 +1,13 @@
+import type { Assembly, Excerpt } from './assemble.ts';
+import type { Match } from './search.ts';
+export interface RenderOptions {
+    /** the line-number gutter; on unless something else is going to eat this */
+    numbers?: boolean;
+    colour?: boolean;
+}
+export declare function renderAssembly(assembly: Assembly, options?: RenderOptions): string;
+export declare function renderExcerpt(file: Excerpt, options?: RenderOptions): string[];
+/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
+export declare const matchRows: (matches: readonly Match[]) => string[][];
+export declare const MATCH_HEADERS: readonly ["id", "kind", "lines", "score", "vec / txt", "heading"];
+//# sourceMappingURL=render.d.ts.map

package/dist/docs/render.js

@@ -0,0 +1,46 @@
+import { bold, dim } from '@zenera/cli/lib';
+export function renderAssembly(assembly, options = {}) {
+    const out = [];
+    for (const file of assembly.files) {
+        out.push(...renderExcerpt(file, options), '');
+    }
+    if (assembly.truncated) {
+        out.push('(cut short by the line budget — raise it with --max-lines)');
+    }
+    return out.join('\n').trimEnd();
+}
+export function renderExcerpt(file, options = {}) {
+    const paint = options.colour === false ? (s) => s : undefined;
+    const strong = paint ?? bold;
+    const faint = paint ?? dim;
+    const width = String(file.lines).length;
+    return [
+        `## ${strong(file.path)} ${faint(`— ${file.shown} of ${file.lines} lines`)}`,
+        '',
+        ...file.pieces.flatMap((piece) => renderPiece(piece, width, options, faint)),
+    ];
+}
+function renderPiece(piece, width, options, faint) {
+    if (piece.type === 'omission') {
+        const named = piece.sections.length > 0 ? ` (${piece.sections.join(', ')})` : '';
+        return [faint(`... ${piece.count} lines omitted${named} ...`), ''];
+    }
+    const lines = piece.lines.map((line, at) => options.numbers === false
+        ? line
+        : `${faint(String(piece.start + at).padStart(width))} ${faint('|')} ${line}`);
+    return [...lines, ''];
+}
+/** The one-line-per-match view, for `--quiet` and for the prompt loop. */
+export const matchRows = (matches) => matches.map((m) => [
+    m.id,
+    m.kind,
+    `${m.bodyStart}-${m.bodyEnd}`,
+    m.score.toFixed(4),
+    // Fusion ranks; these say whether the rank was worth anything.
+    [m.relevance.vector?.toFixed(2), m.relevance.text?.toFixed(1)]
+        .map((v) => v ?? '·')
+        .join(' / '),
+    m.headings.split('\n')[0] ?? '',
+]);
+export const MATCH_HEADERS = ['id', 'kind', 'lines', 'score', 'vec / txt', 'heading'];
+//# sourceMappingURL=render.js.map

package/dist/docs/repl.d.ts

@@ -0,0 +1,7 @@
+import { type AssembleOptions } from './assemble.ts';
+import { type DocsIndex, type DocsQuery } from './search.ts';
+export interface ReplSettings extends AssembleOptions {
+    quiet?: boolean;
+}
+export declare function repl(index: DocsIndex, initial: DocsQuery, settings?: ReplSettings): Promise<void>;
+//# sourceMappingURL=repl.d.ts.map

package/dist/docs/repl.js

@@ -0,0 +1,130 @@
+import { bold, cyan, dim, note, write } from '@zenera/cli/lib';
+import { createInterface } from 'node:readline/promises';
+import { assemble } from "./assemble.js";
+import { CHUNK_KINDS } from "./chunk.js";
+import { renderAssembly } from "./render.js";
+import { SEARCH_MODES } from "./search.js";
+export async function repl(index, initial, settings = {}) {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    let query = { ...initial };
+    note(bold(`${index.manifest.counts.documents} document(s), ${index.manifest.counts.chunks} chunks`));
+    note(dim('  a bare line searches; `help` lists the rest.'));
+    try {
+        if (query.query) {
+            query = await run(index, query, settings);
+        }
+        for (;;) {
+            const line = (await rl.question(cyan('docs> '))).trim();
+            if (line === '') {
+                continue;
+            }
+            if (line === 'quit' || line === 'exit') {
+                return;
+            }
+            if (line === 'help') {
+                help();
+                continue;
+            }
+            if (line === 'reset') {
+                query = {};
+                note(dim('  cleared, narrowings and exclusions alike'));
+                continue;
+            }
+            if (line === 'show') {
+                note(dim(`  ${JSON.stringify(query)}`));
+                continue;
+            }
+            if (line === 'files') {
+                for (const name of index.resolveFiles(query.files, query.exclude_files)) {
+                    note(dim(`  ${name}`));
+                }
+                continue;
+            }
+            const [head, ...rest] = line.split(' ');
+            const tail = rest.join(' ').trim();
+            const narrowed = narrow(query, head ?? '', tail);
+            if (narrowed) {
+                query = narrowed;
+                continue;
+            }
+            query = await run(index, { ...query, query: line }, settings);
+        }
+    }
+    finally {
+        rl.close();
+    }
+}
+/**
+ * The narrowing verbs. Each one replaces its own field rather than adding to
+ * it — `file guides/**` twice in a row means the second pattern, which is what
+ * anyone typing it meant. An empty argument clears the field.
+ */
+function narrow(query, head, tail) {
+    const set = (key, value) => {
+        note(dim(`  ${head} is ${tail || 'anything'}`));
+        return { ...query, [key]: value };
+    };
+    switch (head) {
+        case 'file':
+            return set('files', tail ? [tail] : undefined);
+        case 'not':
+            return set('exclude_files', tail ? [tail] : undefined);
+        case 'section':
+            return set('section', tail ? [tail] : undefined);
+        case 'kind':
+            if (tail && !CHUNK_KINDS.includes(tail)) {
+                note(dim(`  kind is one of ${CHUNK_KINDS.join(', ')}`));
+                return query;
+            }
+            return set('kinds', tail ? [tail] : undefined);
+        case 'mode':
+            if (tail && !SEARCH_MODES.includes(tail)) {
+                note(dim(`  mode is one of ${SEARCH_MODES.join(', ')}`));
+                return query;
+            }
+            return set('mode', (tail || undefined));
+        case 'limit': {
+            const limit = Number(tail);
+            if (!Number.isInteger(limit) || limit < 1) {
+                note(dim('  limit is a whole number of at least 1'));
+                return query;
+            }
+            return set('limit', limit);
+        }
+        default:
+            return undefined;
+    }
+}
+async function run(index, query, settings) {
+    const result = await index.search(query);
+    const excerpt = await assemble(index, result.matches, settings);
+    if (excerpt.files.length > 0) {
+        write(renderAssembly(excerpt, {}));
+    }
+    note(dim(`  ${result.matches.length} passage(s) in ${excerpt.files.length} document(s)` +
+        ` · ${excerpt.shown} line(s) · ${result.files.length} document(s) in scope`));
+    if (result.matches.length === 0) {
+        return query;
+    }
+    const seen = new Set([...(query.exclude_ids ?? []), ...result.matches.map((m) => m.id)]);
+    note(dim(`  ${seen.size} passage(s) now excluded — \`reset\` to see them again`));
+    return { ...query, exclude_ids: [...seen] };
+}
+function help() {
+    for (const line of [
+        '  <text>            search',
+        '  file <pattern>    only documents matching it; blank for all',
+        '  not <pattern>     drop documents matching it',
+        '  section <name>    only under this heading, by title, id or path',
+        `  kind <k>          ${CHUNK_KINDS.join(' | ')}`,
+        `  mode <m>          ${SEARCH_MODES.join(' | ')}`,
+        '  limit <n>         passages per search',
+        '  files             the documents currently in scope',
+        '  show              the query as it stands',
+        '  reset             forget it, narrowings and exclusions alike',
+        '  quit',
+    ]) {
+        note(dim(line));
+    }
+}
+//# sourceMappingURL=repl.js.map

package/dist/docs/search.d.ts

@@ -0,0 +1,92 @@
+import type { Embedder } from '@zenera/neo';
+import { type FileOutline, type HeadingRecord, type Manifest, type OpenDocs, type Outline } from './files.ts';
+import { ChunkStore } from './store.ts';
+export type SearchMode = 'hybrid' | 'vector' | 'text';
+export declare const SEARCH_MODES: readonly SearchMode[];
+export interface DocsQuery {
+    query?: string;
+    mode?: SearchMode;
+    /** patterns over the document name: a glob, a substring, or a regex */
+    files?: readonly string[];
+    exclude_files?: readonly string[];
+    /** a heading title, a structure id, or a structure path */
+    section?: readonly string[];
+    /** one of `CHUNK_KINDS` */
+    kinds?: readonly string[];
+    /** chunk ids already seen, so asking again moves on */
+    exclude_ids?: readonly string[];
+    limit?: number;
+}
+export interface Match {
+    id: string;
+    path: string;
+    kind: string;
+    headings: string;
+    structureId: string;
+    structurePath: string;
+    /** every line this chunk wants shown, headings and table header included */
+    lineNumbers: number[];
+    bodyStart: number;
+    bodyEnd: number;
+    text: string;
+    score: number;
+    /** where it placed in each leg, when it placed at all */
+    ranks: {
+        vector?: number;
+        text?: number;
+    };
+    /**
+     * And how well it did there. Fusion scores a rank, so its numbers say only
+     * that something placed first — a question the corpus cannot answer at all
+     * still returns a confident-looking `1 / (RRF_K + 0)`. These are what tell
+     * a caller whether first place was worth having.
+     */
+    relevance: {
+        vector?: number;
+        text?: number;
+    };
+}
+export interface DocsResult {
+    matches: Match[];
+    mode: SearchMode;
+    /** the documents the narrowings left in play */
+    files: string[];
+    /** the sections they left in play, as structure paths */
+    sections: string[];
+    /** distinct chunks either leg returned, before fusion trimmed them */
+    considered: number;
+}
+export declare const DEFAULT_LIMIT = 8;
+export declare class DocsIndex {
+    #private;
+    readonly manifest: Manifest;
+    readonly outline: Outline;
+    constructor(index: OpenDocs, store: ChunkStore, embedder?: Embedder);
+    /**
+     * An embedder is optional because half of what this index is for needs no
+     * model at all: listing, grepping and reading are exact, and asking for a
+     * credential to do them would be asking for nothing.
+     */
+    static open(dir: string, embedder?: Embedder): Promise<DocsIndex>;
+    /** A document, split the way every line number in this index counts it. */
+    lines(name: string): Promise<string[]>;
+    file(name: string): FileOutline | undefined;
+    /** Document names matching the patterns, or all of them when there are none. */
+    resolveFiles(patterns?: readonly string[], exclude?: readonly string[]): string[];
+    /**
+     * Headings the terms name, over the documents still in play. A term is tried
+     * as a structure id, then a structure path, then as a pattern over the
+     * title — so `--section overview` and `--section sec:3` are the same
+     * argument, and neither has to be explained.
+     */
+    resolveSections(terms: readonly string[], files: readonly string[]): HeadingRecord[];
+    close(): void;
+    search(query: DocsQuery, signal?: AbortSignal): Promise<DocsResult>;
+}
+/**
+ * Inside one of these sections, or anywhere when none were asked for. The
+ * boundary is a path separator and not a prefix, because `doc/sec:1` is a
+ * prefix of `doc/sec:10` and they are different sections.
+ */
+export declare const under: (path: string, sections: readonly string[]) => boolean;
+//# sourceMappingURL=search.d.ts.map