@zenera/rag 1.1.9 → 1.1.11

package/dist/common/progress.js

@@ -46,8 +46,18 @@ export function beginBuild(plan) {
     let summary;
     let done = 0;
     let total = 0;
+    let pending = [];
     let wroteAt = 0;
     let closed = false;
+    const timings = [];
+    let phaseAt = started;
+    /** Closes the running phase off. Called on every transition, and at the end. */
+    const mark = () => {
+        const at = Date.now();
+        timings.push({ name: phase, ms: at - phaseAt });
+        phaseAt = at;
+    };
+    const sofar = () => [...timings, { name: phase, ms: Date.now() - phaseAt }];
     const write = (body) => {
         // Through a temp name, so a reader never catches half a file.
         const target = join(plan.dir, README_FILE);
@@ -62,8 +72,11 @@ export function beginBuild(plan) {
         started,
         now: Date.now(),
         step: plan.phases[phase],
+        phase,
         done,
         total,
+        pending,
+        timings: sofar(),
         summary,
     }));
     const maybe = () => {
@@ -87,7 +100,9 @@ export function beginBuild(plan) {
     report();
     return {
         phase(name) {
+            mark();
             phase = name;
+            pending = [];
             maybe();
         },
         read(seen, count) {
@@ -95,16 +110,33 @@ export function beginBuild(plan) {
             total = count;
             maybe();
         },
-        progress(at, of) {
+        progress(at, of, waiting) {
             done = at;
             total = of;
+            pending = waiting ?? [];
             maybe();
         },
         finish(manifest) {
-            close(plan.report.complete({ dir: plan.dir, manifest, ms: Date.now() - started }));
+            mark();
+            close(plan.report.complete({
+                dir: plan.dir,
+                manifest,
+                ms: Date.now() - started,
+                timings,
+            }));
         },
         fail(reason) {
-            close(plan.report.failed({ documents, step: plan.phases[phase], reason, started }));
+            mark();
+            close(plan.report.failed({
+                documents,
+                step: plan.phases[phase],
+                reason,
+                started,
+                timings,
+            }));
+        },
+        get timings() {
+            return closed ? timings : sofar();
         },
     };
 }

package/dist/common/prose.d.ts

@@ -4,6 +4,13 @@ export declare function fields(rows: readonly string[][]): string[];
 export declare function grid(headers: readonly string[], rows: readonly (readonly string[])[]): string[];
 export declare function plural(n: number, one: string, many?: string): string;
 export declare function duration(ms: number): string;
+/** Sub-second, because a phase breakdown is read to compare phases against each other. */
+export declare function span(ms: number): string;
+/** Where the time went, which is the only way to know what is worth making faster. */
+export declare const breakdown: (timings: readonly {
+    name: string;
+    ms: number;
+}[]) => string;
 /** The first line only: a stack trace in a README helps nobody. */
 export declare function message(reason: unknown): string;
 export declare function searched(indexes: {

package/dist/common/prose.js

@@ -36,6 +36,19 @@ export function duration(ms) {
         ? `${minutes}m${seconds % 60}s`
         : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
 }
+/** Sub-second, because a phase breakdown is read to compare phases against each other. */
+export function span(ms) {
+    if (ms < 1000) {
+        return `${Math.round(ms)}ms`;
+    }
+    if (ms < 60_000) {
+        return `${(ms / 1000).toFixed(1)}s`;
+    }
+    const seconds = Math.round(ms / 1000);
+    return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, '0')}s`;
+}
+/** Where the time went, which is the only way to know what is worth making faster. */
+export const breakdown = (timings) => timings.map((t) => `${t.name} ${span(t.ms)}`).join(' · ');
 /** The first line only: a stack trace in a README helps nobody. */
 export function message(reason) {
     const text = reason instanceof Error ? reason.message : String(reason);

package/dist/docs/assemble.d.ts

@@ -0,0 +1,52 @@
+import type { DocsIndex, Match } from './search.ts';
+export interface AssembleOptions {
+    /** extra lines quoted before each matching body */
+    before?: number;
+    after?: number;
+    /** a ceiling on the whole answer, so one long section cannot eat it */
+    maxLines?: number;
+    /** two ranges closer than this are joined rather than marked */
+    mergeGap?: number;
+}
+export interface Segment {
+    start: number;
+    end: number;
+    /** the lines themselves, verbatim */
+    lines: string[];
+}
+export interface Omission {
+    start: number;
+    end: number;
+    count: number;
+    /** the headings the skipped lines covered, so the gap has a name */
+    sections: string[];
+}
+export type Piece = ({
+    type: 'segment';
+} & Segment) | ({
+    type: 'omission';
+} & Omission);
+export interface Excerpt {
+    path: string;
+    title: string;
+    score: number;
+    /** what landed in this document, best first */
+    matches: Match[];
+    pieces: Piece[];
+    /** lines actually quoted */
+    shown: number;
+    /** the document's length, so a reader knows what fraction this is */
+    lines: number;
+}
+export interface Assembly {
+    files: Excerpt[];
+    shown: number;
+    /** true when the line budget cut something that had matched */
+    truncated: boolean;
+}
+export declare const DEFAULT_BEFORE = 0;
+export declare const DEFAULT_AFTER = 0;
+export declare const DEFAULT_MAX_LINES = 400;
+export declare const DEFAULT_MERGE_GAP = 3;
+export declare function assemble(index: DocsIndex, matches: readonly Match[], options?: AssembleOptions): Promise<Assembly>;
+//# sourceMappingURL=assemble.d.ts.map

package/dist/docs/assemble.js

@@ -0,0 +1,127 @@
+export const DEFAULT_BEFORE = 0;
+export const DEFAULT_AFTER = 0;
+export const DEFAULT_MAX_LINES = 400;
+export const DEFAULT_MERGE_GAP = 3;
+export async function assemble(index, matches, options = {}) {
+    const before = options.before ?? DEFAULT_BEFORE;
+    const after = options.after ?? DEFAULT_AFTER;
+    const gap = options.mergeGap ?? DEFAULT_MERGE_GAP;
+    let budget = options.maxLines ?? DEFAULT_MAX_LINES;
+    const files = [];
+    let truncated = false;
+    let shown = 0;
+    for (const [path, group] of byFile(matches)) {
+        const outline = index.file(path);
+        const source = await index.lines(path);
+        const total = outline?.lines ?? source.length;
+        const wanted = ranges(lineSet(group, before, after, total), gap);
+        const pieces = [];
+        let quoted = 0;
+        let last;
+        for (const range of wanted) {
+            if (budget <= 0) {
+                truncated = true;
+                break;
+            }
+            const end = Math.min(range.end, range.start + budget - 1);
+            if (last) {
+                pieces.push(omission(last.end + 1, range.start - 1, outline?.headings ?? []));
+            }
+            pieces.push({
+                type: 'segment',
+                start: range.start,
+                end,
+                lines: source.slice(range.start - 1, end),
+            });
+            const count = end - range.start + 1;
+            quoted += count;
+            budget -= count;
+            if (end < range.end) {
+                truncated = true;
+            }
+            last = { start: range.start, end };
+        }
+        if (last && last.end < total) {
+            pieces.push(omission(last.end + 1, total, outline?.headings ?? []));
+        }
+        if (pieces.length === 0) {
+            continue;
+        }
+        shown += quoted;
+        files.push({
+            path,
+            title: outline?.title ?? path,
+            score: group[0].score,
+            matches: group,
+            pieces,
+            shown: quoted,
+            lines: total,
+        });
+    }
+    return { files, shown, truncated };
+}
+// ---------------------------------------------------------------------------
+/** Documents in the order their best match placed, matches within them likewise. */
+function byFile(matches) {
+    const groups = new Map();
+    for (const item of matches) {
+        const group = groups.get(item.path);
+        if (group) {
+            group.push(item);
+        }
+        else {
+            groups.set(item.path, [item]);
+        }
+    }
+    return [...groups.entries()];
+}
+/**
+ * Every line the answer wants. A chunk carries its own render set — its
+ * headings, a table's header row, the line its body started on — and `before`
+ * and `after` widen the body only, so context is padding around the match and
+ * never around a heading quoted from elsewhere in the file.
+ */
+function lineSet(matches, before, after, total) {
+    const wanted = new Set();
+    const add = (line) => {
+        if (line >= 1 && line <= total) {
+            wanted.add(line);
+        }
+    };
+    for (const item of matches) {
+        for (const line of item.lineNumbers) {
+            add(line);
+        }
+        for (let line = item.bodyStart - before; line <= item.bodyEnd + after; line++) {
+            add(line);
+        }
+    }
+    return wanted;
+}
+/** Contiguous runs, with runs closer together than `gap` joined into one. */
+function ranges(wanted, gap) {
+    const sorted = [...wanted].sort((a, b) => a - b);
+    const out = [];
+    for (const line of sorted) {
+        const last = out.at(-1);
+        if (last && line - last.end <= gap + 1) {
+            last.end = line;
+        }
+        else {
+            out.push({ start: line, end: line });
+        }
+    }
+    return out;
+}
+function omission(start, end, headings) {
+    const named = headings.filter((h) => h.line >= start && h.line <= end).map((h) => h.title);
+    return {
+        type: 'omission',
+        start,
+        end,
+        count: end - start + 1,
+        // A gap covering thirty headings is a gap; naming all of them is noise.
+        sections: named.slice(0, 6),
+    };
+}
+//# sourceMappingURL=assemble.js.map

package/dist/docs/build.d.ts

@@ -0,0 +1,54 @@
+import type { Embedder } from '@zenera/neo';
+import { type PhaseTiming } from '../common/progress.ts';
+import { type ChunkOptions } from './chunk.ts';
+import { type Counts, type DocRecord, type Manifest } from './files.ts';
+import { type Corpus } from './load.ts';
+import { type ChunkRecord } from './store.ts';
+export interface BuildOptions {
+    /** files, directories or patterns, as they were named */
+    files: readonly string[];
+    cwd: string;
+    out: string;
+    embedder: Embedder;
+    /** the reference as it was written, which is what a later search will type */
+    embeddingRef?: string;
+    /** told the manifest, so a store can say what wrote it */
+    indexer: string;
+    chunk?: ChunkOptions;
+    /**
+     * The width asked of the embedder, when one was asked for. Part of the cache
+     * key, because a truncated vector is a different vector; left undefined when
+     * nobody asked, because that is a different key again from asking for the
+     * number the model would have chosen anyway.
+     */
+    dimensions?: number;
+    /** reuse vectors and parses this machine already has; on by default */
+    cache?: boolean;
+    /** keep them somewhere other than the shared store */
+    cacheDir?: string;
+    signal?: AbortSignal;
+    /** what the documents turned out to hold, before a vector has been paid for */
+    onRead?: (summary: BuildSummary) => void;
+    /** documents parsed so far, and what the pool is still on */
+    onReading?: (done: number, total: number, pending: readonly string[]) => void;
+    onProgress?: (done: number, total: number) => void;
+}
+export interface BuildSummary {
+    sources: DocRecord[];
+    counts: Counts;
+    skipped: Corpus['skipped'];
+}
+export interface BuildResult {
+    manifest: Manifest;
+    chunks: ChunkRecord[];
+    /** what each phase cost, so a slow build can say which part was slow */
+    timings: readonly PhaseTiming[];
+    /** what came out of the shared cache instead of being done again */
+    reused: Reused;
+}
+export interface Reused {
+    parses: number;
+    vectors: number;
+}
+export declare function buildIndex(options: BuildOptions): Promise<BuildResult>;
+//# sourceMappingURL=build.d.ts.map

package/dist/docs/build.js

@@ -0,0 +1,134 @@
+import { embedStream, NO_CACHE, openCache } from "../common/cache.js";
+import { beginBuild } from "../common/progress.js";
+import { formatLines } from "./chunk.js";
+import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
+import { loadDocuments } from "./load.js";
+import { DOCS_REPORT, PHASES } from "./readme.js";
+import { openChunks } from "./store.js";
+export async function buildIndex(options) {
+    const journal = beginBuild({
+        dir: options.out,
+        files: options.files,
+        embedding: options.embeddingRef ?? options.embedder.id,
+        indexer: options.indexer,
+        phases: PHASES,
+        report: DOCS_REPORT,
+    });
+    const ref = options.embeddingRef ?? options.embedder.id;
+    const cache = options.cache === false
+        ? NO_CACHE
+        : openCache(options.embedder, {
+            ref,
+            dir: options.cacheDir,
+            dimensions: options.dimensions,
+        });
+    let writer;
+    try {
+        const corpus = await loadDocuments(options.files, options.cwd, {
+            chunk: options.chunk,
+            cache: options.cache !== false,
+            cacheDir: options.cacheDir,
+            onProgress: (done, total, pending) => {
+                journal.progress(done, total, pending);
+                options.onReading?.(done, total, pending);
+            },
+        });
+        const chunks = recordsOf(corpus);
+        const sources = corpus.docs.map((doc) => ({
+            name: doc.name,
+            file: doc.file,
+            path: `${SOURCES_DIR}/${doc.name}`,
+            sha256: doc.sha256,
+            format: doc.format,
+            title: doc.outline.title,
+            bytes: doc.bytes,
+            lines: doc.outline.lines,
+            sections: doc.outline.headings.length,
+            tables: doc.outline.tables.length,
+            chunks: doc.chunks.length,
+        }));
+        const counts = {
+            documents: sources.length,
+            chunks: chunks.length,
+            lines: sources.reduce((n, s) => n + s.lines, 0),
+            sections: sources.reduce((n, s) => n + s.sections, 0),
+            tables: sources.reduce((n, s) => n + s.tables, 0),
+        };
+        journal.read(counts, chunks.length);
+        options.onRead?.({ sources, counts, skipped: corpus.skipped });
+        journal.phase('embedding');
+        writer = await openChunks(options.out);
+        const dimensions = await embedAll(chunks, options, journal, cache, writer);
+        journal.phase('writing');
+        const written = await writer.finish();
+        const manifest = {
+            version: INDEX_VERSION,
+            kind: 'docs',
+            createdAt: new Date().toISOString(),
+            indexer: options.indexer,
+            embedding: {
+                ref,
+                id: options.embedder.id,
+                dimensions,
+                requested: options.dimensions,
+            },
+            sources,
+            counts,
+            indexes: { fts: written.fts, vector: written.vector },
+        };
+        const outline = { files: corpus.docs.map((doc) => doc.outline) };
+        const documents = Object.fromEntries(corpus.docs.map((doc) => [doc.name, doc.text]));
+        await writeIndex(options.out, { manifest, outline, documents });
+        cache.commit();
+        journal.finish(manifest);
+        return {
+            manifest,
+            chunks,
+            timings: journal.timings,
+            reused: { parses: corpus.cached, vectors: cache.hits },
+        };
+    }
+    catch (err) {
+        writer?.close();
+        cache.abandon();
+        journal.fail(err);
+        throw err;
+    }
+}
+/** One row per chunk, with the render set encoded and the document name on it. */
+function recordsOf(corpus) {
+    return corpus.docs.flatMap((doc) => doc.chunks.map((chunk) => ({
+        id: `${doc.name}#c${chunk.index}`,
+        path: doc.name,
+        ordinal: chunk.index,
+        kind: chunk.kind,
+        text: chunk.text,
+        embedText: chunk.embedText,
+        lineSpec: formatLines(chunk.lineNumbers),
+        bodyStart: chunk.bodyStart,
+        bodyEnd: chunk.bodyEnd,
+        structureId: chunk.structureId,
+        structurePath: chunk.structurePath,
+        headings: chunk.headings,
+        tokens: chunk.tokens,
+    })));
+}
+async function embedAll(chunks, options, journal, cache, writer) {
+    // A window at a time, rather than the whole corpus in one call. How many
+    // texts fit in a request, and how many requests may be in flight, are still
+    // the embedder's to answer — it knows the model's caps and it is the one
+    // that sees the 429s. What the window decides is only how much is resident.
+    return embedStream({
+        embedder: options.embedder,
+        cache,
+        records: chunks,
+        textOf: (chunk) => chunk.embedText,
+        signal: options.signal,
+        onProgress: (done, total) => {
+            journal.progress(done, total);
+            options.onProgress?.(done, total);
+        },
+        onWindow: (window, vectors) => writer.add(window, vectors),
+    });
+}
+//# sourceMappingURL=build.js.map

package/dist/docs/chunk.d.ts

@@ -0,0 +1,73 @@
+import { type ParsedDoc } from './parse.ts';
+/** Soft target, hard ceiling, and the width at which one table row is too wide. */
+export declare const CHUNK_TOKENS = 384;
+export declare const MAX_CHUNK_TOKENS = 512;
+export declare const TABLE_SLICE_TOKENS = 128;
+/** How much of a table its descriptor stands for when no row of it matched. */
+export declare const TABLE_PREVIEW_ROWS = 3;
+/**
+ * And how many rows may share one. The token budget alone would put a narrow
+ * sixteen-row table in a single chunk, so matching one row of it quotes all
+ * sixteen — the rows are independent facts, and a reader asking about one is
+ * not asking about the other fifteen.
+ */
+export declare const TABLE_ROWS_PER_CHUNK = 4;
+/**
+ * Below this a chunk is merged into its neighbour rather than retrieved alone.
+ *
+ * BM25 normalises by document length, so a nine-word chunk that happens to
+ * contain two query terms outscores a real answer that contains them among a
+ * hundred other words. Measured on this repository, a one-line aside about
+ * markdown link syntax took full-text rank 0 for "how to create docs index"
+ * while the vector leg — correctly — put it 185th. It is not that the chunk is
+ * wrong; it is that alone it is not a passage, and a passage is what the
+ * lexical index is scoring.
+ */
+export declare const MIN_CHUNK_TOKENS = 48;
+/**
+ * How much of the block before a chunk may carry for continuity. It is one
+ * line, which is a whisper until the line is the whole of a README's challenge
+ * table on one row: 92kB of badge markup, none of it cited by a line number,
+ * outweighing the seven lines the chunk actually stands for by eighty to one.
+ */
+export declare const CARRY_TOKENS = 32;
+/**
+ * Four characters to a token, which is within about 15% for English prose and
+ * wrong for CJK, for dense numeric cells and for long identifiers. It cannot
+ * cause a request to fail — 512 estimated tokens sits far below any embedding
+ * model's limit, so even a threefold underestimate has headroom. Injectable so
+ * a real tokenizer is a one-line swap if the corpus ever needs one.
+ */
+export declare const tokenCount: (text: string) => number;
+/** The kinds a chunk can be, which is what `--kind` filters on. */
+export declare const CHUNK_KINDS: readonly ["paragraph", "list", "table", "table_row", "code", "frontmatter", "html"];
+export type ChunkKind = (typeof CHUNK_KINDS)[number];
+export interface ChunkOptions {
+    chunkTokens?: number;
+    minChunkTokens?: number;
+    maxChunkTokens?: number;
+    tableSliceTokens?: number;
+    tokenCount?: (text: string) => number;
+}
+export interface Chunk {
+    index: number;
+    kind: ChunkKind;
+    /** the innermost structure node containing the whole body */
+    structureId: string;
+    structurePath: string;
+    /** the breadcrumb, from the document name down to the nearest heading */
+    headings: string;
+    bodyStart: number;
+    bodyEnd: number;
+    /** everything that gets rendered: headings, prelude and body, sorted */
+    lineNumbers: number[];
+    /** the full-text document — wider */
+    text: string;
+    /** the vector's source — tighter */
+    embedText: string;
+    tokens: number;
+}
+export declare function chunkDocument(doc: ParsedDoc, options?: ChunkOptions): Chunk[];
+export declare function formatLines(numbers: readonly number[]): string;
+export declare function parseLines(spec: string): number[];
+//# sourceMappingURL=chunk.d.ts.map