@zenera/rag 1.1.10 → 1.1.11

package/dist/docs/build.js

@@ -1,10 +1,10 @@
+import { embedStream, NO_CACHE, openCache } from "../common/cache.js";
 import { beginBuild } from "../common/progress.js";
 import { formatLines } from "./chunk.js";
 import { INDEX_VERSION, SOURCES_DIR, writeIndex, } from "./files.js";
 import { loadDocuments } from "./load.js";
 import { DOCS_REPORT, PHASES } from "./readme.js";
-import { writeChunks } from "./store.js";
-const DEFAULT_BATCH = 96;
+import { openChunks } from "./store.js";
 export async function buildIndex(options) {
     const journal = beginBuild({
         dir: options.out,
@@ -14,8 +14,25 @@ export async function buildIndex(options) {
         phases: PHASES,
         report: DOCS_REPORT,
     });
+    const ref = options.embeddingRef ?? options.embedder.id;
+    const cache = options.cache === false
+        ? NO_CACHE
+        : openCache(options.embedder, {
+            ref,
+            dir: options.cacheDir,
+            dimensions: options.dimensions,
+        });
+    let writer;
     try {
-        const corpus = await loadDocuments(options.files, options.cwd, options.chunk);
+        const corpus = await loadDocuments(options.files, options.cwd, {
+            chunk: options.chunk,
+            cache: options.cache !== false,
+            cacheDir: options.cacheDir,
+            onProgress: (done, total, pending) => {
+                journal.progress(done, total, pending);
+                options.onReading?.(done, total, pending);
+            },
+        });
         const chunks = recordsOf(corpus);
         const sources = corpus.docs.map((doc) => ({
             name: doc.name,
@@ -40,18 +57,20 @@ export async function buildIndex(options) {
         journal.read(counts, chunks.length);
         options.onRead?.({ sources, counts, skipped: corpus.skipped });
         journal.phase('embedding');
-        const vectors = await embedAll(chunks, options, journal);
+        writer = await openChunks(options.out);
+        const dimensions = await embedAll(chunks, options, journal, cache, writer);
         journal.phase('writing');
-        const written = await writeChunks(options.out, chunks, vectors);
+        const written = await writer.finish();
         const manifest = {
             version: INDEX_VERSION,
             kind: 'docs',
             createdAt: new Date().toISOString(),
             indexer: options.indexer,
             embedding: {
-                ref: options.embeddingRef ?? options.embedder.id,
+                ref,
                 id: options.embedder.id,
-                dimensions: vectors[0]?.length ?? 0,
+                dimensions,
+                requested: options.dimensions,
             },
             sources,
             counts,
@@ -60,10 +79,18 @@ export async function buildIndex(options) {
         const outline = { files: corpus.docs.map((doc) => doc.outline) };
         const documents = Object.fromEntries(corpus.docs.map((doc) => [doc.name, doc.text]));
         await writeIndex(options.out, { manifest, outline, documents });
+        cache.commit();
         journal.finish(manifest);
-        return { manifest, chunks };
+        return {
+            manifest,
+            chunks,
+            timings: journal.timings,
+            reused: { parses: corpus.cached, vectors: cache.hits },
+        };
     }
     catch (err) {
+        writer?.close();
+        cache.abandon();
         journal.fail(err);
         throw err;
     }
@@ -86,23 +113,22 @@ function recordsOf(corpus) {
         tokens: chunk.tokens,
     })));
 }
-async function embedAll(chunks, options, journal) {
-    const size = options.batch ?? DEFAULT_BATCH;
-    const out = [];
-    for (let at = 0; at < chunks.length; at += size) {
-        const slice = chunks.slice(at, at + size);
-        const response = await options.embedder.embed({
-            input: slice.map((c) => c.embedText),
-            taskType: 'document',
-            signal: options.signal,
-        });
-        if (response.vectors.length !== slice.length) {
-            throw new Error(`${options.embedder.id} answered ${response.vectors.length} vectors for ${slice.length} texts`);
-        }
-        out.push(...response.vectors.map((v) => Float32Array.from(v)));
-        journal.progress(out.length, chunks.length);
-        options.onProgress?.(out.length, chunks.length);
-    }
-    return out;
+async function embedAll(chunks, options, journal, cache, writer) {
+    // A window at a time, rather than the whole corpus in one call. How many
+    // texts fit in a request, and how many requests may be in flight, are still
+    // the embedder's to answer — it knows the model's caps and it is the one
+    // that sees the 429s. What the window decides is only how much is resident.
+    return embedStream({
+        embedder: options.embedder,
+        cache,
+        records: chunks,
+        textOf: (chunk) => chunk.embedText,
+        signal: options.signal,
+        onProgress: (done, total) => {
+            journal.progress(done, total);
+            options.onProgress?.(done, total);
+        },
+        onWindow: (window, vectors) => writer.add(window, vectors),
+    });
 }
 //# sourceMappingURL=build.js.map

package/dist/docs/command.js

@@ -1,10 +1,11 @@
-import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, table, usageError, write, } from '@zenera/cli/lib';
+import { bold, CliError, cyan, dim, EXIT, isInteractive, json, note, parse, paths, table, usageError, write, } from '@zenera/cli/lib';
 import { relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
 import { assertSameEmbedding } from "../common/manifest.js";
 import { PatternError } from "../common/match.js";
-import { grid } from "../common/prose.js";
+import { INTERVAL_MS } from "../common/progress.js";
+import { breakdown, grid } from "../common/prose.js";
 import { assemble, DEFAULT_MAX_LINES } from "./assemble.js";
 import { buildIndex } from "./build.js";
 import { CHUNK_KINDS } from "./chunk.js";
@@ -69,8 +70,17 @@ export const command = {
                 '  -o, --out <dir>',
                 dim(`Where the index goes. Default ${DEFAULT_DIR}, or ${DIR_ENV}.`),
             ],
-            ['  --batch <n>', dim('Texts per embedding request. Default 96.')],
+            ['  --batch <n>', dim("Texts per embedding request. Default: the model's own cap.")],
+            [
+                '  --dimensions <n>',
+                dim("Narrower vectors, if the model allows it. Default: the model's own width."),
+            ],
             ['  --chunk-tokens <n>', dim('Target chunk size. Default 384.')],
+            [
+                '  --no-cache',
+                dim('Parse and embed everything again, ignoring what is already kept.'),
+            ],
+            ['  --cache-dir <dir>', dim('Keep the work somewhere other than the shared cache.')],
         ]),
         '',
         dim('  Every document is copied into the index, so it stays portable and'),
@@ -155,40 +165,58 @@ async function index(args, ctx) {
         out: { type: 'string', short: 'o' },
         embedding: { type: 'string' },
         batch: { type: 'string' },
+        dimensions: { type: 'string' },
         'chunk-tokens': { type: 'string' },
+        'no-cache': { type: 'boolean' },
+        'cache-dir': { type: 'string' },
         quiet: { type: 'boolean' },
     }, INDEX_USAGE);
     if (positionals.length === 0) {
         throw usageError('no document, directory or pattern given', INDEX_USAGE);
     }
     const out = outputDir(ctx.cwd, values.out, DOCS_INDEX);
+    const cacheDir = values['cache-dir'] ? resolve(ctx.cwd, values['cache-dir']) : paths.cache();
     const loud = !values.quiet && !ctx.json;
-    const chosen = await resolveEmbedder(values.embedding);
+    // Undefined when unasked, all the way to the cache key: a width nobody
+    // named is not the same key as the width the model happens to default to,
+    // and resolving it here would miss every vector already paid for.
+    const dimensions = values.dimensions ? count(values.dimensions, '--dimensions') : undefined;
+    const chosen = await resolveEmbedder(values.embedding, {
+        maxBatch: values.batch ? count(values.batch, '--batch') : undefined,
+        dimensions,
+    });
     const started = Date.now();
-    const { manifest } = await buildIndex({
+    const { manifest, timings, reused } = await buildIndex({
         files: positionals,
         cwd: ctx.cwd,
         out,
         embedder: chosen,
         embeddingRef: values.embedding,
         indexer: 'zenera-rag',
-        batch: values.batch ? count(values.batch, '--batch') : undefined,
         chunk: values['chunk-tokens']
             ? { chunkTokens: count(values['chunk-tokens'], '--chunk-tokens') }
             : undefined,
+        dimensions,
+        cache: !values['no-cache'],
+        cacheDir: values['cache-dir'] ? cacheDir : undefined,
+        onReading: loud
+            ? throttled((done, total, pending) => note(dim(`  parsed ${done}/${total} · ${Math.floor((done / total) * 100)}% · ` +
+                `${elapsed(started)}${still(pending)}`)))
+            : undefined,
         onRead: loud
             ? (summary) => {
                 printSources(summary.sources);
                 for (const skip of summary.skipped) {
                     note(dim(`  skipped ${skip.name}: ${skip.reason}`));
                 }
-                // The first batch takes a while and says nothing while it
-                // does; this is the line that makes that a wait, not a hang.
+                // Embedding is one call now, and a long one; this is the line
+                // that makes the wait before the first progress report a wait
+                // rather than a hang.
                 note(dim(`  embedding ${summary.counts.chunks} chunks with ${chosen.id} …`));
             }
             : undefined,
         onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.round((done / total) * 100)}% · ${elapsed(started)}`))
+            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
             : undefined,
     });
     if (ctx.json) {
@@ -202,6 +230,10 @@ async function index(args, ctx) {
     note(`  wrote ${bold(String(manifest.counts.chunks))} chunks from ` +
         `${bold(String(manifest.counts.documents))} document(s) to ${bold(out)}, ` +
         `embedded with ${manifest.embedding.ref} (${manifest.embedding.dimensions}d)`);
+    note(dim(`  ${breakdown(timings)}`));
+    note(dim(`  reused ${reused.parses}/${manifest.counts.documents} parses, ` +
+        `${reused.vectors}/${manifest.counts.chunks} vectors` +
+        `${values['no-cache'] ? ' (--no-cache)' : ` from ${cacheDir}`}`));
     const where = out === resolve(ctx.cwd, DEFAULT_DIR) ? '' : ` --dir ${relative(ctx.cwd, out) || out}`;
     note(dim(`  search it: ${cyan(`zen rag docs search${where} "what you are after"`)}`));
 }
@@ -233,6 +265,18 @@ function elapsed(since) {
     const seconds = Math.round((Date.now() - since) / 1000);
     return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
 }
+/** Reading fires per document, and a line per document is not a narration. */
+function throttled(say) {
+    let saidAt = 0;
+    return (...args) => {
+        if (Date.now() - saidAt >= INTERVAL_MS) {
+            saidAt = Date.now();
+            say(...args);
+        }
+    };
+}
+/** Only worth naming when there are few enough to go and look at. */
+const still = (pending) => pending.length > 0 && pending.length <= 2 ? ` · still on ${pending.join(', ')}` : '';
 const MANY = { type: 'string', multiple: true };
 async function search(args, ctx) {
     const { values, positionals } = parse(args, {
@@ -281,7 +325,9 @@ async function search(args, ctx) {
     const manifest = await readManifest(dir);
     const ref = values.embedding ?? manifest.embedding.ref;
     assertSameEmbedding(manifest, ref);
-    const found = await DocsIndex.open(dir, await resolveEmbedder(ref));
+    // A query has to be asked at the width the passages were written at, and
+    // the ref alone does not say what that was.
+    const found = await DocsIndex.open(dir, await resolveEmbedder(ref, { dimensions: manifest.embedding.requested }));
     try {
         if (values.interactive) {
             await repl(found, query, shape);

package/dist/docs/load.d.ts

@@ -1,6 +1,6 @@
-import { type Chunk, type ChunkOptions } from './chunk.ts';
+import type { Chunk, ChunkOptions } from './chunk.ts';
 import type { FileOutline } from './files.ts';
-import { type DocFormat, type ParsedDoc } from './parse.ts';
+import { type DocFormat } from './parse.ts';
 /** Markdown, and plain text read as paragraphs. Anything else is not a document. */
 export declare const DOC_EXTENSIONS: readonly [".md", ".markdown", ".txt", ".text"];
 export interface LoadedDoc {
@@ -11,7 +11,6 @@ export interface LoadedDoc {
     format: DocFormat;
     /** the document verbatim, CRLF normalized: what goes into `sources/` */
     text: string;
-    parsed: ParsedDoc;
     chunks: Chunk[];
     outline: FileOutline;
 }
@@ -22,7 +21,22 @@ export interface Corpus {
         name: string;
         reason: string;
     }[];
+    /** documents whose chunks came from a previous build rather than the parser */
+    cached: number;
 }
-export declare function loadDocuments(inputs: readonly string[], cwd: string, options?: ChunkOptions): Promise<Corpus>;
+export interface LoadOptions {
+    chunk?: ChunkOptions;
+    /** remember what documents parse to; on by default */
+    cache?: boolean;
+    /** keep the parses somewhere other than the shared store */
+    cacheDir?: string;
+    onProgress?: (done: number, total: number, pending: readonly string[]) => void;
+}
+/**
+ * Reading is cheap and parsing is not, so the two are split: every file is read
+ * and hashed here, and only the documents the cache has never seen are handed
+ * to the pool. On a rebuild of an unchanged corpus that is none of them.
+ */
+export declare function loadDocuments(inputs: readonly string[], cwd: string, options?: LoadOptions): Promise<Corpus>;
 export declare const formatOf: (path: string) => DocFormat;
 //# sourceMappingURL=load.d.ts.map

package/dist/docs/load.js

@@ -3,8 +3,9 @@ import { createHash } from 'node:crypto';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { isGlob, wildcard } from "../common/match.js";
-import { chunkDocument } from "./chunk.js";
-import { normalize, parseDocument } from "./parse.js";
+import { NO_PARSE_CACHE, openParseCache, parseKey } from "./parse-cache.js";
+import { normalize } from "./parse.js";
+import { parseAll } from "./pool.js";
 // ---------------------------------------------------------------------------
 // Finding the documents, and reading them
 //
@@ -28,6 +29,11 @@ export const DOC_EXTENSIONS = ['.md', '.markdown', '.txt', '.text'];
 const MAX_BYTES = 16 * 1024 * 1024;
 /** Enough for a documentation tree; far short of a filesystem. */
 const MAX_FILES = 20_000;
+/**
+ * Reading is cheap and parsing is not, so the two are split: every file is read
+ * and hashed here, and only the documents the cache has never seen are handed
+ * to the pool. On a rebuild of an unchanged corpus that is none of them.
+ */
 export async function loadDocuments(inputs, cwd, options = {}) {
     const found = await discover(inputs, cwd);
     if (found.length === 0) {
@@ -35,7 +41,9 @@ export async function loadDocuments(inputs, cwd, options = {}) {
     }
     const root = commonRoot(found);
     const taken = new Set();
-    const docs = [];
+    const chunk = options.chunk ?? {};
+    const cache = options.cache === false ? NO_PARSE_CACHE : openParseCache(options.cacheDir);
+    const read = [];
     const skipped = [];
     for (const absolute of found) {
         const name = distinct(nameOf(root, absolute), taken);
@@ -45,23 +53,55 @@ export async function loadDocuments(inputs, cwd, options = {}) {
             continue;
         }
         const raw = await readFile(absolute);
-        const text = normalize(raw.toString('utf8'));
-        const format = formatOf(absolute);
-        const parsed = parseDocument(text, name, format);
-        const chunks = chunkDocument(parsed, options);
-        docs.push({
+        read.push({
             name,
             file: basename(absolute),
             sha256: createHash('sha256').update(raw).digest('hex'),
             bytes: raw.byteLength,
-            format,
-            text,
-            parsed,
-            chunks,
-            outline: outlineOf(parsed, chunks.length),
+            format: formatOf(absolute),
+            text: normalize(raw.toString('utf8')),
+            key: parseKey(raw, name, chunk),
         });
     }
-    return { docs, skipped };
+    const parsed = new Array(read.length);
+    const misses = [];
+    const missAt = [];
+    for (const [at, doc] of read.entries()) {
+        const hit = doc.key === undefined ? undefined : cache.get(doc.key);
+        if (hit) {
+            parsed[at] = hit;
+        }
+        else {
+            missAt.push(at);
+            misses.push({ name: doc.name, text: doc.text, format: doc.format });
+        }
+    }
+    const done = read.length - misses.length;
+    options.onProgress?.(done, read.length, []);
+    const fresh = await parseAll(misses, {
+        chunk,
+        onProgress: (at, _of, pending) => options.onProgress?.(done + at, read.length, pending),
+    });
+    for (const [i, result] of fresh.entries()) {
+        const at = missAt[i];
+        parsed[at] = result;
+        const key = read[at].key;
+        if (key !== undefined) {
+            cache.put(key, result);
+        }
+    }
+    cache.commit();
+    const docs = read.map((doc, at) => ({
+        name: doc.name,
+        file: doc.file,
+        sha256: doc.sha256,
+        bytes: doc.bytes,
+        format: doc.format,
+        text: doc.text,
+        chunks: parsed[at].chunks,
+        outline: parsed[at].outline,
+    }));
+    return { docs, skipped, cached: cache.hits };
 }
 export const formatOf = (path) => ['.txt', '.text'].includes(extname(path).toLowerCase()) ? 'text' : 'markdown';
 // ---------------------------------------------------------------------------
@@ -160,53 +200,4 @@ function distinct(name, taken) {
     taken.add(candidate);
     return candidate;
 }
-// ---------------------------------------------------------------------------
-// the outline
-// ---------------------------------------------------------------------------
-/**
- * Headings and tables, with the line each ends on. That end is what makes the
- * outline enough on its own: a section runs from its heading to the line before
- * the next heading at the same depth or shallower, so scoping a search to a
- * section, listing what is in one, or naming the sections a skipped range
- * covered are all answerable without reading the document.
- */
-function outlineOf(doc, chunks) {
-    const sections = doc.sections.filter((s) => s.line !== undefined);
-    const headings = sections.map((section, at) => {
-        const next = sections.findIndex((other, i) => i > at && other.level <= section.level);
-        const end = next === -1 ? doc.lines.length : sections[next].line - 1;
-        return {
-            line: section.line,
-            end,
-            level: section.level,
-            title: section.title,
-            id: section.id,
-            path: section.path,
-        };
-    });
-    const tables = doc.blocks
-        .filter((block) => block.table)
-        .map((block) => {
-        const table = block.table;
-        return {
-            id: block.id,
-            path: block.path,
-            section: block.section.path,
-            line: block.start,
-            end: block.end,
-            columns: table.columns,
-            rows: table.rows.length,
-            caption: table.caption,
-        };
-    });
-    return {
-        name: doc.name,
-        title: doc.title,
-        format: doc.format,
-        lines: doc.lines.length,
-        chunks,
-        headings,
-        tables,
-    };
-}
 //# sourceMappingURL=load.js.map

package/dist/docs/outline.d.ts

@@ -0,0 +1,11 @@
+import type { FileOutline } from './files.ts';
+import type { ParsedDoc } from './parse.ts';
+/**
+ * Headings and tables, with the line each ends on. That end is what makes the
+ * outline enough on its own: a section runs from its heading to the line before
+ * the next heading at the same depth or shallower, so scoping a search to a
+ * section, listing what is in one, or naming the sections a skipped range
+ * covered are all answerable without reading the document.
+ */
+export declare function outlineOf(doc: ParsedDoc, chunks: number): FileOutline;
+//# sourceMappingURL=outline.d.ts.map

package/dist/docs/outline.js

@@ -0,0 +1,55 @@
+// ---------------------------------------------------------------------------
+// What a document holds, without the document
+//
+// This lives apart from `load.ts` for one reason: a parse worker needs it, and
+// `load.ts` imports the CLI. A worker that reached for it there would load the
+// whole command layer into every thread to call one pure function. Nothing here
+// may import anything that is not pure.
+// ---------------------------------------------------------------------------
+/**
+ * Headings and tables, with the line each ends on. That end is what makes the
+ * outline enough on its own: a section runs from its heading to the line before
+ * the next heading at the same depth or shallower, so scoping a search to a
+ * section, listing what is in one, or naming the sections a skipped range
+ * covered are all answerable without reading the document.
+ */
+export function outlineOf(doc, chunks) {
+    const sections = doc.sections.filter((s) => s.line !== undefined);
+    const headings = sections.map((section, at) => {
+        const next = sections.findIndex((other, i) => i > at && other.level <= section.level);
+        const end = next === -1 ? doc.lines.length : sections[next].line - 1;
+        return {
+            line: section.line,
+            end,
+            level: section.level,
+            title: section.title,
+            id: section.id,
+            path: section.path,
+        };
+    });
+    const tables = doc.blocks
+        .filter((block) => block.table)
+        .map((block) => {
+        const table = block.table;
+        return {
+            id: block.id,
+            path: block.path,
+            section: block.section.path,
+            line: block.start,
+            end: block.end,
+            columns: table.columns,
+            rows: table.rows.length,
+            caption: table.caption,
+        };
+    });
+    return {
+        name: doc.name,
+        title: doc.title,
+        format: doc.format,
+        lines: doc.lines.length,
+        chunks,
+        headings,
+        tables,
+    };
+}
+//# sourceMappingURL=outline.js.map

package/dist/docs/parse-cache.d.ts

@@ -0,0 +1,20 @@
+import type { ChunkOptions } from './chunk.ts';
+import type { Parsed } from './pool.ts';
+export declare const PARSE_KIND = "docs-parse";
+/** Bumped when chunking changes shape, which invalidates every entry. */
+export declare const PARSE_VERSION = 1;
+export interface ParseCache {
+    get(key: string): Parsed | undefined;
+    put(key: string, parsed: Parsed): void;
+    commit(): void;
+    readonly hits: number;
+}
+export declare const NO_PARSE_CACHE: ParseCache;
+/**
+ * What the chunks depend on, and nothing else. `tokenCount` is a function and
+ * cannot be hashed; a caller that supplies one gets no cache rather than a key
+ * that quietly ignores it.
+ */
+export declare function parseKey(bytes: Buffer, name: string, options: ChunkOptions): string | undefined;
+export declare function openParseCache(dir?: string): ParseCache;
+//# sourceMappingURL=parse-cache.d.ts.map

package/dist/docs/parse-cache.js

@@ -0,0 +1,60 @@
+import { Cache, cacheKey } from '@zenera/cli/lib';
+import { createHash } from 'node:crypto';
+// ---------------------------------------------------------------------------
+// Not parsing the same document twice
+//
+// Once vectors are cached, parsing is what a rebuild of an unchanged corpus
+// spends all of its time on — the threads make it several times faster, and
+// this makes it nothing at all. A document that has not changed produces
+// exactly the chunks it produced last time, so the chunks are what is kept.
+//
+// The key is the file's bytes together with everything that decides how they
+// are cut: change the document, the chunk settings or this module, and the
+// entry is a miss rather than a wrong answer. It sits in the machine's shared
+// cache next to the vectors, so a corpus indexed into two directories is read
+// once, and it follows the same rule everything there does — every error is a
+// miss and nothing else.
+// ---------------------------------------------------------------------------
+export const PARSE_KIND = 'docs-parse';
+/** Bumped when chunking changes shape, which invalidates every entry. */
+export const PARSE_VERSION = 1;
+export const NO_PARSE_CACHE = {
+    get: () => undefined,
+    put: () => { },
+    commit: () => { },
+    hits: 0,
+};
+/**
+ * What the chunks depend on, and nothing else. `tokenCount` is a function and
+ * cannot be hashed; a caller that supplies one gets no cache rather than a key
+ * that quietly ignores it.
+ */
+export function parseKey(bytes, name, options) {
+    if (typeof options.tokenCount === 'function') {
+        return undefined;
+    }
+    return cacheKey(PARSE_VERSION, name, options.chunkTokens, options.minChunkTokens, options.maxChunkTokens, options.tableSliceTokens, createHash('sha256').update(bytes).digest('hex'));
+}
+export function openParseCache(dir) {
+    return new StoredParses(new Cache(PARSE_KIND, { dir }));
+}
+class StoredParses {
+    #store;
+    constructor(store) {
+        this.#store = store;
+    }
+    get hits() {
+        return this.#store.hits;
+    }
+    get(key) {
+        const found = this.#store.get(key);
+        return found?.chunks && found.outline ? found : undefined;
+    }
+    put(key, parsed) {
+        this.#store.put(key, { chunks: parsed.chunks, outline: parsed.outline });
+    }
+    commit() {
+        this.#store.commit();
+    }
+}
+//# sourceMappingURL=parse-cache.js.map

package/dist/docs/parse-worker.d.ts

@@ -0,0 +1,13 @@
+export interface ParseJob {
+    at: number;
+    name: string;
+    text: string;
+    format: 'markdown' | 'text';
+}
+export interface ParseDone {
+    at: number;
+    chunks?: unknown;
+    outline?: unknown;
+    error?: string;
+}
+//# sourceMappingURL=parse-worker.d.ts.map

package/dist/docs/parse-worker.js

@@ -0,0 +1,21 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { chunkDocument } from "./chunk.js";
+import { outlineOf } from "./outline.js";
+import { parseDocument } from "./parse.js";
+const options = (workerData ?? {});
+parentPort?.on('message', (job) => {
+    try {
+        const parsed = parseDocument(job.text, job.name, job.format);
+        const chunks = chunkDocument(parsed, options);
+        parentPort.postMessage({ at: job.at, chunks, outline: outlineOf(parsed, chunks.length) });
+    }
+    catch (err) {
+        // Sent back rather than thrown: an uncaught throw here kills the thread,
+        // and the pool would lose the other documents queued behind this one.
+        parentPort.postMessage({
+            at: job.at,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+});
+//# sourceMappingURL=parse-worker.js.map