@arnilo/prism-rag 0.3.0 → 0.3.2

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [0.3.2] - 2026-08-29
+
+### Changed
+- Plan 035-039 changed-package cut: additive runtime performance, tooling, and documentation deltas; peer window refresh.
+
+## [0.3.1] - 2026-08-26
+
+### Added
+- Hybrid retrieve (`lexical` / RRF / `retrieval` labels), multi-scope retrieve (`scopes: RagScope[]` — one embed, per-scope legs, one RRF, one rerank; `scope` stays valid), content-hash skip, heading-stack chunk metadata, generation stamping, `RagTelemetry` seam, `createTeiReranker`, `fuseReciprocalRank`, `isValidContentHash`. `HARD_RETRIEVE_SCOPE_CAP`. Provenance carries `tenantId`/`resourceId`/`corpusId`. Retrieve fails closed on embedder id/dimension drift (`ERR_PRISM_RAG_EMBEDDER_MISMATCH`).
+
 ## [0.1.0] - 2026-08-09
 
 ### Changed

package/dist/chunk.js

@@ -18,12 +18,26 @@ function chunkDocument(text, options, markdown) {
     if (text.length > limits.maxDocumentChars) {
         throw new RagLimitError(`document exceeds ${limits.maxDocumentChars} characters`);
     }
-    if (options.metadata)
-        assertBytes(options.metadata, 64 * 1024, "chunk metadata");
+    const callerMeta = options.metadata;
+    if (callerMeta)
+        assertBytes(callerMeta, 64 * 1024, "chunk metadata");
     if (!text.trim())
         return Object.freeze([]);
+    const headings = [];
+    if (markdown) {
+        let pos = 0;
+        for (const line of text.split("\n")) {
+            const trimmed = line.trimStart();
+            const match = trimmed.match(/^(#{1,6})\s+(.+)$/);
+            if (match)
+                headings.push({ pos, level: match[1].length, text: match[2].trim() });
+            pos += line.length + 1; // +1 for the newline character
+        }
+    }
     const chunks = [];
     let start = 0;
+    let headingIdx = 0;
+    const headingStack = [];
     while (start < text.length) {
         while (start < text.length && /\s/.test(text[start]))
             start += 1;
@@ -33,8 +47,23 @@ function chunkDocument(text, options, markdown) {
         const end = ceiling === text.length ? ceiling : preferredEnd(text, start, ceiling, markdown);
         const raw = text.slice(start, end).trimEnd();
         if (raw) {
+            // Advance heading stack to the current chunk start position.
+            while (headingIdx < headings.length && headings[headingIdx].pos <= start) {
+                const h = headings[headingIdx];
+                while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= h.level) {
+                    headingStack.pop();
+                }
+                headingStack.push({ level: h.level, text: h.text });
+                headingIdx++;
+            }
             const index = chunks.length;
             const citationId = `${sourceId}#${String(index + 1).padStart(4, "0")}`;
+            const meta = callerMeta ? { ...callerMeta } : {};
+            // Stamp heading metadata unless the caller supplied explicit heading data.
+            if (markdown && meta.heading === undefined && headingStack.length > 0) {
+                meta.heading = headingStack.map((h) => h.text);
+            }
+            const metadata = Object.keys(meta).length > 0 ? Object.freeze(meta) : undefined;
             chunks.push(Object.freeze({
                 id: citationId,
                 citationId,
@@ -43,7 +72,7 @@ function chunkDocument(text, options, markdown) {
                 start,
                 end: start + raw.length,
                 text: raw,
-                ...(options.metadata ? { metadata: Object.freeze({ ...options.metadata }) } : {}),
+                ...(metadata ? { metadata } : {}),
             }));
             if (chunks.length > limits.maxChunks)
                 throw new RagLimitError(`chunk count exceeds ${limits.maxChunks}`);

package/dist/fusion.d.ts

@@ -0,0 +1,18 @@
+import type { MemoryVectorHit } from "@arnilo/prism-memory";
+/** Which retrieval leg(s) surfaced a hit before fusion. */
+export type RetrievalLeg = "vector" | "lexical";
+export interface FusedCandidate {
+    readonly hit: MemoryVectorHit;
+    readonly retrieval: RetrievalLeg | "hybrid";
+}
+/** N-list RRF. Fusion key includes scope so the same id in two corpora stays two hits. */
+export declare function fuseReciprocalRankLists(lists: readonly {
+    readonly hits: readonly MemoryVectorHit[];
+    readonly leg: RetrievalLeg;
+}[], rrfK: number): readonly FusedCandidate[];
+/**
+ * Reciprocal-rank fusion: score(id) = Σ 1/(rrfK + rank) over each list containing the id.
+ * Pure and store-free; ties break by best rank, then id, so output order is deterministic.
+ * When a record appears in both legs the vector-leg hit object wins (display score stays cosine).
+ */
+export declare function fuseReciprocalRank(vectorHits: readonly MemoryVectorHit[], lexicalHits: readonly MemoryVectorHit[], rrfK: number): readonly FusedCandidate[];

package/dist/hash.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Document hashes are supplied by the host (never computed over unbounded bytes inside the
+ * engine). Accept any hex digest of 32..128 characters so hosts can pick the algorithm;
+ * comparison is exact-match on the normalized lowercase form.
+ */
+export declare function isValidContentHash(value: unknown): value is string;

package/dist/hash.js

@@ -0,0 +1,9 @@
+/**
+ * Document hashes are supplied by the host (never computed over unbounded bytes inside the
+ * engine). Accept any hex digest of 32..128 characters so hosts can pick the algorithm;
+ * comparison is exact-match on the normalized lowercase form.
+ */
+export function isValidContentHash(value) {
+    return typeof value === "string" && /^[0-9a-f]{32,128}$/.test(value);
+}
+//# sourceMappingURL=hash.js.map

package/dist/index.d.ts

@@ -1,14 +1,20 @@
 export { chunkMarkdown, chunkText } from "./chunk.js";
 export { createRagContextProvider } from "./context.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export type { FusedCandidate, RetrievalLeg } from "./fusion.js";
+export { fuseReciprocalRank } from "./fusion.js";
+export { isValidContentHash } from "./hash.js";
 export { indexChunks } from "./indexing.js";
 export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
 export type { RagLimits, RagLimitsInput } from "./limits.js";
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_RETRIEVE_SCOPE_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
 export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
 export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
 export type { SourceMutationResult } from "./sources.js";
 export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
-export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
+export type { CreateTeiRerankerOptions } from "./tei-reranker.js";
+export { createTeiReranker } from "./tei-reranker.js";
+export type { RagTelemetry, RagTelemetryAttributeValue, RagTelemetrySpan } from "./telemetry.js";
+export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, ReusableEmbedding, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
 export declare const packageName = "@arnilo/prism-rag";

package/dist/index.js

@@ -1,12 +1,15 @@
 export { chunkMarkdown, chunkText } from "./chunk.js";
 export { createRagContextProvider } from "./context.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export { fuseReciprocalRank } from "./fusion.js";
+export { isValidContentHash } from "./hash.js";
 export { indexChunks } from "./indexing.js";
 export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_RETRIEVE_SCOPE_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
 export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
 export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
 export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
+export { createTeiReranker } from "./tei-reranker.js";
 export const packageName = "@arnilo/prism-rag";
 //# sourceMappingURL=index.js.map

package/dist/indexing.js

@@ -1,5 +1,6 @@
 import { resolveRedactor } from "@arnilo/prism";
 import { RagValidationError } from "./errors.js";
+import { isValidContentHash } from "./hash.js";
 import { ingestionStatus } from "./ingestion-status.js";
 import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
 import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId } from "./util.js";
@@ -20,6 +21,7 @@ export async function indexChunkBatches(options, write) {
         options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
+    nonEmpty(options.embedder.id, "embedder.id");
     if (options.chunks.length > limits.maxChunks)
         throw new RagValidationError(`chunk count exceeds ${limits.maxChunks}`);
     const redactor = resolveRedactor(options.redactor, options.secrets);
@@ -59,13 +61,29 @@ export async function indexChunkBatches(options, write) {
         }
     };
     await setStatus("pending");
+    const embedSpan = options.telemetry?.startSpan("embedding.index", { "rag.embedder_id": options.embedder.id }, options.telemetryParent);
     try {
         for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
             assertNotAborted(options.signal);
             const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
             const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
-            const vectors = await options.embedder.embed(texts, { signal: options.signal });
-            if (vectors.length !== batch.length)
+            // Embed only the delta: chunks whose id+text match a stored record reuse its embedding.
+            const vectors = batch.map((chunk, index) => {
+                const reused = options.reuseEmbeddings?.get(chunk.id);
+                if (!reused || reused.text !== texts[index])
+                    return undefined;
+                return [...reused.embedding];
+            });
+            const pending = vectors.flatMap((vector, index) => (vector === undefined ? [index] : []));
+            if (pending.length) {
+                const fresh = await options.embedder.embed(pending.map((index) => texts[index]), { signal: options.signal });
+                if (fresh.length !== pending.length)
+                    throw new RagValidationError("embedder returned unexpected vector count");
+                pending.forEach((index, position) => {
+                    vectors[index] = [...fresh[position]];
+                });
+            }
+            if (vectors.some((vector) => vector === undefined))
                 throw new RagValidationError("embedder returned unexpected vector count");
             const records = batch.map((chunk, index) => {
                 const embedding = vectors[index];
@@ -75,7 +93,14 @@ export async function indexChunkBatches(options, write) {
                 const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? chunk.metadata ?? {};
                 const metadata = {
                     ...safeMetadata,
-                    _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end },
+                    _rag: {
+                        sourceId: chunk.sourceId,
+                        citationId: chunk.citationId,
+                        chunkIndex: chunk.index,
+                        start: chunk.start,
+                        end: chunk.end,
+                        ...(options.contentHash && isValidContentHash(options.contentHash) ? { contentHash: options.contentHash.toLowerCase() } : {}),
+                    },
                 };
                 assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
                 return {
@@ -86,6 +111,7 @@ export async function indexChunkBatches(options, write) {
                     text: texts[index],
                     embedding,
                     sequence: chunk.index,
+                    embedderId: options.embedder.id,
                     metadata: metadata,
                     createdAt: new Date(0).toISOString(),
                 };
@@ -99,9 +125,13 @@ export async function indexChunkBatches(options, write) {
         await setStatus("indexed");
     }
     catch (error) {
+        embedSpan?.recordError();
         await setStatus([...written.values()].some((progress) => progress.chunks > 0) ? "partial" : "failed", error);
         throw error;
     }
+    finally {
+        embedSpan?.end();
+    }
     return Object.freeze({ indexed: options.chunks.length, sourceIds: Object.freeze([...sourceIds].sort()) });
 }
 function add(target, sourceId, bytes, chunks) {

package/dist/limits.d.ts

@@ -31,6 +31,9 @@ export declare const DEFAULT_MAX_RERANK_MS = 2000;
 export declare const HARD_MAX_RERANK_MS_CAP = 10000;
 export declare const DEFAULT_RERANK_CONCURRENCY = 2;
 export declare const HARD_RERANK_CONCURRENCY_CAP = 8;
+export declare const DEFAULT_RRF_K = 60;
+export declare const HARD_RRF_K_CAP = 1000;
+export declare const HARD_RETRIEVE_SCOPE_CAP = 8;
 export declare const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
 export declare const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 export interface RagLimits {
@@ -51,6 +54,7 @@ export interface RagLimits {
     readonly maxRerankBytes: number;
     readonly maxRerankMs: number;
     readonly rerankConcurrency: number;
+    readonly rrfK: number;
     readonly ingestionStatusPageSize: number;
 }
 export type RagLimitsInput = Partial<RagLimits>;

package/dist/limits.js

@@ -32,6 +32,9 @@ export const DEFAULT_MAX_RERANK_MS = 2_000;
 export const HARD_MAX_RERANK_MS_CAP = 10_000;
 export const DEFAULT_RERANK_CONCURRENCY = 2;
 export const HARD_RERANK_CONCURRENCY_CAP = 8;
+export const DEFAULT_RRF_K = 60;
+export const HARD_RRF_K_CAP = 1_000;
+export const HARD_RETRIEVE_SCOPE_CAP = 8;
 export const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
 export const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 function integer(value, fallback, cap, label, minimum = 1) {
@@ -70,6 +73,7 @@ export function resolveRagLimits(input = {}) {
         maxRerankBytes: integer(input.maxRerankBytes, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, "maxRerankBytes"),
         maxRerankMs: integer(input.maxRerankMs, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, "maxRerankMs"),
         rerankConcurrency: integer(input.rerankConcurrency, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, "rerankConcurrency"),
+        rrfK: integer(input.rrfK, DEFAULT_RRF_K, HARD_RRF_K_CAP, "rrfK"),
         ingestionStatusPageSize: integer(input.ingestionStatusPageSize, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, "ingestionStatusPageSize"),
     });
 }

package/dist/parsers.js

@@ -36,7 +36,19 @@ async function parsePdf(document, options = {}) {
     const pages = [...pdf.matchAll(/\/Type\s*\/Page\b/gu)].length;
     if (pages > limits.maxPdfPages)
         throw new RagLimitError(`PDF exceeds ${limits.maxPdfPages} pages`);
-    const blocks = pdf.match(/BT[\s\S]*?ET/gu) ?? [];
+    // Index-scanned BT..ET block extraction instead of /BT[\s\S]*?ET/gu (CodeQL js/polynomial-redos, alert 11).
+    const blocks = [];
+    let cursor = 0;
+    while (cursor < pdf.length) {
+        const begin = pdf.indexOf("BT", cursor);
+        if (begin === -1)
+            break;
+        const end = pdf.indexOf("ET", begin + 2);
+        if (end === -1)
+            break;
+        blocks.push(pdf.slice(begin, end + 2));
+        cursor = end + 2;
+    }
     const text = blocks.flatMap(pdfBlockText).join("\n").trim();
     if (!text)
         throw new RagValidationError("PDF has no uncompressed text");
@@ -65,17 +77,86 @@ function assertParseTime(started, maxParseMs) {
     if (Date.now() - started > maxParseMs)
         throw new RagLimitError(`document parsing exceeded ${maxParseMs}ms`);
 }
+// Single-pass linear HTML-to-text scanner (CodeQL js/incomplete-multi-character-sanitization
+// and js/polynomial-redos on the former regex chain, alerts 17-19): comments, script/style
+// bodies, and tags are consumed by index — hostile adjacency cannot re-form dangerous tags.
 function htmlToText(html) {
-    return decodeEntities(html
-        .replace(/<!--[\s\S]*?-->/gu, "")
-        .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/giu, "")
-        .replace(/<(?:br|p|div|li|h[1-6]|tr)\b[^>]*>/giu, "\n")
-        .replace(/<[^>]*>/gu, " "))
-        .replace(/[ \t]{2,}/gu, " ")
-        .replace(/[ \t]+\n/gu, "\n")
-        .replace(/\n[ \t]+/gu, "\n")
-        .replace(/\n{3,}/gu, "\n\n")
-        .trim();
+    const out = [];
+    let i = 0;
+    while (i < html.length) {
+        if (html.startsWith("<!--", i)) {
+            const close = html.indexOf("-->", i + 4);
+            i = close === -1 ? html.length : close + 3;
+            continue;
+        }
+        if (html.charCodeAt(i) === 60) {
+            const gt = html.indexOf(">", i + 1);
+            if (gt === -1) {
+                out.push(html.slice(i));
+                break;
+            }
+            const tag = html.slice(i + 1, gt);
+            i = gt + 1;
+            const closing = tag.startsWith("/");
+            const name = (closing ? tag.slice(1) : tag).match(/^[A-Za-z]+/u)?.[0]?.toLowerCase() ?? "";
+            if (closing) {
+                out.push(" ");
+            }
+            else if (name === "br" || name === "p" || name === "div" || name === "li" || name === "tr" || /^h[1-6]$/u.test(name)) {
+                out.push("\n");
+            }
+            else if (name === "script" || name === "style") {
+                // Consume the raw element body through its matching close tag.
+                const closeIdx = html.toLowerCase().indexOf(`</${name}`, i);
+                const next = closeIdx === -1 ? html.length : html.indexOf(">", closeIdx);
+                i = next === -1 ? html.length : next + 1;
+            }
+            else {
+                out.push(" ");
+            }
+            continue;
+        }
+        out.push(html[i++]);
+    }
+    return collapseTextWhitespace(decodeEntities(out.join(""))).trim();
+}
+/** Linear whitespace normalization: `[ \t]` runs collapse; 3+ newlines collapse to 2; edge spaces drop. */
+function collapseTextWhitespace(text) {
+    const out = [];
+    let i = 0;
+    let lineStart = true;
+    let spaceRun = false;
+    let newlineRun = 0;
+    while (i < text.length) {
+        const c = text[i];
+        if (c === " " || c === "\t") {
+            if (!lineStart)
+                spaceRun = true;
+            i += 1;
+            continue;
+        }
+        if (c === "\n") {
+            newlineRun += 1;
+            spaceRun = false;
+            lineStart = true;
+            i += 1;
+            continue;
+        }
+        if (newlineRun > 0) {
+            out.push("\n".repeat(Math.min(newlineRun, 2)));
+            newlineRun = 0;
+        }
+        else if (spaceRun) {
+            out.push(" ");
+            spaceRun = false;
+        }
+        lineStart = false;
+        out.push(c);
+        i += 1;
+    }
+    if (newlineRun > 0)
+        out.push("\n".repeat(Math.min(newlineRun, 2)));
+    return out.join("");
 }
 function decodeEntities(text) {
     const entities = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", "#39": "'" };
@@ -83,11 +164,70 @@ function decodeEntities(text) {
 }
 function pdfBlockText(block) {
     const strings = [];
-    for (const match of block.matchAll(/\((?:\\.|[^\\)])*\)\s*(?:Tj|['"])/gu))
-        strings.push(decodePdfString(match[0].replace(/\s*(?:Tj|['"])$/u, "")));
-    for (const match of block.matchAll(/\[([\s\S]*?)\]\s*TJ/gu)) {
-        for (const value of match[1].matchAll(/\((?:\\.|[^\\)])*\)/gu))
-            strings.push(decodePdfString(value[0]));
+    // Linear index scanners replace ambiguous nested-quantifier regexes (CodeQL js/polynomial-redos, alerts 15-16, 20-21).
+    const extractLiteral = (text, from) => {
+        const open = text.indexOf("(", from);
+        if (open === -1)
+            return undefined;
+        let i = open + 1;
+        while (i < text.length) {
+            const c = text[i];
+            if (c === "\\") {
+                i += 2;
+                continue;
+            }
+            if (c === ")")
+                return { literal: text.slice(open + 1, i), next: i + 1 };
+            i += 1;
+        }
+        return undefined; // unterminated literal - ignore
+    };
+    let cursor = 0;
+    while (cursor < block.length) {
+        const literal = extractLiteral(block, cursor);
+        if (!literal)
+            break;
+        let j = literal.next;
+        while (j < block.length && /\s/.test(block[j]))
+            j += 1;
+        if (block[j] === "T" && block[j + 1] === "j") {
+            strings.push(decodePdfString(`(${literal.literal})`));
+            cursor = j + 2;
+        }
+        else if (block[j] === "'" || block[j] === '"') {
+            strings.push(decodePdfString(`(${literal.literal})`));
+            cursor = j + 1;
+        }
+        else {
+            cursor = literal.next;
+        }
+    }
+    cursor = 0;
+    while (cursor < block.length) {
+        const open = block.indexOf("[", cursor);
+        if (open === -1)
+            break;
+        const close = block.indexOf("]", open + 1);
+        if (close === -1)
+            break;
+        let j = close + 1;
+        while (j < block.length && /\s/.test(block[j]))
+            j += 1;
+        if (block.startsWith("TJ", j)) {
+            const arr = block.slice(open + 1, close);
+            let arrCursor = 0;
+            for (;;) {
+                const literal = extractLiteral(arr, arrCursor);
+                if (!literal)
+                    break;
+                strings.push(decodePdfString(`(${literal.literal})`));
+                arrCursor = literal.next;
+            }
+            cursor = j + 2;
+        }
+        else {
+            cursor = open + 1;
+        }
     }
     return strings.filter(Boolean);
 }