@arnilo/prism-rag 0.3.0 → 0.3.1

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.3.1] - 2026-08-26
+
+### Added
+- Hybrid retrieve (`lexical` / RRF / `retrieval` labels), multi-scope retrieve (`scopes: RagScope[]` — one embed, per-scope legs, one RRF, one rerank; `scope` stays valid), content-hash skip, heading-stack chunk metadata, generation stamping, `RagTelemetry` seam, `createTeiReranker`, `fuseReciprocalRank`, `isValidContentHash`. `HARD_RETRIEVE_SCOPE_CAP`. Provenance carries `tenantId`/`resourceId`/`corpusId`. Retrieve fails closed on embedder id/dimension drift (`ERR_PRISM_RAG_EMBEDDER_MISMATCH`).
+
 ## [0.1.0] - 2026-08-09
 
 ### Changed

package/dist/chunk.js

@@ -18,12 +18,26 @@ function chunkDocument(text, options, markdown) {
     if (text.length > limits.maxDocumentChars) {
         throw new RagLimitError(`document exceeds ${limits.maxDocumentChars} characters`);
     }
-    if (options.metadata)
-        assertBytes(options.metadata, 64 * 1024, "chunk metadata");
+    const callerMeta = options.metadata;
+    if (callerMeta)
+        assertBytes(callerMeta, 64 * 1024, "chunk metadata");
     if (!text.trim())
         return Object.freeze([]);
+    const headings = [];
+    if (markdown) {
+        let pos = 0;
+        for (const line of text.split("\n")) {
+            const trimmed = line.trimStart();
+            const match = trimmed.match(/^(#{1,6})\s+(.+)$/);
+            if (match)
+                headings.push({ pos, level: match[1].length, text: match[2].trim() });
+            pos += line.length + 1; // +1 for the newline character
+        }
+    }
     const chunks = [];
     let start = 0;
+    let headingIdx = 0;
+    const headingStack = [];
     while (start < text.length) {
         while (start < text.length && /\s/.test(text[start]))
             start += 1;
@@ -33,8 +47,23 @@ function chunkDocument(text, options, markdown) {
         const end = ceiling === text.length ? ceiling : preferredEnd(text, start, ceiling, markdown);
         const raw = text.slice(start, end).trimEnd();
         if (raw) {
+            // Advance heading stack to the current chunk start position.
+            while (headingIdx < headings.length && headings[headingIdx].pos <= start) {
+                const h = headings[headingIdx];
+                while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= h.level) {
+                    headingStack.pop();
+                }
+                headingStack.push({ level: h.level, text: h.text });
+                headingIdx++;
+            }
             const index = chunks.length;
             const citationId = `${sourceId}#${String(index + 1).padStart(4, "0")}`;
+            const meta = callerMeta ? { ...callerMeta } : {};
+            // Stamp heading metadata unless the caller supplied explicit heading data.
+            if (markdown && meta.heading === undefined && headingStack.length > 0) {
+                meta.heading = headingStack.map((h) => h.text);
+            }
+            const metadata = Object.keys(meta).length > 0 ? Object.freeze(meta) : undefined;
             chunks.push(Object.freeze({
                 id: citationId,
                 citationId,
@@ -43,7 +72,7 @@ function chunkDocument(text, options, markdown) {
                 start,
                 end: start + raw.length,
                 text: raw,
-                ...(options.metadata ? { metadata: Object.freeze({ ...options.metadata }) } : {}),
+                ...(metadata ? { metadata } : {}),
             }));
             if (chunks.length > limits.maxChunks)
                 throw new RagLimitError(`chunk count exceeds ${limits.maxChunks}`);

package/dist/fusion.d.ts

@@ -0,0 +1,18 @@
+import type { MemoryVectorHit } from "@arnilo/prism-memory";
+/** Which retrieval leg(s) surfaced a hit before fusion. */
+export type RetrievalLeg = "vector" | "lexical";
+export interface FusedCandidate {
+    readonly hit: MemoryVectorHit;
+    readonly retrieval: RetrievalLeg | "hybrid";
+}
+/** N-list RRF. Fusion key includes scope so the same id in two corpora stays two hits. */
+export declare function fuseReciprocalRankLists(lists: readonly {
+    readonly hits: readonly MemoryVectorHit[];
+    readonly leg: RetrievalLeg;
+}[], rrfK: number): readonly FusedCandidate[];
+/**
+ * Reciprocal-rank fusion: score(id) = Σ 1/(rrfK + rank) over each list containing the id.
+ * Pure and store-free; ties break by best rank, then id, so output order is deterministic.
+ * When a record appears in both legs the vector-leg hit object wins (display score stays cosine).
+ */
+export declare function fuseReciprocalRank(vectorHits: readonly MemoryVectorHit[], lexicalHits: readonly MemoryVectorHit[], rrfK: number): readonly FusedCandidate[];

package/dist/hash.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Document hashes are supplied by the host (never computed over unbounded bytes inside the
+ * engine). Accept any hex digest of 32..128 characters so hosts can pick the algorithm;
+ * comparison is exact-match on the normalized lowercase form.
+ */
+export declare function isValidContentHash(value: unknown): value is string;

package/dist/hash.js

@@ -0,0 +1,9 @@
+/**
+ * Document hashes are supplied by the host (never computed over unbounded bytes inside the
+ * engine). Accept any hex digest of 32..128 characters so hosts can pick the algorithm;
+ * comparison is exact-match on the normalized lowercase form.
+ */
+export function isValidContentHash(value) {
+    return typeof value === "string" && /^[0-9a-f]{32,128}$/.test(value);
+}
+//# sourceMappingURL=hash.js.map

package/dist/index.d.ts

@@ -1,14 +1,20 @@
 export { chunkMarkdown, chunkText } from "./chunk.js";
 export { createRagContextProvider } from "./context.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export type { FusedCandidate, RetrievalLeg } from "./fusion.js";
+export { fuseReciprocalRank } from "./fusion.js";
+export { isValidContentHash } from "./hash.js";
 export { indexChunks } from "./indexing.js";
 export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
 export type { RagLimits, RagLimitsInput } from "./limits.js";
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_RETRIEVE_SCOPE_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
 export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
 export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
 export type { SourceMutationResult } from "./sources.js";
 export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
-export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
+export type { CreateTeiRerankerOptions } from "./tei-reranker.js";
+export { createTeiReranker } from "./tei-reranker.js";
+export type { RagTelemetry, RagTelemetryAttributeValue, RagTelemetrySpan } from "./telemetry.js";
+export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, ReusableEmbedding, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
 export declare const packageName = "@arnilo/prism-rag";

package/dist/index.js

@@ -1,12 +1,15 @@
 export { chunkMarkdown, chunkText } from "./chunk.js";
 export { createRagContextProvider } from "./context.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export { fuseReciprocalRank } from "./fusion.js";
+export { isValidContentHash } from "./hash.js";
 export { indexChunks } from "./indexing.js";
 export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_RETRIEVE_SCOPE_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
 export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
 export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
 export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
+export { createTeiReranker } from "./tei-reranker.js";
 export const packageName = "@arnilo/prism-rag";
 //# sourceMappingURL=index.js.map

package/dist/indexing.js

@@ -1,5 +1,6 @@
 import { resolveRedactor } from "@arnilo/prism";
 import { RagValidationError } from "./errors.js";
+import { isValidContentHash } from "./hash.js";
 import { ingestionStatus } from "./ingestion-status.js";
 import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
 import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId } from "./util.js";
@@ -20,6 +21,7 @@ export async function indexChunkBatches(options, write) {
         options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
+    nonEmpty(options.embedder.id, "embedder.id");
     if (options.chunks.length > limits.maxChunks)
         throw new RagValidationError(`chunk count exceeds ${limits.maxChunks}`);
     const redactor = resolveRedactor(options.redactor, options.secrets);
@@ -59,13 +61,29 @@ export async function indexChunkBatches(options, write) {
         }
     };
     await setStatus("pending");
+    const embedSpan = options.telemetry?.startSpan("embedding.index", { "rag.embedder_id": options.embedder.id }, options.telemetryParent);
     try {
         for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
             assertNotAborted(options.signal);
             const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
             const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
-            const vectors = await options.embedder.embed(texts, { signal: options.signal });
-            if (vectors.length !== batch.length)
+            // Embed only the delta: chunks whose id+text match a stored record reuse its embedding.
+            const vectors = batch.map((chunk, index) => {
+                const reused = options.reuseEmbeddings?.get(chunk.id);
+                if (!reused || reused.text !== texts[index])
+                    return undefined;
+                return [...reused.embedding];
+            });
+            const pending = vectors.flatMap((vector, index) => (vector === undefined ? [index] : []));
+            if (pending.length) {
+                const fresh = await options.embedder.embed(pending.map((index) => texts[index]), { signal: options.signal });
+                if (fresh.length !== pending.length)
+                    throw new RagValidationError("embedder returned unexpected vector count");
+                pending.forEach((index, position) => {
+                    vectors[index] = [...fresh[position]];
+                });
+            }
+            if (vectors.some((vector) => vector === undefined))
                 throw new RagValidationError("embedder returned unexpected vector count");
             const records = batch.map((chunk, index) => {
                 const embedding = vectors[index];
@@ -75,7 +93,14 @@ export async function indexChunkBatches(options, write) {
                 const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? chunk.metadata ?? {};
                 const metadata = {
                     ...safeMetadata,
-                    _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end },
+                    _rag: {
+                        sourceId: chunk.sourceId,
+                        citationId: chunk.citationId,
+                        chunkIndex: chunk.index,
+                        start: chunk.start,
+                        end: chunk.end,
+                        ...(options.contentHash && isValidContentHash(options.contentHash) ? { contentHash: options.contentHash.toLowerCase() } : {}),
+                    },
                 };
                 assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
                 return {
@@ -86,6 +111,7 @@ export async function indexChunkBatches(options, write) {
                     text: texts[index],
                     embedding,
                     sequence: chunk.index,
+                    embedderId: options.embedder.id,
                     metadata: metadata,
                     createdAt: new Date(0).toISOString(),
                 };
@@ -99,9 +125,13 @@ export async function indexChunkBatches(options, write) {
         await setStatus("indexed");
     }
     catch (error) {
+        embedSpan?.recordError();
         await setStatus([...written.values()].some((progress) => progress.chunks > 0) ? "partial" : "failed", error);
         throw error;
     }
+    finally {
+        embedSpan?.end();
+    }
     return Object.freeze({ indexed: options.chunks.length, sourceIds: Object.freeze([...sourceIds].sort()) });
 }
 function add(target, sourceId, bytes, chunks) {

package/dist/limits.d.ts

@@ -31,6 +31,9 @@ export declare const DEFAULT_MAX_RERANK_MS = 2000;
 export declare const HARD_MAX_RERANK_MS_CAP = 10000;
 export declare const DEFAULT_RERANK_CONCURRENCY = 2;
 export declare const HARD_RERANK_CONCURRENCY_CAP = 8;
+export declare const DEFAULT_RRF_K = 60;
+export declare const HARD_RRF_K_CAP = 1000;
+export declare const HARD_RETRIEVE_SCOPE_CAP = 8;
 export declare const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
 export declare const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 export interface RagLimits {
@@ -51,6 +54,7 @@ export interface RagLimits {
     readonly maxRerankBytes: number;
     readonly maxRerankMs: number;
     readonly rerankConcurrency: number;
+    readonly rrfK: number;
     readonly ingestionStatusPageSize: number;
 }
 export type RagLimitsInput = Partial<RagLimits>;

package/dist/limits.js

@@ -32,6 +32,9 @@ export const DEFAULT_MAX_RERANK_MS = 2_000;
 export const HARD_MAX_RERANK_MS_CAP = 10_000;
 export const DEFAULT_RERANK_CONCURRENCY = 2;
 export const HARD_RERANK_CONCURRENCY_CAP = 8;
+export const DEFAULT_RRF_K = 60;
+export const HARD_RRF_K_CAP = 1_000;
+export const HARD_RETRIEVE_SCOPE_CAP = 8;
 export const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
 export const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 function integer(value, fallback, cap, label, minimum = 1) {
@@ -70,6 +73,7 @@ export function resolveRagLimits(input = {}) {
         maxRerankBytes: integer(input.maxRerankBytes, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, "maxRerankBytes"),
         maxRerankMs: integer(input.maxRerankMs, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, "maxRerankMs"),
         rerankConcurrency: integer(input.rerankConcurrency, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, "rerankConcurrency"),
+        rrfK: integer(input.rrfK, DEFAULT_RRF_K, HARD_RRF_K_CAP, "rrfK"),
         ingestionStatusPageSize: integer(input.ingestionStatusPageSize, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, "ingestionStatusPageSize"),
     });
 }

package/dist/retrieve.js

@@ -1,14 +1,15 @@
 import { resolveRedactor } from "@arnilo/prism";
-import { RagScopeError, RagValidationError } from "./errors.js";
-import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+import { fuseReciprocalRankLists } from "./fusion.js";
+import { HARD_CHUNK_SIZE_CAP, HARD_RETRIEVE_SCOPE_CAP, resolveRagLimits } from "./limits.js";
 import { rerankHits } from "./rerank.js";
-import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
+import { assertBytes, assertNotAborted, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
 const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
 export async function retrieveContext(query, options) {
     nonEmpty(query, "query");
     if (query.length > HARD_CHUNK_SIZE_CAP)
         throw new RagValidationError(`query exceeds ${HARD_CHUNK_SIZE_CAP} characters`);
-    const scope = requireScope(options.scope);
+    const scopes = resolveRetrieveScopes(options);
     const limits = resolveRagLimits({
         topK: options.topK,
         queryCandidates: options.queryCandidates,
@@ -19,6 +20,7 @@ export async function retrieveContext(query, options) {
         maxRerankBytes: options.maxRerankBytes,
         maxRerankMs: options.maxRerankMs,
         rerankConcurrency: options.rerankConcurrency,
+        rrfK: options.rrfK,
     });
     if (!Number.isInteger(options.embedder.dimensions) ||
         options.embedder.dimensions <= 0 ||
@@ -27,100 +29,220 @@ export async function retrieveContext(query, options) {
     }
     if (options.filter)
         assertBytes(options.filter, limits.maxMetadataBytes, "metadata filter");
-    const redactor = resolveRedactor(options.redactor, options.secrets);
-    const safeQuery = redactor?.redact(query) ?? query;
-    assertNotAborted(options.signal);
-    const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
-    const embedding = vectors[0];
-    if (vectors.length !== 1 ||
-        !embedding ||
-        embedding.length !== options.embedder.dimensions ||
-        embedding.some((value) => !Number.isFinite(value))) {
-        throw new RagValidationError("embedder returned invalid query vector");
+    const lexical = options.lexical ?? (options.store.lexicalQuery ? "fts" : "off");
+    if (lexical !== "fts" && lexical !== "bm25" && lexical !== "off") {
+        throw new RagValidationError('lexical must be "fts", "bm25", or "off"');
     }
-    const candidates = await options.store.query({
-        tenantId: scope.tenantId,
-        resourceId: scope.resourceId,
-        threadId: scope.corpusId,
-        embedding,
-        topK: limits.queryCandidates,
-        signal: options.signal,
-    });
-    assertNotAborted(options.signal);
-    const retrievedAt = new Date().toISOString();
-    const retrieved = [];
-    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
-        assertScope(scope, candidate);
-        const parsed = parseHit(candidate, retrieved.length, retrievedAt);
-        if (!matchesFilter(parsed.metadata, options.filter))
-            continue;
-        retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
+    if (options.fusion !== undefined && options.fusion !== "rrf") {
+        throw new RagValidationError('fusion must be "rrf"');
+    }
+    if (lexical !== "off" && !options.store.lexicalQuery) {
+        throw new RagValidationError(`lexical "${lexical}" requested but the store has no lexicalQuery capability`);
+    }
+    if (lexical === "bm25" && !options.store.lexicalModes?.includes("bm25")) {
+        throw new RagValidationError('lexical "bm25" requested but the store does not declare BM25 support');
     }
-    const ranked = options.reranker
-        ? await rerankHits(safeQuery, retrieved, {
-            reranker: options.reranker,
-            maxBytes: limits.maxRerankBytes,
-            maxMs: limits.maxRerankMs,
-            concurrency: limits.rerankConcurrency,
-            signal: options.signal,
-            redactor: options.redactor,
-            secrets: options.secrets,
-        })
-        : retrieved;
-    const hits = [];
-    const citations = [];
-    const rendered = [];
-    const maxChars = limits.maxContextTokens * 4;
-    let usedBytes = 0;
-    let usedChars = 0;
-    let truncated = false;
-    for (const hit of ranked) {
-        if (hits.length >= limits.topK)
-            break;
-        const prefix = `[${hit.citationId}] `;
-        const separator = rendered.length ? "\n\n" : "";
-        const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
-        const availableChars = maxChars - usedChars - separator.length - prefix.length;
-        if (availableBytes <= 0 || availableChars <= 0) {
-            truncated = true;
-            break;
+    const useLexical = lexical !== "off";
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const telemetry = options.telemetry;
+    const root = telemetry?.startSpan("rag_request", {
+        ...(scopes[0] ? { "rag.scope.tenant_id": scopes[0].tenantId } : {}),
+        "rag.scope_count": scopes.length,
+        "rag.embedder_id": options.embedder.id,
+        "rag.top_k": limits.topK,
+        "rag.lexical_mode": lexical,
+    });
+    try {
+        const safeQuery = redactor?.redact(query) ?? query;
+        assertNotAborted(options.signal);
+        if (scopes.length === 0) {
+            return emptyResult(safeQuery);
+        }
+        if (scopes.length === 1) {
+            const currentGeneration = await options.store.getCurrentGeneration?.({
+                tenantId: scopes[0].tenantId,
+                resourceId: scopes[0].resourceId,
+                threadId: scopes[0].corpusId,
+            });
+            if (currentGeneration !== undefined)
+                root?.setAttribute("rag.index_generation", Number(currentGeneration));
+        }
+        const vectors = await span(telemetry, "embedding.query", undefined, root, () => options.embedder.embed([safeQuery], { signal: options.signal }));
+        const embedding = vectors[0];
+        if (vectors.length !== 1 ||
+            !embedding ||
+            embedding.length !== options.embedder.dimensions ||
+            embedding.some((value) => !Number.isFinite(value))) {
+            throw new RagValidationError("embedder returned invalid query vector");
         }
-        let text = hit.text.slice(0, availableChars);
-        text = truncateUtf8(text, availableBytes);
-        if (!text) {
-            truncated = true;
-            break;
+        const vectorLists = [];
+        await span(telemetry, "retrieval.vector_search", undefined, root, async (leg) => {
+            let total = 0;
+            for (const scope of scopes) {
+                assertNotAborted(options.signal);
+                const found = await options.store.query({
+                    tenantId: scope.tenantId,
+                    resourceId: scope.resourceId,
+                    threadId: scope.corpusId,
+                    embedding,
+                    topK: limits.queryCandidates,
+                    signal: options.signal,
+                });
+                const sliced = found.slice(0, limits.queryCandidates);
+                vectorLists.push(sliced);
+                total += sliced.length;
+            }
+            leg?.setAttribute("rag.vector_candidates", total);
+        });
+        const lexicalLists = [];
+        if (useLexical && options.store.lexicalQuery) {
+            await span(telemetry, "retrieval.lexical", undefined, root, async (leg) => {
+                let total = 0;
+                for (const scope of scopes) {
+                    assertNotAborted(options.signal);
+                    const found = await options.store.lexicalQuery({
+                        tenantId: scope.tenantId,
+                        resourceId: scope.resourceId,
+                        threadId: scope.corpusId,
+                        text: safeQuery,
+                        topK: limits.queryCandidates,
+                        signal: options.signal,
+                    });
+                    const sliced = found.slice(0, limits.queryCandidates);
+                    lexicalLists.push(sliced);
+                    total += sliced.length;
+                }
+                leg?.setAttribute("rag.lexical_candidates", total);
+            });
+        }
+        const retrievedAt = new Date().toISOString();
+        const retrieved = [];
+        const fused = await span(telemetry, "retrieval.fusion", undefined, root, (fusion) => {
+            const lists = [
+                ...vectorLists.map((hits) => ({ hits, leg: "vector" })),
+                ...lexicalLists.map((hits) => ({ hits, leg: "lexical" })),
+            ];
+            const fusedCandidates = fuseReciprocalRankLists(lists, limits.rrfK);
+            fusion?.setAttribute("rag.fused_candidates", fusedCandidates.length);
+            return fusedCandidates;
+        });
+        for (const { hit: candidate, retrieval } of fused) {
+            assertRequestedScope(scopes, candidate);
+            if (candidate.embedderId === undefined) {
+                throw new RagError(`stored record ${candidate.id} has no embedderId; re-index the source to stamp embedder identity`, "ERR_PRISM_RAG_EMBEDDER_MISMATCH");
+            }
+            if (candidate.embedderId !== options.embedder.id || candidate.embedding.length !== options.embedder.dimensions) {
+                throw new RagError(`embedder mismatch: record ${candidate.id} was embedded by "${candidate.embedderId}" (${candidate.embedding.length} dims) but the query embedder is "${options.embedder.id}" (${options.embedder.dimensions} dims)`, "ERR_PRISM_RAG_EMBEDDER_MISMATCH");
+            }
+            const parsed = parseHit(candidate, retrieved.length, retrievedAt, retrieval);
+            if (!matchesFilter(parsed.metadata, options.filter))
+                continue;
+            retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
+        }
+        const reranker = options.reranker;
+        const ranked = reranker
+            ? await span(telemetry, "retrieval.rerank", undefined, root, () => rerankHits(safeQuery, retrieved, {
+                reranker,
+                maxBytes: limits.maxRerankBytes,
+                maxMs: limits.maxRerankMs,
+                concurrency: limits.rerankConcurrency,
+                signal: options.signal,
+                redactor: options.redactor,
+                secrets: options.secrets,
+            }))
+            : retrieved;
+        const hits = [];
+        const citations = [];
+        const rendered = [];
+        const maxChars = limits.maxContextTokens * 4;
+        let usedBytes = 0;
+        let usedChars = 0;
+        let truncated = false;
+        const assemblySpan = telemetry?.startSpan("prompt.assembly", undefined, root);
+        for (const hit of ranked) {
+            if (hits.length >= limits.topK)
+                break;
+            const prefix = `[${hit.citationId}] `;
+            const separator = rendered.length ? "\n\n" : "";
+            const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
+            const availableChars = maxChars - usedChars - separator.length - prefix.length;
+            if (availableBytes <= 0 || availableChars <= 0) {
+                truncated = true;
+                break;
+            }
+            let text = hit.text.slice(0, availableChars);
+            text = truncateUtf8(text, availableBytes);
+            if (!text) {
+                truncated = true;
+                break;
+            }
+            if (text.length < hit.text.length)
+                truncated = true;
+            const renderedHit = Object.freeze({ ...hit, text });
+            const citation = Object.freeze({
+                id: renderedHit.citationId,
+                sourceId: renderedHit.sourceId,
+                chunkId: renderedHit.id,
+                provenance: renderedHit.provenance,
+                trust: renderedHit.trust,
+                ...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
+            });
+            const block = `${separator}${prefix}${text}`;
+            rendered.push(block);
+            usedBytes += byteLength(block);
+            usedChars += block.length;
+            hits.push(renderedHit);
+            citations.push(citation);
+            if (truncated)
+                break;
+        }
+        assemblySpan?.setAttribute("rag.result_count", hits.length);
+        assemblySpan?.end();
+        for (const hit of hits) {
+            root?.addEvent("chunk_retrieved", {
+                "rag.chunk.source_id": hit.sourceId,
+                "rag.chunk.id": hit.id,
+                "rag.chunk.rank": hit.retrievalRank,
+                "rag.chunk.score": hit.score,
+                "rag.chunk.embedder_id": options.embedder.id,
+                "rag.chunk.tenant_id": hit.provenance.tenantId,
+                "rag.chunk.corpus_id": hit.provenance.corpusId,
+            });
         }
-        if (text.length < hit.text.length)
-            truncated = true;
-        const renderedHit = Object.freeze({ ...hit, text });
-        const citation = Object.freeze({
-            id: renderedHit.citationId,
-            sourceId: renderedHit.sourceId,
-            chunkId: renderedHit.id,
-            provenance: renderedHit.provenance,
-            trust: renderedHit.trust,
-            ...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
+        return Object.freeze({
+            query: safeQuery,
+            trust: RETRIEVED_CONTENT_TRUST,
+            text: rendered.join(""),
+            hits: Object.freeze(hits),
+            citations: Object.freeze(citations),
+            truncated,
         });
-        const block = `${separator}${prefix}${text}`;
-        rendered.push(block);
-        usedBytes += byteLength(block);
-        usedChars += block.length;
-        hits.push(renderedHit);
-        citations.push(citation);
-        if (truncated)
-            break;
     }
-    return Object.freeze({
-        query: safeQuery,
-        trust: RETRIEVED_CONTENT_TRUST,
-        text: rendered.join(""),
-        hits: Object.freeze(hits),
-        citations: Object.freeze(citations),
-        truncated,
-    });
+    catch (error) {
+        root?.recordError();
+        throw error;
+    }
+    finally {
+        root?.end();
+    }
+}
+/** Opens a child span only when telemetry is present; otherwise runs the section untouched. */
+async function span(telemetry, name, attributes, parent, fn) {
+    if (!telemetry)
+        return await fn(undefined);
+    const child = telemetry.startSpan(name, attributes, parent);
+    try {
+        return await fn(child);
+    }
+    catch (error) {
+        child.recordError();
+        throw error;
+    }
+    finally {
+        child.end();
+    }
 }
-function parseHit(hit, retrievalRank, retrievedAt) {
+function parseHit(hit, retrievalRank, retrievedAt, retrieval) {
     const metadata = hit.metadata;
     const rag = metadata?._rag;
     if (!isJsonObject(rag))
@@ -144,7 +266,17 @@ function parseHit(hit, retrievalRank, retrievedAt) {
             userMetadata[key] = value;
     const web = isJsonObject(userMetadata.web) ? userMetadata.web : undefined;
     const provider = typeof web?.provider === "string" && web.provider.trim() ? web.provider : "host";
-    const provenance = Object.freeze({ sourceId, chunkId: hit.id, citationId, provider, retrieval: "vector", retrievedAt });
+    const provenance = Object.freeze({
+        sourceId,
+        chunkId: hit.id,
+        citationId,
+        provider,
+        tenantId: hit.tenantId,
+        resourceId: hit.resourceId,
+        corpusId: hit.threadId,
+        retrieval,
+        retrievedAt,
+    });
     return {
         id: hit.id,
         citationId,
@@ -160,4 +292,42 @@ function parseHit(hit, retrievalRank, retrievedAt) {
         ...(Object.keys(userMetadata).length ? { metadata: userMetadata } : {}),
     };
 }
+function resolveRetrieveScopes(options) {
+    const hasScope = options.scope !== undefined;
+    const hasScopes = options.scopes !== undefined;
+    if (hasScope && hasScopes)
+        throw new RagValidationError("provide either scope or scopes, not both");
+    if (!hasScope && !hasScopes)
+        throw new RagValidationError("scope or scopes is required");
+    const raw = hasScopes ? options.scopes : [options.scope];
+    if (raw.length > HARD_RETRIEVE_SCOPE_CAP)
+        throw new RagLimitError(`scopes exceeds hard cap ${HARD_RETRIEVE_SCOPE_CAP}`);
+    const seen = new Set();
+    const resolved = [];
+    for (const item of raw) {
+        const scope = requireScope(item);
+        const key = `${scope.tenantId}${scope.resourceId}${scope.corpusId}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        resolved.push(scope);
+    }
+    return resolved;
+}
+function assertRequestedScope(scopes, actual) {
+    if (scopes.some((scope) => scope.tenantId === actual.tenantId && scope.resourceId === actual.resourceId && scope.corpusId === actual.threadId)) {
+        return;
+    }
+    throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
+}
+function emptyResult(query) {
+    return Object.freeze({
+        query,
+        trust: RETRIEVED_CONTENT_TRUST,
+        text: "",
+        hits: Object.freeze([]),
+        citations: Object.freeze([]),
+        truncated: false,
+    });
+}
 //# sourceMappingURL=retrieve.js.map

package/dist/sources.d.ts

@@ -3,6 +3,8 @@ export interface SourceMutationResult {
     readonly sourceId: string;
     readonly deleted: number;
     readonly indexed: number;
+    /** Set when an unchanged document hash short-circuited the replace. */
+    readonly skipped?: true;
 }
 export declare function replaceSource(options: ReplaceSourceOptions): Promise<SourceMutationResult>;
 export declare function deleteSource(options: DeleteSourceOptions): Promise<SourceMutationResult>;

package/dist/sources.js

@@ -1,9 +1,14 @@
 import { resolveRedactor } from "@arnilo/prism";
 import { chunkText } from "./chunk.js";
 import { RagScopeError, RagValidationError } from "./errors.js";
+import { isValidContentHash } from "./hash.js";
 import { indexChunkBatches } from "./indexing.js";
 import { ingestionStatus } from "./ingestion-status.js";
 import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
+function storedDocHash(record) {
+    const value = record.metadata?._rag?.contentHash;
+    return isValidContentHash(value) ? value : undefined;
+}
 export async function replaceSource(options) {
     const sourceId = requireSourceId(options.sourceId);
     const scope = requireScope(options.scope);
@@ -12,6 +17,10 @@ export async function replaceSource(options) {
         throw new RagValidationError("replaceSource chunks must all belong to sourceId");
     }
     const redactor = resolveRedactor(options.redactor, options.secrets);
+    if (options.contentHash !== undefined && !isValidContentHash(options.contentHash)) {
+        throw new RagValidationError("contentHash must be a hex digest of 32..128 characters");
+    }
+    const contentHash = options.contentHash?.toLowerCase();
     const totalBytes = options.chunks.reduce((total, chunk) => total + byteLength(redactor?.redact(chunk.text) ?? chunk.text), 0);
     const setStatus = async (state, error) => {
         if (!options.statusStore)
@@ -20,29 +29,82 @@ export async function replaceSource(options) {
         await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? (redactor?.redact(message) ?? message) : undefined));
     };
     await setStatus("pending");
+    const telemetry = options.telemetry;
+    const root = telemetry?.startSpan("rag_index", {
+        "rag.scope.tenant_id": scope.tenantId,
+        "rag.source_id": sourceId,
+        "rag.embedder_id": options.embedder.id,
+        "rag.chunk_count": options.chunks.length,
+    });
     try {
+        // One read decides the skip; unchanged sources cost zero embeds and zero writes.
+        const previous = await sourceRecords(options.store, sourceId, scope, options.signal);
+        if (contentHash &&
+            options.skipIfUnchanged !== false &&
+            previous.length > 0 &&
+            previous.every((record) => storedDocHash(record) === contentHash)) {
+            // Incoming stats describe the now-live content even though nothing was rewritten.
+            await setStatus("indexed", undefined);
+            return Object.freeze({ sourceId, deleted: 0, indexed: 0, skipped: true });
+        }
+        const reuseEmbeddings = new Map();
+        if (options.skipIfUnchanged !== false) {
+            // skipIfUnchanged: false means rebuild everything — no embedding reuse either.
+            for (const record of previous) {
+                if (record.embedderId === options.embedder.id) {
+                    reuseEmbeddings.set(record.id, { text: record.text, embedding: record.embedding });
+                }
+            }
+        }
         const staged = [];
-        const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
+        const indexed = await indexChunkBatches({ ...options, statusStore: undefined, contentHash, reuseEmbeddings, telemetry, telemetryParent: root }, async (records) => {
             staged.push(...records);
         });
         assertNotAborted(options.signal);
         const result = await options.store.transaction(async (store) => {
+            // Generation visibility: stamp chunks at N+1 and advance the scope pointer in the
+            // same transaction as the swap. Stores without generation tracking keep legacy behavior.
+            const getCurrent = store.getCurrentGeneration?.bind(store);
+            const setCurrent = store.setCurrentGeneration?.bind(store);
+            let nextGeneration;
+            if (getCurrent && setCurrent) {
+                const current = await getCurrent({
+                    tenantId: scope.tenantId,
+                    resourceId: scope.resourceId,
+                    threadId: scope.corpusId,
+                });
+                nextGeneration = (current === undefined ? 0 : Number(current)) + 1;
+                root?.setAttribute("rag.index_generation", nextGeneration);
+            }
             const previous = await sourceRecords(store, sourceId, scope, options.signal);
             assertNotAborted(options.signal);
             if (previous.length) {
                 await store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: previous.map((record) => record.id) }, { signal: options.signal });
             }
-            if (staged.length)
-                await store.upsert(staged, { signal: options.signal });
+            if (staged.length) {
+                const stamped = nextGeneration === undefined ? staged : staged.map((record) => ({ ...record, generation: nextGeneration }));
+                await store.upsert(stamped, { signal: options.signal });
+            }
+            if (setCurrent && nextGeneration !== undefined) {
+                await setCurrent({
+                    tenantId: scope.tenantId,
+                    resourceId: scope.resourceId,
+                    threadId: scope.corpusId,
+                }, nextGeneration);
+            }
             return Object.freeze({ sourceId, deleted: previous.length, indexed: indexed.indexed });
         }, { signal: options.signal });
         await setStatus("indexed");
         return result;
     }
     catch (error) {
+        root?.recordError();
         await setStatus("failed", error);
         throw error;
     }
+    finally {
+        root?.end();
+    }
 }
 export async function deleteSource(options) {
     const sourceId = requireSourceId(options.sourceId);

package/dist/tei-reranker.d.ts

@@ -0,0 +1,35 @@
+import { type SsrfPolicy } from "@arnilo/prism";
+import type { Reranker } from "./types.js";
+/**
+ * Hugging Face TEI rerank adapter (plan 034 Task 8 / request P8):
+ * `POST <baseUrl>/rerank` with `{query, texts, raw_scores:false}` →
+ * `{results: [{index, score}]}` mapped to a permutation-only reorder of the
+ * provided `RagHit[]` (same object references — provenance/trust untouched).
+ *
+ * - URL shape validated at construction (absolute, http/https, no embedded
+ *   credentials or fragment); SSRF/enforcement is host-side via `ssrf`,
+ *   `allowLoopback`, or an injected `fetch`. Default transport is the core
+ *   `pinnedFetch` primitive (DNS-pinned, redirect-free, byte-bounded).
+ * - Out-of-range/duplicate/missing indices, non-finite scores, HTTP errors,
+ *   timeouts, and oversized bodies all fail closed in the rerank error
+ *   family. Seam caps (`maxRerankBytes`, `maxRerankMs`, `rerankConcurrency`)
+ *   stay enforced by `rerankHits` around this adapter.
+ * - No credentials, no SaaS default URL.
+ */
+export interface CreateTeiRerankerOptions {
+    /** Base URL of the TEI service, e.g. `http://tei.svc:8080`. `/rerank` is appended. */
+    readonly baseUrl: string;
+    /** Optional model name sent in the rerank body. */
+    readonly model?: string;
+    /** Per-call timeout combined with the caller signal; aborts fail closed. */
+    readonly timeoutMs?: number;
+    /** SSRF policy applied on resolved hosts (default: core default). */
+    readonly ssrf?: SsrfPolicy;
+    /** Allow loopback destinations (local/dev TEI). Default `false`. */
+    readonly allowLoopback?: boolean;
+    /** Maximum response body bytes. Default 65,536 (plan 021 ceiling precedent). */
+    readonly maxResponseBytes?: number;
+    /** Trusted custom transport; host owns DNS/Bonding protection (OPA precedent). */
+    readonly fetch?: typeof globalThis.fetch;
+}
+export declare function createTeiReranker(options: CreateTeiRerankerOptions): Reranker;

package/dist/tei-reranker.js

@@ -0,0 +1,138 @@
+import { MediaContentError, pinnedFetch } from "@arnilo/prism";
+import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
+const DEFAULT_TIMEOUT_MS = 2000;
+const DEFAULT_MAX_RESPONSE_BYTES = 64 * 1024;
+export function createTeiReranker(options) {
+    const { baseUrl, model, timeoutMs = DEFAULT_TIMEOUT_MS, maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES, ssrf, allowLoopback } = options;
+    if (!baseUrl.trim())
+        throw new RagValidationError("TEI reranker baseUrl is required");
+    let url;
+    try {
+        url = new URL(baseUrl.trim());
+    }
+    catch {
+        throw new RagValidationError("TEI reranker baseUrl must be an absolute URL");
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new RagValidationError(`TEI reranker baseUrl must use http(s) (got ${url.protocol})`);
+    }
+    if (url.username || url.password)
+        throw new RagValidationError("TEI reranker baseUrl must not embed credentials");
+    if (url.hash)
+        throw new RagValidationError("TEI reranker baseUrl must not contain a fragment");
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)
+        throw new RagValidationError("TEI reranker timeoutMs must be a positive integer");
+    if (!Number.isSafeInteger(maxResponseBytes) || maxResponseBytes < 1) {
+        throw new RagValidationError("TEI reranker maxResponseBytes must be a positive integer");
+    }
+    // Join without doubling a trailing slash.
+    const endpoint = `${url.toString().replace(/\/+$/, "")}/rerank`;
+    const pinnedRerankFetch = (input, init) => pinnedFetch(input instanceof URL ? input : new URL(String(input)), init, {
+        errorPrefix: "TEI rerank",
+        hostnameErrorPrefix: "TEI rerank",
+        ssrf,
+        allowLoopback,
+        maxResponseBytes,
+    });
+    const transport = options.fetch ?? pinnedRerankFetch;
+    return {
+        async rerank({ query, hits, signal }) {
+            const payload = JSON.stringify({ query, texts: hits.map((hit) => hit.text), raw_scores: false, ...(model ? { model } : {}) });
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeoutMs);
+            const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+            let response;
+            try {
+                response = await transport(endpoint, {
+                    method: "POST",
+                    headers: { "content-type": "application/json", accept: "application/json" },
+                    body: payload,
+                    signal: combined,
+                    redirect: "manual",
+                });
+            }
+            catch (error) {
+                if (error instanceof MediaContentError) {
+                    throw new RagValidationError(`TEI rerank request denied: ${error.message}`);
+                }
+                if (signal?.aborted)
+                    throw new RagAbortError();
+                throw new RagLimitError(`TEI rerank exceeded ${timeoutMs}ms or failed to reach ${url.host}`);
+            }
+            finally {
+                clearTimeout(timer);
+            }
+            if (!response.ok)
+                throw new RagValidationError(`TEI rerank endpoint returned HTTP ${response.status}`);
+            const text = await readBoundedBody(response, maxResponseBytes);
+            let parsed;
+            try {
+                parsed = JSON.parse(text);
+            }
+            catch {
+                throw new RagValidationError("TEI rerank response is not valid JSON");
+            }
+            return orderByScores(hits, parsed);
+        },
+    };
+}
+/** Strictly parse `{results: [{index, score}]}` and reorder hits by score desc. */
+function orderByScores(hits, parsed) {
+    if (typeof parsed !== "object" || parsed === null || !Array.isArray(parsed.results)) {
+        throw new RagValidationError("TEI rerank response missing results array");
+    }
+    const results = parsed.results;
+    if (results.length !== hits.length) {
+        throw new RagValidationError(`TEI rerank returned ${results.length} scores for ${hits.length} hits`);
+    }
+    const scores = new Array(hits.length);
+    const seen = new Set();
+    for (const item of results) {
+        if (typeof item !== "object" || item === null)
+            throw new RagValidationError("TEI rerank result must be an object");
+        const index = item.index;
+        const score = item.score;
+        if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= hits.length) {
+            throw new RagValidationError("TEI rerank returned an out-of-range index");
+        }
+        if (seen.has(index))
+            throw new RagValidationError("TEI rerank returned a duplicate index");
+        if (typeof score !== "number" || !Number.isFinite(score))
+            throw new RagValidationError("TEI rerank returned a non-finite score");
+        seen.add(index);
+        scores[index] = score;
+    }
+    // Stable sort by descending score; ties keep original array order.
+    const ordered = hits
+        .map((hit, i) => ({ hit, score: scores[i] }))
+        .sort((a, b) => b.score - a.score)
+        .map((entry) => entry.hit);
+    return Object.freeze(ordered);
+}
+async function readBoundedBody(response, maxBytes) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return "";
+    const decoder = new TextDecoder();
+    let out = "";
+    let total = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            total += value.byteLength;
+            if (total > maxBytes)
+                throw new RagLimitError(`TEI rerank response exceeds ${maxBytes} bytes`);
+            out += decoder.decode(value, { stream: true });
+        }
+        return out + decoder.decode();
+    }
+    catch (error) {
+        // The pinned transport may already have errorred the stream (MediaContentError).
+        if (error instanceof RagLimitError)
+            throw error;
+        throw new RagLimitError(`TEI rerank response exceeds ${maxBytes} bytes`);
+    }
+}
+//# sourceMappingURL=tei-reranker.js.map

package/dist/telemetry.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Dependency-free telemetry seam for RAG operations. When `telemetry` is omitted every
+ * call site short-circuits on an optional chain — zero allocations, zero per-span work.
+ * Implementations own timing/status; the seam only carries structure and attributes.
+ */
+export type RagTelemetryAttributeValue = string | number | boolean;
+export interface RagTelemetrySpan {
+    setAttribute(name: string, value: RagTelemetryAttributeValue): void;
+    addEvent(name: string, attributes?: Readonly<Record<string, RagTelemetryAttributeValue>>): void;
+    /** Flags the span as failed without carrying any error text (no user data leaves the host). */
+    recordError(): void;
+    end(): void;
+}
+export interface RagTelemetry {
+    startSpan(name: string, attributes?: Readonly<Record<string, RagTelemetryAttributeValue>>, parent?: RagTelemetrySpan): RagTelemetrySpan;
+}

package/dist/telemetry.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=telemetry.js.map

package/dist/types.d.ts

@@ -1,5 +1,6 @@
 import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
 import type { Embedder, MemoryVectorRecord, VectorStore } from "@arnilo/prism-memory";
+import type { RagTelemetry, RagTelemetrySpan } from "./telemetry.js";
 export interface RagScope {
     readonly tenantId: string;
     readonly resourceId: string;
@@ -15,6 +16,11 @@ export interface RagChunk {
     readonly text: string;
     readonly metadata?: JsonObject;
 }
+/** Stored embedding of a previous record offered for reuse when its text is unchanged. */
+export interface ReusableEmbedding {
+    readonly text: string;
+    readonly embedding: readonly number[];
+}
 export interface ChunkOptions {
     readonly sourceId: string;
     readonly metadata?: JsonObject;
@@ -111,6 +117,13 @@ export interface IndexChunksOptions {
     readonly secrets?: readonly (string | undefined)[];
     readonly statusStore?: IngestionStatusStore;
     readonly signal?: AbortSignal;
+    /** Host-supplied document digest stamped into each record's `_rag.contentHash`. */
+    readonly contentHash?: string;
+    /** Chunk id → previous text+embedding; embeddings are reused (no embed call) when texts match. */
+    readonly reuseEmbeddings?: ReadonlyMap<string, ReusableEmbedding>;
+    /** Optional telemetry seam; spans nest under `telemetryParent` when supplied. */
+    readonly telemetry?: RagTelemetry;
+    readonly telemetryParent?: RagTelemetrySpan;
 }
 export interface IndexChunksResult {
     readonly indexed: number;
@@ -120,6 +133,10 @@ export interface ReplaceSourceOptions extends Omit<IndexChunksOptions, "chunks"
     readonly sourceId: string;
     readonly chunks: readonly RagChunk[];
     readonly store: TransactionalVectorStore;
+    /** Host-computed document digest; enables the unchanged-source skip. */
+    readonly contentHash?: string;
+    /** Skip re-indexing when the stored document hash matches. Default true when contentHash is present. */
+    readonly skipIfUnchanged?: boolean;
 }
 export interface DeleteSourceOptions {
     readonly sourceId: string;
@@ -143,7 +160,11 @@ export interface RagProvenance {
     readonly chunkId: string;
     readonly citationId: string;
     readonly provider: string;
-    readonly retrieval: "vector";
+    readonly tenantId: string;
+    readonly resourceId: string;
+    readonly corpusId: string;
+    /** Which retrieval leg(s) surfaced this hit after RRF fusion. */
+    readonly retrieval: "vector" | "lexical" | "hybrid";
     readonly retrievedAt: string;
 }
 export interface RagContentTrust {
@@ -175,9 +196,21 @@ export interface Reranker {
 export interface RetrieveContextOptions {
     readonly embedder: Embedder;
     readonly store: VectorStore;
-    readonly scope: RagScope;
+    /** Single exact scope. Provide `scope` or `scopes`, never both. */
+    readonly scope?: RagScope;
+    /** One or more exact scopes (empty = no hits, no embed). */
+    readonly scopes?: readonly RagScope[];
+    /** Optional telemetry seam; when omitted, instrumentation costs nothing. */
+    readonly telemetry?: RagTelemetry;
     readonly topK?: number;
     readonly queryCandidates?: number;
+    /** Lexical leg mode. Default runs fts when the store supports it and silently skips otherwise;
+     * explicitly requesting "fts"/"bm25" on an unsupported store fails closed. */
+    readonly lexical?: "fts" | "bm25" | "off";
+    /** Only reciprocal-rank fusion is supported. */
+    readonly fusion?: "rrf";
+    /** RRF smoothing constant (default 60, capped). */
+    readonly rrfK?: number;
     readonly filter?: JsonObject;
     readonly maxResultBytes?: number;
     readonly maxContextTokens?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arnilo/prism-rag",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Optional bounded text and Markdown RAG primitives for Prism.",
   "type": "module",
   "main": "./dist/index.js",