@arnilo/prism-rag 0.0.14 → 0.0.15

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [0.0.15] - 2026-07-26
+
+### Added
+
+- Added atomic scoped `replaceSource()` / `deleteSource()` and `replaceDocument()` over source-aware transactional vector storage; the in-memory reference store implements `getBySource()` plus transactions.
+- Added bounded `DocumentLoader` / `Parser` seams, reference text/Markdown/HTML/PDF parsers, and host resource/web-tools document loaders with citation/trust preservation.
+- Added host-owned bounded reranking, canonical citation provenance/content-trust metadata, and scoped capped ingestion-status storage/querying with a process-local reference adapter.
+
 ## [0.0.14] - 2026-07-26
 
 ### Changed

package/README.md

@@ -1,6 +1,6 @@
 # @arnilo/prism-rag
 
-Optional bounded text/Markdown retrieval-augmented generation primitives for Prism. Reuses `Embedder` and `VectorStore` from `@arnilo/prism-memory`; no document framework, network loader, or core activation.
+Optional bounded source lifecycle, document parsing, host reranking, ingestion status, and retrieval-augmented generation primitives for Prism. Reuses `Embedder` and `VectorStore` from `@arnilo/prism-memory`; no document framework, network loader, or core activation.
 
 ## Install
 
@@ -12,7 +12,7 @@ npm install @arnilo/prism-rag @arnilo/prism-memory @arnilo/prism
 
 ```ts
 import { createHashEmbedder, createMemoryVectorStore } from "@arnilo/prism-memory";
-import { chunkMarkdown, createRagContextProvider, indexChunks } from "@arnilo/prism-rag";
+import { chunkMarkdown, createMemoryIngestionStatusStore, createRagContextProvider, listIngestionStatus, replaceSource } from "@arnilo/prism-rag";
 
 const embedder = createHashEmbedder(); // demo/test only
 const store = createMemoryVectorStore();
@@ -20,7 +20,9 @@ const scope = { tenantId: "t1", resourceId: "docs", corpusId: "handbook" };
 const chunks = chunkMarkdown("# Approval\n\nRecheck policy before side effects.", {
   sourceId: "security-guide",
 });
-await indexChunks({ chunks, embedder, store, scope });
+const statusStore = createMemoryIngestionStatusStore();
+await replaceSource({ sourceId: "security-guide", chunks, embedder, store, scope, statusStore }); // atomic with the reference store
+console.log(await listIngestionStatus({ store: statusStore, scope }));
 const context = createRagContextProvider({ embedder, store, scope, topK: 4 });
 ```
 
@@ -28,11 +30,15 @@ const context = createRagContextProvider({ embedder, store, scope, topK: 4 });
 
 - `chunkText()` / `chunkMarkdown()` — deterministic boundary-aware character chunks with overlap and stable citations.
 - `indexChunks()` — bounded batch embedding and scoped vector upsert.
-- `retrieveContext()` — bounded candidate query, shallow metadata filter, top-K hits, and citation rendering.
+- `replaceSource()` / `deleteSource()` — exact-scope source lifecycle; replacement requires transactional `getBySource()` storage.
+- `replaceDocument()` + `DocumentLoader` / `Parser` — host-authorized load, bounded parse, chunk, and replacement.
+- `textParser` / `markdownParser` / `htmlParser` / `pdfParser` — focused reference parsers (`./parsers`); `createResourceDocumentLoader` / `createWebFetchDocumentLoader` are in `./loaders`.
+- `retrieveContext()` — bounded candidate query, optional host `Reranker`, shallow metadata filter, top-K hits, attributable citations, and untrusted/inert/injection-capable trust metadata.
+- `createMemoryIngestionStatusStore()` / `listIngestionStatus()` — optional capped exact-scope pending/indexed/failed/partial source progress; implement `IngestionStatusStore` for durable status.
 - `createRagContextProvider()` — explicit inert context injection through Prism's existing seam.
 
 ## Security
 
-Every operation requires tenant/resource/corpus scope. Configure `redactor` or `secrets` before external embedding/persistence. Package performs no I/O; load remote/local sources through host-owned resource/media policies. Retrieved text is untrusted context and grants no tools or permissions.
+Every operation requires tenant/resource/corpus scope. Configure `redactor` or `secrets` before external embedding/persistence. Package performs no I/O: resource loading delegates to a host `ResourceLoader`, and web loading delegates to an existing web-tools adapter. Replacement fails closed without a scoped transaction; rerankers get redacted finite candidates and cannot alter canonical provenance/trust; HTML/PDF/web output is untrusted inert context and grants no tools or permissions.
 
 See [RAG](../../docs/rag.md).

package/dist/context.js

@@ -17,7 +17,7 @@ export function createRagContextProvider(options) {
                     id: `${options.name ?? "rag"}:context`,
                     title: options.title ?? "Retrieved context",
                     content: result.text,
-                    metadata: { citations: result.citations, inert: true },
+                    metadata: { citations: result.citations, trust: result.trust, inert: true, untrusted: true, injectionCapable: true },
                 }];
         },
     };

package/dist/index.d.ts

@@ -1,9 +1,14 @@
-export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
 export type { RagLimits, RagLimitsInput } from "./limits.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { chunkMarkdown, chunkText } from "./chunk.js";
+export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
+export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { indexChunks } from "./indexing.js";
+export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
+export type { SourceMutationResult } from "./sources.js";
+export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
 export { retrieveContext } from "./retrieve.js";
 export { createRagContextProvider } from "./context.js";
-export type { ChunkOptions, IndexChunksOptions, IndexChunksResult, RagChunk, RagCitation, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagScope, RetrieveContextOptions, } from "./types.js";
+export type { ChunkOptions, Chunker, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, RagChunk, LoadedDocument, ParsedDocument, Parser, RagCitation, RagContextProvider, RagContextProviderOptions, RagContextResult, RagContentTrust, RagHit, RagProvenance, Reranker, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
 export declare const packageName = "@arnilo/prism-rag";

package/dist/index.js

@@ -1,7 +1,11 @@
-export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
+export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
 export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { chunkMarkdown, chunkText } from "./chunk.js";
+export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
+export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { indexChunks } from "./indexing.js";
+export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
+export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
 export { retrieveContext } from "./retrieve.js";
 export { createRagContextProvider } from "./context.js";
 export const packageName = "@arnilo/prism-rag";

package/dist/indexing.d.ts

@@ -1,2 +1,4 @@
+import type { MemoryVectorRecord } from "@arnilo/prism-memory";
 import type { IndexChunksOptions, IndexChunksResult } from "./types.js";
 export declare function indexChunks(options: IndexChunksOptions): Promise<IndexChunksResult>;
+export declare function indexChunkBatches(options: IndexChunksOptions, write: (records: readonly MemoryVectorRecord[]) => Promise<void>): Promise<IndexChunksResult>;

package/dist/indexing.js

@@ -1,9 +1,11 @@
-import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
 import { RagValidationError } from "./errors.js";
-import { assertBytes, assertNotAborted, nonEmpty, requireScope, requireSourceId, resolveRedactor } from "./util.js";
-// ponytail: stable IDs make identical retries idempotent; replacing a source with
-// fewer chunks requires host deletion of stale source IDs before this generic upsert.
+import { ingestionStatus } from "./ingestion-status.js";
+import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId, resolveRedactor } from "./util.js";
 export async function indexChunks(options) {
+    return indexChunkBatches(options, async (records) => options.store.upsert(records, { signal: options.signal }));
+}
+export async function indexChunkBatches(options, write) {
     const scope = requireScope(options.scope);
     const limits = resolveRagLimits({
         embedBatchSize: options.batchSize,
@@ -12,78 +14,87 @@ export async function indexChunks(options) {
         maxVectorDimensions: options.maxVectorDimensions,
         maxMetadataBytes: options.maxMetadataBytes,
     });
-    if (!Number.isInteger(options.embedder.dimensions)
-        || options.embedder.dimensions <= 0
-        || options.embedder.dimensions > limits.maxVectorDimensions) {
+    if (!Number.isInteger(options.embedder.dimensions) || options.embedder.dimensions <= 0 || options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
-    if (options.chunks.length > limits.maxChunks) {
+    if (options.chunks.length > limits.maxChunks)
         throw new RagValidationError(`chunk count exceeds ${limits.maxChunks}`);
-    }
     const redactor = resolveRedactor(options.redactor, options.secrets);
     const sourceIds = new Set();
     const chunkIds = new Set();
+    const total = new Map();
+    const written = new Map();
     for (const chunk of options.chunks) {
         nonEmpty(chunk.id, "chunk.id");
         requireSourceId(chunk.sourceId);
-        if (chunk.id !== chunk.citationId || !chunk.citationId.startsWith(`${chunk.sourceId}#`)) {
+        if (chunk.id !== chunk.citationId || !chunk.citationId.startsWith(`${chunk.sourceId}#`))
             throw new RagValidationError("chunk has inconsistent citation identity");
-        }
         if (chunkIds.has(chunk.id))
             throw new RagValidationError(`duplicate chunk id: ${chunk.id}`);
-        if (!Number.isInteger(chunk.index)
-            || chunk.index < 0
-            || !Number.isInteger(chunk.start)
-            || chunk.start < 0
-            || !Number.isInteger(chunk.end)
-            || chunk.end < chunk.start) {
+        if (!Number.isInteger(chunk.index) || chunk.index < 0 || !Number.isInteger(chunk.start) || chunk.start < 0 || !Number.isInteger(chunk.end) || chunk.end < chunk.start) {
             throw new RagValidationError("chunk has invalid index or offsets");
         }
         if (chunk.text.length > limits.chunkSize)
             throw new RagValidationError(`chunk text exceeds ${limits.chunkSize} characters`);
         chunkIds.add(chunk.id);
+        sourceIds.add(chunk.sourceId);
+        add(total, chunk.sourceId, byteLength(redactor?.redact(chunk.text) ?? chunk.text), 1);
     }
-    for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
-        assertNotAborted(options.signal);
-        const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
-        const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
-        const vectors = await options.embedder.embed(texts, { signal: options.signal });
-        if (vectors.length !== batch.length)
-            throw new RagValidationError("embedder returned unexpected vector count");
-        const records = batch.map((chunk, index) => {
-            const embedding = vectors[index];
-            if (embedding.length !== options.embedder.dimensions
-                || embedding.some((value) => !Number.isFinite(value))) {
-                throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
-            }
-            const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? (chunk.metadata ?? {});
-            const metadata = {
-                ...safeMetadata,
-                _rag: {
-                    sourceId: chunk.sourceId,
-                    citationId: chunk.citationId,
-                    chunkIndex: chunk.index,
-                    start: chunk.start,
-                    end: chunk.end,
-                },
-            };
-            assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
-            sourceIds.add(chunk.sourceId);
-            return {
-                id: chunk.id,
-                tenantId: scope.tenantId,
-                resourceId: scope.resourceId,
-                threadId: scope.corpusId,
-                text: texts[index],
-                embedding,
-                sequence: chunk.index,
-                metadata: metadata,
-                createdAt: new Date(0).toISOString(),
-            };
-        });
-        assertNotAborted(options.signal);
-        await options.store.upsert(records, { signal: options.signal });
+    const setStatus = async (state, error) => {
+        if (!options.statusStore)
+            return;
+        const message = error instanceof Error ? error.message : error === undefined ? undefined : "indexing failed";
+        const safeError = message ? redactor?.redact(message) ?? message : undefined;
+        for (const sourceId of sourceIds) {
+            const progress = state === "indexed" ? total.get(sourceId) : written.get(sourceId) ?? { bytes: 0, chunks: 0 };
+            await options.statusStore.set(ingestionStatus(scope, sourceId, state, progress.bytes, progress.chunks, safeError));
+        }
+    };
+    await setStatus("pending");
+    try {
+        for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
+            assertNotAborted(options.signal);
+            const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
+            const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
+            const vectors = await options.embedder.embed(texts, { signal: options.signal });
+            if (vectors.length !== batch.length)
+                throw new RagValidationError("embedder returned unexpected vector count");
+            const records = batch.map((chunk, index) => {
+                const embedding = vectors[index];
+                if (embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
+                    throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
+                }
+                const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? (chunk.metadata ?? {});
+                const metadata = { ...safeMetadata, _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end } };
+                assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
+                return {
+                    id: chunk.id,
+                    tenantId: scope.tenantId,
+                    resourceId: scope.resourceId,
+                    threadId: scope.corpusId,
+                    text: texts[index],
+                    embedding,
+                    sequence: chunk.index,
+                    metadata: metadata,
+                    createdAt: new Date(0).toISOString(),
+                };
+            });
+            assertNotAborted(options.signal);
+            await write(records);
+            for (const chunk of batch)
+                add(written, chunk.sourceId, byteLength(redactor?.redact(chunk.text) ?? chunk.text), 1);
+            await setStatus("pending");
+        }
+        await setStatus("indexed");
+    }
+    catch (error) {
+        await setStatus([...written.values()].some((progress) => progress.chunks > 0) ? "partial" : "failed", error);
+        throw error;
     }
     return Object.freeze({ indexed: options.chunks.length, sourceIds: Object.freeze([...sourceIds].sort()) });
 }
+function add(target, sourceId, bytes, chunks) {
+    const current = target.get(sourceId) ?? { bytes: 0, chunks: 0 };
+    target.set(sourceId, { bytes: current.bytes + bytes, chunks: current.chunks + chunks });
+}
 //# sourceMappingURL=indexing.js.map

package/dist/ingestion-status.d.ts

@@ -0,0 +1,7 @@
+import type { IngestionStatus, IngestionStatusQuery, IngestionStatusStore, RagScope } from "./types.js";
+export declare function listIngestionStatus(options: IngestionStatusQuery): Promise<{
+    readonly entries: readonly IngestionStatus[];
+    readonly nextCursor?: string;
+}>;
+export declare function createMemoryIngestionStatusStore(): IngestionStatusStore;
+export declare function ingestionStatus(scope: RagScope, sourceId: string, state: IngestionStatus["state"], bytes: number, chunks: number, error?: string): IngestionStatus;

package/dist/ingestion-status.js

@@ -0,0 +1,85 @@
+import { RagScopeError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
+export async function listIngestionStatus(options) {
+    const scope = requireScope(options.scope);
+    const limit = resolveRagLimits({ ingestionStatusPageSize: options.limit }).ingestionStatusPageSize;
+    if (options.cursor !== undefined)
+        requireSourceId(options.cursor);
+    assertNotAborted(options.signal);
+    const page = await options.store.list(scope, { limit, cursor: options.cursor, signal: options.signal });
+    if (page.entries.length > limit)
+        throw new RagValidationError("ingestion status store exceeded requested page limit");
+    for (const entry of page.entries)
+        assertStatus(entry, scope);
+    if (page.nextCursor !== undefined)
+        requireSourceId(page.nextCursor);
+    return Object.freeze({ entries: Object.freeze([...page.entries]), ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}) });
+}
+export function createMemoryIngestionStatusStore() {
+    const scopes = new Map();
+    return {
+        async set(status, options = {}) {
+            assertNotAborted(options.signal);
+            const scope = requireScope(status.scope);
+            assertStatus(status, scope);
+            const key = scopeKey(scope);
+            const entries = scopes.get(key) ?? new Map();
+            entries.set(status.sourceId, Object.freeze({ ...status, scope }));
+            scopes.set(key, entries);
+        },
+        async delete(scope, sourceId, options = {}) {
+            assertNotAborted(options.signal);
+            const entries = scopes.get(scopeKey(requireScope(scope)));
+            entries?.delete(requireSourceId(sourceId));
+        },
+        async list(scope, options) {
+            assertNotAborted(options.signal);
+            const required = requireScope(scope);
+            const limit = options.limit;
+            if (!Number.isInteger(limit) || limit < 1)
+                throw new RagValidationError("ingestion status limit must be a positive integer");
+            if (options.cursor !== undefined)
+                requireSourceId(options.cursor);
+            const entries = [...(scopes.get(scopeKey(required))?.values() ?? [])]
+                .filter((entry) => entry.sourceId > (options.cursor ?? ""))
+                .sort((left, right) => left.sourceId.localeCompare(right.sourceId));
+            const page = entries.slice(0, limit);
+            const next = entries.length > page.length ? page.at(-1)?.sourceId : undefined;
+            return Object.freeze({ entries: Object.freeze(page), ...(next ? { nextCursor: next } : {}) });
+        },
+    };
+}
+export function ingestionStatus(scope, sourceId, state, bytes, chunks, error) {
+    const status = {
+        sourceId: requireSourceId(sourceId),
+        scope: requireScope(scope),
+        state,
+        bytes,
+        chunks,
+        ...(error ? { error } : {}),
+        updatedAt: new Date().toISOString(),
+    };
+    assertStatus(status, status.scope);
+    return Object.freeze(status);
+}
+function assertStatus(status, scope) {
+    requireSourceId(status.sourceId);
+    if (status.scope.tenantId !== scope.tenantId || status.scope.resourceId !== scope.resourceId || status.scope.corpusId !== scope.corpusId) {
+        throw new RagScopeError("ingestion status crossed tenant/resource/corpus boundary");
+    }
+    if (!["pending", "indexed", "failed", "partial"].includes(status.state))
+        throw new RagValidationError("ingestion status state is invalid");
+    if (!Number.isSafeInteger(status.bytes) || status.bytes < 0 || !Number.isSafeInteger(status.chunks) || status.chunks < 0) {
+        throw new RagValidationError("ingestion status bytes and chunks must be non-negative safe integers");
+    }
+    if (!Number.isFinite(Date.parse(status.updatedAt)))
+        throw new RagValidationError("ingestion status updatedAt must be an ISO timestamp");
+    if (status.error !== undefined && (typeof status.error !== "string" || byteLength(status.error) > 4_096)) {
+        throw new RagValidationError("ingestion status error must be a string <= 4096 bytes");
+    }
+}
+function scopeKey(scope) {
+    return `${scope.tenantId}\0${scope.resourceId}\0${scope.corpusId}`;
+}
+//# sourceMappingURL=ingestion-status.js.map

package/dist/limits.d.ts

@@ -4,6 +4,12 @@ export declare const DEFAULT_CHUNK_OVERLAP = 100;
 export declare const HARD_CHUNK_OVERLAP_CAP = 4096;
 export declare const DEFAULT_MAX_DOCUMENT_CHARS = 1048576;
 export declare const HARD_MAX_DOCUMENT_CHARS_CAP = 8388608;
+export declare const DEFAULT_MAX_DOCUMENT_BYTES = 1048576;
+export declare const HARD_MAX_DOCUMENT_BYTES_CAP: number;
+export declare const DEFAULT_MAX_PARSE_MS = 30000;
+export declare const HARD_MAX_PARSE_MS_CAP = 30000;
+export declare const DEFAULT_MAX_PDF_PAGES = 256;
+export declare const HARD_MAX_PDF_PAGES_CAP = 256;
 export declare const DEFAULT_MAX_CHUNKS = 2048;
 export declare const HARD_MAX_CHUNKS_CAP = 8192;
 export declare const DEFAULT_EMBED_BATCH_SIZE = 32;
@@ -19,10 +25,21 @@ export declare const HARD_MAX_CONTEXT_TOKENS_CAP = 8000;
 export declare const DEFAULT_MAX_METADATA_BYTES: number;
 export declare const HARD_MAX_METADATA_BYTES_CAP: number;
 export declare const DEFAULT_MAX_VECTOR_DIMENSIONS = 4096;
+export declare const DEFAULT_MAX_RERANK_BYTES: number;
+export declare const HARD_MAX_RERANK_BYTES_CAP: number;
+export declare const DEFAULT_MAX_RERANK_MS = 2000;
+export declare const HARD_MAX_RERANK_MS_CAP = 10000;
+export declare const DEFAULT_RERANK_CONCURRENCY = 2;
+export declare const HARD_RERANK_CONCURRENCY_CAP = 8;
+export declare const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
+export declare const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 export interface RagLimits {
     readonly chunkSize: number;
     readonly chunkOverlap: number;
     readonly maxDocumentChars: number;
+    readonly maxDocumentBytes: number;
+    readonly maxParseMs: number;
+    readonly maxPdfPages: number;
     readonly maxChunks: number;
     readonly embedBatchSize: number;
     readonly topK: number;
@@ -31,6 +48,10 @@ export interface RagLimits {
     readonly maxContextTokens: number;
     readonly maxMetadataBytes: number;
     readonly maxVectorDimensions: number;
+    readonly maxRerankBytes: number;
+    readonly maxRerankMs: number;
+    readonly rerankConcurrency: number;
+    readonly ingestionStatusPageSize: number;
 }
 export type RagLimitsInput = Partial<RagLimits>;
 export declare function resolveRagLimits(input?: RagLimitsInput): RagLimits;

package/dist/limits.js

@@ -5,6 +5,12 @@ export const DEFAULT_CHUNK_OVERLAP = 100;
 export const HARD_CHUNK_OVERLAP_CAP = 4_096;
 export const DEFAULT_MAX_DOCUMENT_CHARS = 1_048_576;
 export const HARD_MAX_DOCUMENT_CHARS_CAP = 8_388_608;
+export const DEFAULT_MAX_DOCUMENT_BYTES = 1_048_576;
+export const HARD_MAX_DOCUMENT_BYTES_CAP = 8 * 1024 * 1024;
+export const DEFAULT_MAX_PARSE_MS = 30_000;
+export const HARD_MAX_PARSE_MS_CAP = 30_000;
+export const DEFAULT_MAX_PDF_PAGES = 256;
+export const HARD_MAX_PDF_PAGES_CAP = 256;
 export const DEFAULT_MAX_CHUNKS = 2_048;
 export const HARD_MAX_CHUNKS_CAP = 8_192;
 export const DEFAULT_EMBED_BATCH_SIZE = 32;
@@ -20,6 +26,14 @@ export const HARD_MAX_CONTEXT_TOKENS_CAP = 8_000;
 export const DEFAULT_MAX_METADATA_BYTES = 16 * 1024;
 export const HARD_MAX_METADATA_BYTES_CAP = 64 * 1024;
 export const DEFAULT_MAX_VECTOR_DIMENSIONS = 4_096;
+export const DEFAULT_MAX_RERANK_BYTES = 64 * 1024;
+export const HARD_MAX_RERANK_BYTES_CAP = 256 * 1024;
+export const DEFAULT_MAX_RERANK_MS = 2_000;
+export const HARD_MAX_RERANK_MS_CAP = 10_000;
+export const DEFAULT_RERANK_CONCURRENCY = 2;
+export const HARD_RERANK_CONCURRENCY_CAP = 8;
+export const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
+export const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 function integer(value, fallback, cap, label, minimum = 1) {
     const resolved = value ?? fallback;
     if (!Number.isInteger(resolved) || resolved < minimum) {
@@ -42,6 +56,9 @@ export function resolveRagLimits(input = {}) {
         chunkSize,
         chunkOverlap,
         maxDocumentChars: integer(input.maxDocumentChars, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, "maxDocumentChars"),
+        maxDocumentBytes: integer(input.maxDocumentBytes, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, "maxDocumentBytes"),
+        maxParseMs: integer(input.maxParseMs, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, "maxParseMs"),
+        maxPdfPages: integer(input.maxPdfPages, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, "maxPdfPages"),
         maxChunks: integer(input.maxChunks, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, "maxChunks"),
         embedBatchSize: integer(input.embedBatchSize, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, "embedBatchSize"),
         topK,
@@ -50,6 +67,10 @@ export function resolveRagLimits(input = {}) {
         maxContextTokens: integer(input.maxContextTokens, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, "maxContextTokens"),
         maxMetadataBytes: integer(input.maxMetadataBytes, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, "maxMetadataBytes"),
         maxVectorDimensions: integer(input.maxVectorDimensions, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_VECTOR_DIMENSIONS, "maxVectorDimensions"),
+        maxRerankBytes: integer(input.maxRerankBytes, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, "maxRerankBytes"),
+        maxRerankMs: integer(input.maxRerankMs, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, "maxRerankMs"),
+        rerankConcurrency: integer(input.rerankConcurrency, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, "rerankConcurrency"),
+        ingestionStatusPageSize: integer(input.ingestionStatusPageSize, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, "ingestionStatusPageSize"),
     });
 }
 //# sourceMappingURL=limits.js.map

package/dist/loaders.d.ts

@@ -0,0 +1,24 @@
+import type { ResourceLoadContext, ResourceLoader } from "@arnilo/prism";
+import type { DocumentLoader } from "./types.js";
+export interface ResourceDocumentLoaderOptions {
+    readonly loader: ResourceLoader;
+    readonly context?: ResourceLoadContext;
+}
+export interface WebFetchDocument {
+    readonly citationId: string;
+    readonly provider: string;
+    readonly url: string;
+    readonly markdown: string;
+    readonly retrievedAt: string;
+    readonly untrusted: true;
+}
+export interface WebFetchAdapter {
+    fetch(url: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<WebFetchDocument>;
+}
+export interface WebFetchDocumentLoaderOptions {
+    readonly fetcher: WebFetchAdapter;
+}
+export declare function createResourceDocumentLoader(options: ResourceDocumentLoaderOptions): DocumentLoader;
+export declare function createWebFetchDocumentLoader(options: WebFetchDocumentLoaderOptions): DocumentLoader;

package/dist/loaders.js

@@ -0,0 +1,78 @@
+import { RagLimitError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { nonEmpty } from "./util.js";
+export function createResourceDocumentLoader(options) {
+    return {
+        async load(uri, loadOptions = {}) {
+            nonEmpty(uri, "uri");
+            const context = { ...options.context, ...(loadOptions.signal ? { signal: loadOptions.signal } : {}) };
+            const resource = await options.loader.load(uri, context);
+            if (resource.text === undefined && resource.data === undefined)
+                throw new RagValidationError("resource has no text or data");
+            const document = Object.freeze({
+                uri: resource.uri,
+                ...(resource.mediaType ? { mediaType: resource.mediaType } : {}),
+                ...(resource.text !== undefined ? { text: resource.text } : { data: new Uint8Array(resource.data) }),
+            });
+            assertDocumentBytes(document, loadOptions.maxBytes);
+            return document;
+        },
+    };
+}
+export function createWebFetchDocumentLoader(options) {
+    return {
+        async load(uri, loadOptions = {}) {
+            const url = publicWebUrl(uri);
+            const document = await options.fetcher.fetch(url, { signal: loadOptions.signal });
+            publicWebUrl(document.url);
+            if (!document.untrusted || !document.markdown)
+                throw new RagValidationError("web fetcher returned an invalid untrusted document");
+            const loaded = Object.freeze({
+                uri: document.url,
+                sourceId: document.citationId,
+                mediaType: "text/markdown",
+                text: document.markdown,
+                metadata: {
+                    web: {
+                        citationId: document.citationId,
+                        provider: document.provider,
+                        url: document.url,
+                        retrievedAt: document.retrievedAt,
+                    },
+                    untrusted: true,
+                },
+            });
+            assertDocumentBytes(loaded, loadOptions.maxBytes);
+            return loaded;
+        },
+    };
+}
+function assertDocumentBytes(document, maxBytes) {
+    const limit = resolveRagLimits({ maxDocumentBytes: maxBytes }).maxDocumentBytes;
+    const bytes = document.data ?? Buffer.from(document.text ?? "", "utf8");
+    if (bytes.byteLength > limit)
+        throw new RagLimitError(`document exceeds ${limit} bytes`);
+}
+function publicWebUrl(input) {
+    let url;
+    try {
+        url = new URL(input);
+    }
+    catch {
+        throw new RagValidationError("web document URI must be an absolute HTTP(S) URL");
+    }
+    const host = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+    if (!/^https?:$/u.test(url.protocol) || url.username || url.password || privateOrLocalHost(host)) {
+        throw new RagValidationError("web document URI must use a public hostname without credentials");
+    }
+    url.hash = "";
+    return url.toString();
+}
+function privateOrLocalHost(host) {
+    if (!host || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local"))
+        return true;
+    if (/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(host) || host.includes(":"))
+        return true; // Require fetcher DNS policy for hostnames; never fetch IP literals.
+    return false;
+}
+//# sourceMappingURL=loaders.js.map

package/dist/parsers.d.ts

@@ -0,0 +1,5 @@
+import type { Parser } from "./types.js";
+export declare const textParser: Parser;
+export declare const markdownParser: Parser;
+export declare const htmlParser: Parser;
+export declare const pdfParser: Parser;

package/dist/parsers.js

@@ -0,0 +1,99 @@
+import { RagLimitError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { assertNotAborted } from "./util.js";
+export const textParser = { parse: (document, options) => parseText(document, options, ["text/plain"]) };
+export const markdownParser = { parse: (document, options) => parseText(document, options, ["text/markdown", "text/x-markdown"]) };
+export const htmlParser = { parse: (document, options) => parseText(document, options, ["text/html", "application/xhtml+xml"], htmlToText) };
+export const pdfParser = { parse: parsePdf };
+async function parseText(document, options = {}, mediaTypes, transform = (text) => text) {
+    assertMediaType(document, mediaTypes);
+    const { bytes, limits, started } = boundedDocument(document, options);
+    assertNotAborted(options.signal);
+    let text;
+    try {
+        text = document.text ?? new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+    }
+    catch {
+        throw new RagValidationError("document is not valid UTF-8 text");
+    }
+    const parsed = transform(text);
+    assertParseTime(started, limits.maxParseMs);
+    assertNotAborted(options.signal);
+    return Object.freeze({ text: parsed, ...(document.metadata ? { metadata: document.metadata } : {}) });
+}
+async function parsePdf(document, options = {}) {
+    assertMediaType(document, ["application/pdf"]);
+    const { bytes, limits, started } = boundedDocument(document, options);
+    if (document.text !== undefined)
+        throw new RagValidationError("PDF parser requires binary document data");
+    const pdf = Buffer.from(bytes).toString("latin1");
+    if (!pdf.startsWith("%PDF-"))
+        throw new RagValidationError("document is not a PDF");
+    if (/\/Filter\s*\//u.test(pdf))
+        throw new RagValidationError("compressed PDFs require a host parser");
+    const pages = [...pdf.matchAll(/\/Type\s*\/Page\b/gu)].length;
+    if (pages > limits.maxPdfPages)
+        throw new RagLimitError(`PDF exceeds ${limits.maxPdfPages} pages`);
+    const blocks = pdf.match(/BT[\s\S]*?ET/gu) ?? [];
+    const text = blocks.flatMap(pdfBlockText).join("\n").trim();
+    if (!text)
+        throw new RagValidationError("PDF has no uncompressed text");
+    assertParseTime(started, limits.maxParseMs);
+    assertNotAborted(options.signal);
+    return Object.freeze({ text, ...(document.metadata ? { metadata: document.metadata } : {}) });
+}
+function boundedDocument(document, options) {
+    assertNotAborted(options.signal);
+    const limits = resolveRagLimits({
+        maxDocumentBytes: options.maxBytes,
+        maxParseMs: options.maxParseMs,
+        maxPdfPages: options.maxPages,
+    });
+    const bytes = document.data ?? Buffer.from(document.text ?? "", "utf8");
+    if (bytes.byteLength > limits.maxDocumentBytes)
+        throw new RagLimitError(`document exceeds ${limits.maxDocumentBytes} bytes`);
+    return { bytes, limits, started: Date.now() };
+}
+function assertMediaType(document, expected) {
+    if (document.mediaType && !expected.includes(document.mediaType.toLowerCase())) {
+        throw new RagValidationError(`parser does not accept ${document.mediaType}`);
+    }
+}
+function assertParseTime(started, maxParseMs) {
+    if (Date.now() - started > maxParseMs)
+        throw new RagLimitError(`document parsing exceeded ${maxParseMs}ms`);
+}
+function htmlToText(html) {
+    return decodeEntities(html
+        .replace(/<!--[\s\S]*?-->/gu, "")
+        .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/giu, "")
+        .replace(/<(?:br|p|div|li|h[1-6]|tr)\b[^>]*>/giu, "\n")
+        .replace(/<[^>]*>/gu, " "))
+        .replace(/[ \t]{2,}/gu, " ")
+        .replace(/[ \t]+\n/gu, "\n")
+        .replace(/\n[ \t]+/gu, "\n")
+        .replace(/\n{3,}/gu, "\n\n")
+        .trim();
+}
+function decodeEntities(text) {
+    const entities = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", "#39": "'" };
+    return text.replace(/&([a-z]+|#39);/giu, (_match, name) => entities[name.toLowerCase()] ?? `&${name};`);
+}
+function pdfBlockText(block) {
+    const strings = [];
+    for (const match of block.matchAll(/\((?:\\.|[^\\)])*\)\s*(?:Tj|['"])/gu))
+        strings.push(decodePdfString(match[0].replace(/\s*(?:Tj|['"])$/u, "")));
+    for (const match of block.matchAll(/\[([\s\S]*?)\]\s*TJ/gu)) {
+        for (const value of match[1].matchAll(/\((?:\\.|[^\\)])*\)/gu))
+            strings.push(decodePdfString(value[0]));
+    }
+    return strings.filter(Boolean);
+}
+function decodePdfString(value) {
+    return value.slice(1, -1).replace(/\\([0-7]{1,3}|[nrtbf()\\])/gu, (_match, escape) => {
+        if (/^[0-7]+$/u.test(escape))
+            return String.fromCharCode(Number.parseInt(escape, 8));
+        return { n: "\n", r: "\r", t: "\t", b: "\b", f: "\f", "(": "(", ")": ")", "\\": "\\" }[escape];
+    });
+}
+//# sourceMappingURL=parsers.js.map

package/dist/rerank.d.ts

@@ -0,0 +1,11 @@
+import type { RagHit, Reranker } from "./types.js";
+import { resolveRedactor } from "./util.js";
+export declare function rerankHits(query: string, hits: readonly RagHit[], options: {
+    readonly reranker: Reranker;
+    readonly maxBytes: number;
+    readonly maxMs: number;
+    readonly concurrency: number;
+    readonly signal?: AbortSignal;
+    readonly redactor?: Parameters<typeof resolveRedactor>[0];
+    readonly secrets?: Parameters<typeof resolveRedactor>[1];
+}): Promise<readonly RagHit[]>;

package/dist/rerank.js

@@ -0,0 +1,59 @@
+import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
+import { assertNotAborted, byteLength, resolveRedactor } from "./util.js";
+const active = new WeakMap();
+export async function rerankHits(query, hits, options) {
+    assertNotAborted(options.signal);
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const safeQuery = redactor?.redact(query) ?? query;
+    const safeHits = hits.map((hit) => redactor?.redact(hit) ?? hit);
+    if (byteLength({ query: safeQuery, hits: safeHits }) > options.maxBytes) {
+        throw new RagLimitError(`reranker input exceeds ${options.maxBytes} bytes`);
+    }
+    const running = active.get(options.reranker) ?? 0;
+    if (running >= options.concurrency)
+        throw new RagLimitError(`reranker concurrency exceeds ${options.concurrency}`);
+    active.set(options.reranker, running + 1);
+    const release = () => {
+        const remaining = (active.get(options.reranker) ?? 1) - 1;
+        if (remaining)
+            active.set(options.reranker, remaining);
+        else
+            active.delete(options.reranker);
+    };
+    try {
+        const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => { void operation.then(release, release); }, options.maxMs, options.signal);
+        const originals = new Map(hits.map((hit) => [hit.id, hit]));
+        if (ordered.length !== hits.length || new Set(ordered.map((hit) => hit.id)).size !== hits.length) {
+            throw new RagValidationError("reranker must return each retrieved hit exactly once");
+        }
+        const output = ordered.map((hit) => originals.get(hit.id));
+        if (output.some((hit) => !hit))
+            throw new RagValidationError("reranker returned an unknown hit");
+        return Object.freeze(output);
+    }
+    catch (error) {
+        if (error instanceof RagAbortError || error instanceof RagLimitError || error instanceof RagValidationError)
+            throw error;
+        const message = error instanceof Error ? error.message : "reranker failed";
+        throw new RagValidationError(`reranker failed: ${redactor?.redact(message) ?? message}`);
+    }
+}
+async function boundedRerank(run, onStart, maxMs, signal) {
+    const controller = new AbortController();
+    const onAbort = () => controller.abort();
+    signal?.addEventListener("abort", onAbort, { once: true });
+    const timeout = setTimeout(() => controller.abort(), maxMs);
+    const aborted = new Promise((_, reject) => controller.signal.addEventListener("abort", () => {
+        reject(signal?.aborted ? new RagAbortError() : new RagLimitError(`reranker exceeded ${maxMs}ms`));
+    }, { once: true }));
+    const operation = Promise.resolve().then(() => run(controller.signal));
+    onStart(operation);
+    try {
+        return await Promise.race([operation, aborted]);
+    }
+    finally {
+        clearTimeout(timeout);
+        signal?.removeEventListener("abort", onAbort);
+    }
+}
+//# sourceMappingURL=rerank.js.map

package/dist/retrieve.js

@@ -1,11 +1,12 @@
 import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
 import { RagScopeError, RagValidationError } from "./errors.js";
+import { rerankHits } from "./rerank.js";
 import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, resolveRedactor, truncateUtf8, } from "./util.js";
+const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
 export async function retrieveContext(query, options) {
     nonEmpty(query, "query");
-    if (query.length > HARD_CHUNK_SIZE_CAP) {
+    if (query.length > HARD_CHUNK_SIZE_CAP)
         throw new RagValidationError(`query exceeds ${HARD_CHUNK_SIZE_CAP} characters`);
-    }
     const scope = requireScope(options.scope);
     const limits = resolveRagLimits({
         topK: options.topK,
@@ -14,10 +15,11 @@ export async function retrieveContext(query, options) {
         maxContextTokens: options.maxContextTokens,
         maxMetadataBytes: options.maxMetadataBytes,
         maxVectorDimensions: options.maxVectorDimensions,
+        maxRerankBytes: options.maxRerankBytes,
+        maxRerankMs: options.maxRerankMs,
+        rerankConcurrency: options.rerankConcurrency,
     });
-    if (!Number.isInteger(options.embedder.dimensions)
-        || options.embedder.dimensions <= 0
-        || options.embedder.dimensions > limits.maxVectorDimensions) {
+    if (!Number.isInteger(options.embedder.dimensions) || options.embedder.dimensions <= 0 || options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
     if (options.filter)
@@ -27,10 +29,7 @@ export async function retrieveContext(query, options) {
     assertNotAborted(options.signal);
     const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
     const embedding = vectors[0];
-    if (vectors.length !== 1
-        || !embedding
-        || embedding.length !== options.embedder.dimensions
-        || embedding.some((value) => !Number.isFinite(value))) {
+    if (vectors.length !== 1 || !embedding || embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
         throw new RagValidationError("embedder returned invalid query vector");
     }
     const candidates = await options.store.query({
@@ -42,6 +41,26 @@ export async function retrieveContext(query, options) {
         signal: options.signal,
     });
     assertNotAborted(options.signal);
+    const retrievedAt = new Date().toISOString();
+    const retrieved = [];
+    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
+        assertScope(scope, candidate);
+        const parsed = parseHit(candidate, retrieved.length, retrievedAt);
+        if (!matchesFilter(parsed.metadata, options.filter))
+            continue;
+        retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
+    }
+    const ranked = options.reranker
+        ? await rerankHits(safeQuery, retrieved, {
+            reranker: options.reranker,
+            maxBytes: limits.maxRerankBytes,
+            maxMs: limits.maxRerankMs,
+            concurrency: limits.rerankConcurrency,
+            signal: options.signal,
+            redactor: options.redactor,
+            secrets: options.secrets,
+        })
+        : retrieved;
     const hits = [];
     const citations = [];
     const rendered = [];
@@ -49,15 +68,10 @@ export async function retrieveContext(query, options) {
     let usedBytes = 0;
     let usedChars = 0;
     let truncated = false;
-    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
-        assertScope(scope, candidate);
-        const parsed = parseHit(candidate);
-        if (!matchesFilter(parsed.metadata, options.filter))
-            continue;
+    for (const hit of ranked) {
         if (hits.length >= limits.topK)
             break;
-        const safe = redactor?.redact(parsed) ?? parsed;
-        const prefix = `[${parsed.citationId}] `;
+        const prefix = `[${hit.citationId}] `;
         const separator = rendered.length ? "\n\n" : "";
         const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
         const availableChars = maxChars - usedChars - separator.length - prefix.length;
@@ -65,61 +79,60 @@ export async function retrieveContext(query, options) {
             truncated = true;
             break;
         }
-        let text = safe.text.slice(0, availableChars);
+        let text = hit.text.slice(0, availableChars);
         text = truncateUtf8(text, availableBytes);
         if (!text) {
             truncated = true;
             break;
         }
-        if (text.length < safe.text.length)
+        if (text.length < hit.text.length)
             truncated = true;
-        const hit = Object.freeze({ ...safe, text });
+        const renderedHit = Object.freeze({ ...hit, text });
         const citation = Object.freeze({
-            id: hit.citationId,
-            sourceId: hit.sourceId,
-            chunkId: hit.id,
-            ...(hit.metadata ? { metadata: hit.metadata } : {}),
+            id: renderedHit.citationId,
+            sourceId: renderedHit.sourceId,
+            chunkId: renderedHit.id,
+            provenance: renderedHit.provenance,
+            trust: renderedHit.trust,
+            ...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
         });
         const block = `${separator}${prefix}${text}`;
         rendered.push(block);
         usedBytes += byteLength(block);
         usedChars += block.length;
-        hits.push(hit);
+        hits.push(renderedHit);
         citations.push(citation);
         if (truncated)
             break;
     }
     return Object.freeze({
         query: safeQuery,
+        trust: RETRIEVED_CONTENT_TRUST,
         text: rendered.join(""),
         hits: Object.freeze(hits),
         citations: Object.freeze(citations),
         truncated,
     });
 }
-function parseHit(hit) {
+function parseHit(hit, retrievalRank, retrievedAt) {
     const metadata = hit.metadata;
     const rag = metadata?._rag;
     if (!isJsonObject(rag))
         throw new RagScopeError("vector hit is missing RAG source metadata");
     const sourceId = requireSourceId(rag.sourceId);
     const citationId = nonEmpty(rag.citationId, "metadata._rag.citationId");
-    if (!Number.isInteger(rag.chunkIndex)
-        || Number(rag.chunkIndex) < 0
-        || !Number.isInteger(rag.start)
-        || Number(rag.start) < 0
-        || !Number.isInteger(rag.end)
-        || Number(rag.end) < Number(rag.start)
-        || !Number.isFinite(hit.score)) {
+    if (!Number.isInteger(rag.chunkIndex) || Number(rag.chunkIndex) < 0 || !Number.isInteger(rag.start) || Number(rag.start) < 0 || !Number.isInteger(rag.end) || Number(rag.end) < Number(rag.start) || !Number.isFinite(hit.score)) {
         throw new RagValidationError("vector hit has invalid RAG offsets");
     }
-    if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`)) {
+    if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`))
         throw new RagValidationError("vector hit has inconsistent citation identity");
-    }
     const userMetadata = {};
     for (const [key, value] of Object.entries(metadata ?? {}))
         if (key !== "_rag")
             userMetadata[key] = value;
+    const web = isJsonObject(userMetadata.web) ? userMetadata.web : undefined;
+    const provider = typeof web?.provider === "string" && web.provider.trim() ? web.provider : "host";
+    const provenance = Object.freeze({ sourceId, chunkId: hit.id, citationId, provider, retrieval: "vector", retrievedAt });
     return {
         id: hit.id,
         citationId,
@@ -129,6 +142,9 @@ function parseHit(hit) {
         end: rag.end,
         text: hit.text,
         score: hit.score,
+        retrievalRank,
+        provenance,
+        trust: RETRIEVED_CONTENT_TRUST,
         ...(Object.keys(userMetadata).length ? { metadata: userMetadata } : {}),
     };
 }

package/dist/sources.d.ts

@@ -0,0 +1,9 @@
+import type { DeleteSourceOptions, ReplaceDocumentOptions, ReplaceSourceOptions } from "./types.js";
+export interface SourceMutationResult {
+    readonly sourceId: string;
+    readonly deleted: number;
+    readonly indexed: number;
+}
+export declare function replaceSource(options: ReplaceSourceOptions): Promise<SourceMutationResult>;
+export declare function deleteSource(options: DeleteSourceOptions): Promise<SourceMutationResult>;
+export declare function replaceDocument(options: ReplaceDocumentOptions): Promise<SourceMutationResult>;

package/dist/sources.js

@@ -0,0 +1,113 @@
+import { chunkText } from "./chunk.js";
+import { RagScopeError, RagValidationError } from "./errors.js";
+import { ingestionStatus } from "./ingestion-status.js";
+import { indexChunkBatches } from "./indexing.js";
+import { assertNotAborted, byteLength, requireScope, requireSourceId, resolveRedactor } from "./util.js";
+export async function replaceSource(options) {
+    const sourceId = requireSourceId(options.sourceId);
+    const scope = requireScope(options.scope);
+    assertTransactionalStore(options.store);
+    if (options.chunks.some((chunk) => chunk.sourceId !== sourceId)) {
+        throw new RagValidationError("replaceSource chunks must all belong to sourceId");
+    }
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const totalBytes = options.chunks.reduce((total, chunk) => total + byteLength(redactor?.redact(chunk.text) ?? chunk.text), 0);
+    const setStatus = async (state, error) => {
+        if (!options.statusStore)
+            return;
+        const message = error instanceof Error ? error.message : error === undefined ? undefined : "source replacement failed";
+        await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? redactor?.redact(message) ?? message : undefined));
+    };
+    await setStatus("pending");
+    try {
+        const staged = [];
+        const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => { staged.push(...records); });
+        assertNotAborted(options.signal);
+        const result = await options.store.transaction(async (store) => {
+            const previous = await sourceRecords(store, sourceId, scope, options.signal);
+            assertNotAborted(options.signal);
+            if (previous.length) {
+                await store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: previous.map((record) => record.id) }, { signal: options.signal });
+            }
+            if (staged.length)
+                await store.upsert(staged, { signal: options.signal });
+            return Object.freeze({ sourceId, deleted: previous.length, indexed: indexed.indexed });
+        }, { signal: options.signal });
+        await setStatus("indexed");
+        return result;
+    }
+    catch (error) {
+        await setStatus("failed", error);
+        throw error;
+    }
+}
+export async function deleteSource(options) {
+    const sourceId = requireSourceId(options.sourceId);
+    const scope = requireScope(options.scope);
+    assertSourceStore(options.store);
+    const records = await sourceRecords(options.store, sourceId, scope, options.signal);
+    assertNotAborted(options.signal);
+    const deleted = records.length
+        ? await options.store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: records.map((record) => record.id) }, { signal: options.signal })
+        : 0;
+    await options.statusStore?.delete(scope, sourceId);
+    return Object.freeze({ sourceId, deleted, indexed: 0 });
+}
+export async function replaceDocument(options) {
+    const loaded = await options.loader.load(options.uri, {
+        ...options.loaderOptions,
+        signal: options.signal ?? options.loaderOptions?.signal,
+    });
+    assertNotAborted(options.signal);
+    const parsed = await options.parser.parse(loaded, {
+        ...options.parserOptions,
+        signal: options.signal ?? options.parserOptions?.signal,
+    });
+    assertNotAborted(options.signal);
+    const sourceId = options.sourceId ?? loaded.sourceId;
+    if (!sourceId)
+        throw new RagValidationError("replaceDocument requires sourceId when its loader does not provide one");
+    const { metadata, ...chunk } = options.chunk ?? {};
+    const mergedMetadata = mergeMetadata(parsed.metadata, metadata);
+    const chunks = (options.chunker ?? chunkText)(parsed.text, {
+        ...chunk,
+        sourceId,
+        ...(mergedMetadata ? { metadata: mergedMetadata } : {}),
+    });
+    return replaceSource({
+        ...options,
+        sourceId,
+        chunks,
+    });
+}
+function mergeMetadata(parser, supplied) {
+    if (!parser && !supplied)
+        return undefined;
+    return { ...supplied, ...parser }; // Loader trust metadata wins over caller-supplied metadata.
+}
+async function sourceRecords(store, sourceId, scope, signal) {
+    assertNotAborted(signal);
+    const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, { signal });
+    for (const record of records) {
+        if (record.tenantId !== scope.tenantId || record.resourceId !== scope.resourceId || record.threadId !== scope.corpusId) {
+            throw new RagScopeError("source lookup crossed tenant/resource/corpus boundary");
+        }
+        const rag = record.metadata?._rag;
+        if (typeof rag !== "object" || rag === null || Array.isArray(rag) || rag.sourceId !== sourceId) {
+            throw new RagScopeError("source lookup returned a different source");
+        }
+    }
+    return records;
+}
+function assertSourceStore(store) {
+    if (!store || typeof store !== "object" || typeof store.getBySource !== "function") {
+        throw new RagValidationError("source deletion requires a scoped source-aware vector store");
+    }
+}
+function assertTransactionalStore(store) {
+    if (!store || typeof store !== "object" || typeof store.transaction !== "function") {
+        throw new RagValidationError("atomic source replacement requires a transactional vector store");
+    }
+    assertSourceStore(store);
+}
+//# sourceMappingURL=sources.js.map

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
-import type { Embedder, VectorStore } from "@arnilo/prism-memory";
+import type { Embedder, MemoryVectorRecord, VectorStore } from "@arnilo/prism-memory";
 export interface RagScope {
     readonly tenantId: string;
     readonly resourceId: string;
@@ -23,6 +23,80 @@ export interface ChunkOptions {
     readonly maxDocumentChars?: number;
     readonly maxChunks?: number;
 }
+export interface LoadedDocument {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly mediaType?: string;
+    readonly text?: string;
+    readonly data?: Uint8Array;
+    readonly metadata?: JsonObject;
+}
+export interface ParsedDocument {
+    readonly text: string;
+    readonly metadata?: JsonObject;
+}
+export interface DocumentLoadOptions {
+    readonly signal?: AbortSignal;
+    readonly maxBytes?: number;
+}
+export interface DocumentParseOptions extends DocumentLoadOptions {
+    readonly maxParseMs?: number;
+    readonly maxPages?: number;
+}
+export interface DocumentLoader {
+    load(uri: string, options?: DocumentLoadOptions): Promise<LoadedDocument>;
+}
+export interface Parser {
+    parse(document: LoadedDocument, options?: DocumentParseOptions): Promise<ParsedDocument>;
+}
+export type Chunker = (text: string, options: ChunkOptions) => readonly RagChunk[];
+export interface SourceVectorStore extends VectorStore {
+    getBySource(scope: {
+        readonly tenantId: string;
+        readonly resourceId: string;
+        readonly threadId: string;
+    }, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<readonly MemoryVectorRecord[]>;
+}
+export interface TransactionalVectorStore extends SourceVectorStore {
+    transaction<T>(operation: (store: SourceVectorStore) => Promise<T>, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<T>;
+}
+export type IngestionState = "pending" | "indexed" | "failed" | "partial";
+export interface IngestionStatus {
+    readonly sourceId: string;
+    readonly scope: RagScope;
+    readonly state: IngestionState;
+    readonly bytes: number;
+    readonly chunks: number;
+    readonly error?: string;
+    readonly updatedAt: string;
+}
+export interface IngestionStatusStore {
+    set(status: IngestionStatus, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    delete(scope: RagScope, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    list(scope: RagScope, options: {
+        readonly limit: number;
+        readonly cursor?: string;
+        readonly signal?: AbortSignal;
+    }): Promise<{
+        readonly entries: readonly IngestionStatus[];
+        readonly nextCursor?: string;
+    }>;
+}
+export interface IngestionStatusQuery {
+    readonly store: IngestionStatusStore;
+    readonly scope: RagScope;
+    readonly limit?: number;
+    readonly cursor?: string;
+    readonly signal?: AbortSignal;
+}
 export interface IndexChunksOptions {
     readonly chunks: readonly RagChunk[];
     readonly embedder: Embedder;
@@ -35,20 +109,68 @@ export interface IndexChunksOptions {
     readonly maxMetadataBytes?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
+    readonly statusStore?: IngestionStatusStore;
     readonly signal?: AbortSignal;
 }
 export interface IndexChunksResult {
     readonly indexed: number;
     readonly sourceIds: readonly string[];
 }
+export interface ReplaceSourceOptions extends Omit<IndexChunksOptions, "chunks" | "store"> {
+    readonly sourceId: string;
+    readonly chunks: readonly RagChunk[];
+    readonly store: TransactionalVectorStore;
+}
+export interface DeleteSourceOptions {
+    readonly sourceId: string;
+    readonly store: SourceVectorStore;
+    readonly scope: RagScope;
+    readonly statusStore?: IngestionStatusStore;
+    readonly signal?: AbortSignal;
+}
+export interface ReplaceDocumentOptions extends Omit<ReplaceSourceOptions, "sourceId" | "chunks"> {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly loader: DocumentLoader;
+    readonly parser: Parser;
+    readonly chunker?: Chunker;
+    readonly chunk?: Omit<ChunkOptions, "sourceId">;
+    readonly loaderOptions?: DocumentLoadOptions;
+    readonly parserOptions?: DocumentParseOptions;
+}
+export interface RagProvenance {
+    readonly sourceId: string;
+    readonly chunkId: string;
+    readonly citationId: string;
+    readonly provider: string;
+    readonly retrieval: "vector";
+    readonly retrievedAt: string;
+}
+export interface RagContentTrust {
+    readonly untrusted: true;
+    readonly inert: true;
+    readonly injectionCapable: true;
+}
 export interface RagCitation {
     readonly id: string;
     readonly sourceId: string;
     readonly chunkId: string;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
     readonly metadata?: JsonObject;
 }
 export interface RagHit extends RagChunk {
     readonly score: number;
+    readonly retrievalRank: number;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
+}
+export interface Reranker {
+    rerank(input: {
+        readonly query: string;
+        readonly hits: readonly RagHit[];
+        readonly signal?: AbortSignal;
+    }): Promise<readonly RagHit[]>;
 }
 export interface RetrieveContextOptions {
     readonly embedder: Embedder;
@@ -61,12 +183,17 @@ export interface RetrieveContextOptions {
     readonly maxContextTokens?: number;
     readonly maxMetadataBytes?: number;
     readonly maxVectorDimensions?: number;
+    readonly reranker?: Reranker;
+    readonly maxRerankBytes?: number;
+    readonly maxRerankMs?: number;
+    readonly rerankConcurrency?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
     readonly signal?: AbortSignal;
 }
 export interface RagContextResult {
     readonly query: string;
+    readonly trust: RagContentTrust;
     readonly text: string;
     readonly hits: readonly RagHit[];
     readonly citations: readonly RagCitation[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arnilo/prism-rag",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "description": "Optional bounded text and Markdown RAG primitives for Prism.",
   "type": "module",
   "main": "./dist/index.js",
@@ -9,6 +9,14 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./loaders": {
+      "types": "./dist/loaders.d.ts",
+      "default": "./dist/loaders.js"
+    },
+    "./parsers": {
+      "types": "./dist/parsers.d.ts",
+      "default": "./dist/parsers.js"
     }
   },
   "files": [
@@ -25,8 +33,8 @@
     "pack:dry-run": "npm pack --dry-run"
   },
   "peerDependencies": {
-    "@arnilo/prism": "0.0.14",
-    "@arnilo/prism-memory": "0.0.14"
+    "@arnilo/prism": "0.0.15",
+    "@arnilo/prism-memory": "0.0.15"
   },
   "devDependencies": {
     "@arnilo/prism": "file:../..",