@arnilo/prism-rag 0.0.14 → 0.0.16

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [0.0.16] - 2026-07-26
+
+### Changed
+- Sourced `resolveRedactor` from `@arnilo/prism` core and deleted the private duplicate; no public API change.
+
+## [0.0.15] - 2026-07-26
+
+### Added
+
+- Added atomic scoped `replaceSource()` / `deleteSource()` and `replaceDocument()` over source-aware transactional vector storage; the in-memory reference store implements `getBySource()` plus transactions.
+- Added bounded `DocumentLoader` / `Parser` seams, reference text/Markdown/HTML/PDF parsers, and host resource/web-tools document loaders with citation/trust preservation.
+- Added host-owned bounded reranking, canonical citation provenance/content-trust metadata, and scoped capped ingestion-status storage/querying with a process-local reference adapter.
+
 ## [0.0.14] - 2026-07-26
 
 ### Changed

package/README.md

@@ -1,6 +1,6 @@
 # @arnilo/prism-rag
 
-Optional bounded text/Markdown retrieval-augmented generation primitives for Prism. Reuses `Embedder` and `VectorStore` from `@arnilo/prism-memory`; no document framework, network loader, or core activation.
+Optional bounded source lifecycle, document parsing, host reranking, ingestion status, and retrieval-augmented generation primitives for Prism. Reuses `Embedder` and `VectorStore` from `@arnilo/prism-memory`; no document framework, network loader, or core activation.
 
 ## Install
 
@@ -12,7 +12,7 @@ npm install @arnilo/prism-rag @arnilo/prism-memory @arnilo/prism
 
 ```ts
 import { createHashEmbedder, createMemoryVectorStore } from "@arnilo/prism-memory";
-import { chunkMarkdown, createRagContextProvider, indexChunks } from "@arnilo/prism-rag";
+import { chunkMarkdown, createMemoryIngestionStatusStore, createRagContextProvider, listIngestionStatus, replaceSource } from "@arnilo/prism-rag";
 
 const embedder = createHashEmbedder(); // demo/test only
 const store = createMemoryVectorStore();
@@ -20,7 +20,9 @@ const scope = { tenantId: "t1", resourceId: "docs", corpusId: "handbook" };
 const chunks = chunkMarkdown("# Approval\n\nRecheck policy before side effects.", {
   sourceId: "security-guide",
 });
-await indexChunks({ chunks, embedder, store, scope });
+const statusStore = createMemoryIngestionStatusStore();
+await replaceSource({ sourceId: "security-guide", chunks, embedder, store, scope, statusStore }); // atomic with the reference store
+console.log(await listIngestionStatus({ store: statusStore, scope }));
 const context = createRagContextProvider({ embedder, store, scope, topK: 4 });
 ```
 
@@ -28,11 +30,15 @@ const context = createRagContextProvider({ embedder, store, scope, topK: 4 });
 
 - `chunkText()` / `chunkMarkdown()` — deterministic boundary-aware character chunks with overlap and stable citations.
 - `indexChunks()` — bounded batch embedding and scoped vector upsert.
-- `retrieveContext()` — bounded candidate query, shallow metadata filter, top-K hits, and citation rendering.
+- `replaceSource()` / `deleteSource()` — exact-scope source lifecycle; replacement requires transactional `getBySource()` storage.
+- `replaceDocument()` + `DocumentLoader` / `Parser` — host-authorized load, bounded parse, chunk, and replacement.
+- `textParser` / `markdownParser` / `htmlParser` / `pdfParser` — focused reference parsers (`./parsers`); `createResourceDocumentLoader` / `createWebFetchDocumentLoader` are in `./loaders`.
+- `retrieveContext()` — bounded candidate query, optional host `Reranker`, shallow metadata filter, top-K hits, attributable citations, and untrusted/inert/injection-capable trust metadata.
+- `createMemoryIngestionStatusStore()` / `listIngestionStatus()` — optional capped exact-scope pending/indexed/failed/partial source progress; implement `IngestionStatusStore` for durable status.
 - `createRagContextProvider()` — explicit inert context injection through Prism's existing seam.
 
 ## Security
 
-Every operation requires tenant/resource/corpus scope. Configure `redactor` or `secrets` before external embedding/persistence. Package performs no I/O; load remote/local sources through host-owned resource/media policies. Retrieved text is untrusted context and grants no tools or permissions.
+Every operation requires tenant/resource/corpus scope. Configure `redactor` or `secrets` before external embedding/persistence. Package performs no I/O: resource loading delegates to a host `ResourceLoader`, and web loading delegates to an existing web-tools adapter. Replacement fails closed without a scoped transaction; rerankers get redacted finite candidates and cannot alter canonical provenance/trust; HTML/PDF/web output is untrusted inert context and grants no tools or permissions.
 
 See [RAG](../../docs/rag.md).

package/dist/chunk.d.ts

@@ -1,3 +1,3 @@
-import type { RagChunk, ChunkOptions } from "./types.js";
+import type { ChunkOptions, RagChunk } from "./types.js";
 export declare function chunkText(text: string, options: ChunkOptions): readonly RagChunk[];
 export declare function chunkMarkdown(markdown: string, options: ChunkOptions): readonly RagChunk[];

package/dist/chunk.js

@@ -1,5 +1,5 @@
-import { resolveRagLimits } from "./limits.js";
 import { RagLimitError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
 import { assertBytes, requireSourceId } from "./util.js";
 export function chunkText(text, options) {
     return chunkDocument(text, options, false);

package/dist/context.js

@@ -7,18 +7,20 @@ export function createRagContextProvider(options) {
             context.signal?.throwIfAborted();
             const query = typeof options.query === "function"
                 ? options.query({ messages: context.messages })
-                : options.query ?? latestUserText(context.messages);
+                : (options.query ?? latestUserText(context.messages));
             if (!query?.trim())
                 return [];
             const result = await retrieveContext(query, { ...options, signal: context.signal });
             if (!result.text)
                 return [];
-            return [{
+            return [
+                {
                     id: `${options.name ?? "rag"}:context`,
                     title: options.title ?? "Retrieved context",
                     content: result.text,
-                    metadata: { citations: result.citations, inert: true },
-                }];
+                    metadata: { citations: result.citations, trust: result.trust, inert: true, untrusted: true, injectionCapable: true },
+                },
+            ];
         },
     };
 }

package/dist/index.d.ts

@@ -1,9 +1,14 @@
-export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
-export type { RagLimits, RagLimitsInput } from "./limits.js";
-export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { chunkMarkdown, chunkText } from "./chunk.js";
+export { createRagContextProvider } from "./context.js";
+export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { indexChunks } from "./indexing.js";
+export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
+export type { RagLimits, RagLimitsInput } from "./limits.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
+export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
-export { createRagContextProvider } from "./context.js";
-export type { ChunkOptions, IndexChunksOptions, IndexChunksResult, RagChunk, RagCitation, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagScope, RetrieveContextOptions, } from "./types.js";
+export type { SourceMutationResult } from "./sources.js";
+export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
+export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
 export declare const packageName = "@arnilo/prism-rag";

package/dist/index.js

@@ -1,8 +1,12 @@
-export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
-export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { chunkMarkdown, chunkText } from "./chunk.js";
+export { createRagContextProvider } from "./context.js";
+export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export { indexChunks } from "./indexing.js";
+export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
+export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
+export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
 export { retrieveContext } from "./retrieve.js";
-export { createRagContextProvider } from "./context.js";
+export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
 export const packageName = "@arnilo/prism-rag";
 //# sourceMappingURL=index.js.map

package/dist/indexing.d.ts

@@ -1,2 +1,4 @@
+import type { MemoryVectorRecord } from "@arnilo/prism-memory";
 import type { IndexChunksOptions, IndexChunksResult } from "./types.js";
 export declare function indexChunks(options: IndexChunksOptions): Promise<IndexChunksResult>;
+export declare function indexChunkBatches(options: IndexChunksOptions, write: (records: readonly MemoryVectorRecord[]) => Promise<void>): Promise<IndexChunksResult>;

package/dist/indexing.js

@@ -1,9 +1,12 @@
-import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { resolveRedactor } from "@arnilo/prism";
 import { RagValidationError } from "./errors.js";
-import { assertBytes, assertNotAborted, nonEmpty, requireScope, requireSourceId, resolveRedactor } from "./util.js";
-// ponytail: stable IDs make identical retries idempotent; replacing a source with
-// fewer chunks requires host deletion of stale source IDs before this generic upsert.
+import { ingestionStatus } from "./ingestion-status.js";
+import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId } from "./util.js";
 export async function indexChunks(options) {
+    return indexChunkBatches(options, async (records) => options.store.upsert(records, { signal: options.signal }));
+}
+export async function indexChunkBatches(options, write) {
     const scope = requireScope(options.scope);
     const limits = resolveRagLimits({
         embedBatchSize: options.batchSize,
@@ -12,78 +15,97 @@ export async function indexChunks(options) {
         maxVectorDimensions: options.maxVectorDimensions,
         maxMetadataBytes: options.maxMetadataBytes,
     });
-    if (!Number.isInteger(options.embedder.dimensions)
-        || options.embedder.dimensions <= 0
-        || options.embedder.dimensions > limits.maxVectorDimensions) {
+    if (!Number.isInteger(options.embedder.dimensions) ||
+        options.embedder.dimensions <= 0 ||
+        options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
-    if (options.chunks.length > limits.maxChunks) {
+    if (options.chunks.length > limits.maxChunks)
         throw new RagValidationError(`chunk count exceeds ${limits.maxChunks}`);
-    }
     const redactor = resolveRedactor(options.redactor, options.secrets);
     const sourceIds = new Set();
     const chunkIds = new Set();
+    const total = new Map();
+    const written = new Map();
     for (const chunk of options.chunks) {
         nonEmpty(chunk.id, "chunk.id");
         requireSourceId(chunk.sourceId);
-        if (chunk.id !== chunk.citationId || !chunk.citationId.startsWith(`${chunk.sourceId}#`)) {
+        if (chunk.id !== chunk.citationId || !chunk.citationId.startsWith(`${chunk.sourceId}#`))
             throw new RagValidationError("chunk has inconsistent citation identity");
-        }
         if (chunkIds.has(chunk.id))
             throw new RagValidationError(`duplicate chunk id: ${chunk.id}`);
-        if (!Number.isInteger(chunk.index)
-            || chunk.index < 0
-            || !Number.isInteger(chunk.start)
-            || chunk.start < 0
-            || !Number.isInteger(chunk.end)
-            || chunk.end < chunk.start) {
+        if (!Number.isInteger(chunk.index) ||
+            chunk.index < 0 ||
+            !Number.isInteger(chunk.start) ||
+            chunk.start < 0 ||
+            !Number.isInteger(chunk.end) ||
+            chunk.end < chunk.start) {
             throw new RagValidationError("chunk has invalid index or offsets");
         }
         if (chunk.text.length > limits.chunkSize)
             throw new RagValidationError(`chunk text exceeds ${limits.chunkSize} characters`);
         chunkIds.add(chunk.id);
+        sourceIds.add(chunk.sourceId);
+        add(total, chunk.sourceId, byteLength(redactor?.redact(chunk.text) ?? chunk.text), 1);
     }
-    for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
-        assertNotAborted(options.signal);
-        const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
-        const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
-        const vectors = await options.embedder.embed(texts, { signal: options.signal });
-        if (vectors.length !== batch.length)
-            throw new RagValidationError("embedder returned unexpected vector count");
-        const records = batch.map((chunk, index) => {
-            const embedding = vectors[index];
-            if (embedding.length !== options.embedder.dimensions
-                || embedding.some((value) => !Number.isFinite(value))) {
-                throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
-            }
-            const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? (chunk.metadata ?? {});
-            const metadata = {
-                ...safeMetadata,
-                _rag: {
-                    sourceId: chunk.sourceId,
-                    citationId: chunk.citationId,
-                    chunkIndex: chunk.index,
-                    start: chunk.start,
-                    end: chunk.end,
-                },
-            };
-            assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
-            sourceIds.add(chunk.sourceId);
-            return {
-                id: chunk.id,
-                tenantId: scope.tenantId,
-                resourceId: scope.resourceId,
-                threadId: scope.corpusId,
-                text: texts[index],
-                embedding,
-                sequence: chunk.index,
-                metadata: metadata,
-                createdAt: new Date(0).toISOString(),
-            };
-        });
-        assertNotAborted(options.signal);
-        await options.store.upsert(records, { signal: options.signal });
+    const setStatus = async (state, error) => {
+        if (!options.statusStore)
+            return;
+        const message = error instanceof Error ? error.message : error === undefined ? undefined : "indexing failed";
+        const safeError = message ? (redactor?.redact(message) ?? message) : undefined;
+        for (const sourceId of sourceIds) {
+            const progress = state === "indexed" ? total.get(sourceId) : (written.get(sourceId) ?? { bytes: 0, chunks: 0 });
+            await options.statusStore.set(ingestionStatus(scope, sourceId, state, progress.bytes, progress.chunks, safeError));
+        }
+    };
+    await setStatus("pending");
+    try {
+        for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
+            assertNotAborted(options.signal);
+            const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
+            const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
+            const vectors = await options.embedder.embed(texts, { signal: options.signal });
+            if (vectors.length !== batch.length)
+                throw new RagValidationError("embedder returned unexpected vector count");
+            const records = batch.map((chunk, index) => {
+                const embedding = vectors[index];
+                if (embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
+                    throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
+                }
+                const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? chunk.metadata ?? {};
+                const metadata = {
+                    ...safeMetadata,
+                    _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end },
+                };
+                assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
+                return {
+                    id: chunk.id,
+                    tenantId: scope.tenantId,
+                    resourceId: scope.resourceId,
+                    threadId: scope.corpusId,
+                    text: texts[index],
+                    embedding,
+                    sequence: chunk.index,
+                    metadata: metadata,
+                    createdAt: new Date(0).toISOString(),
+                };
+            });
+            assertNotAborted(options.signal);
+            await write(records);
+            for (const chunk of batch)
+                add(written, chunk.sourceId, byteLength(redactor?.redact(chunk.text) ?? chunk.text), 1);
+            await setStatus("pending");
+        }
+        await setStatus("indexed");
+    }
+    catch (error) {
+        await setStatus([...written.values()].some((progress) => progress.chunks > 0) ? "partial" : "failed", error);
+        throw error;
     }
     return Object.freeze({ indexed: options.chunks.length, sourceIds: Object.freeze([...sourceIds].sort()) });
 }
+function add(target, sourceId, bytes, chunks) {
+    const current = target.get(sourceId) ?? { bytes: 0, chunks: 0 };
+    target.set(sourceId, { bytes: current.bytes + bytes, chunks: current.chunks + chunks });
+}
 //# sourceMappingURL=indexing.js.map

package/dist/ingestion-status.d.ts

@@ -0,0 +1,7 @@
+import type { IngestionStatus, IngestionStatusQuery, IngestionStatusStore, RagScope } from "./types.js";
+export declare function listIngestionStatus(options: IngestionStatusQuery): Promise<{
+    readonly entries: readonly IngestionStatus[];
+    readonly nextCursor?: string;
+}>;
+export declare function createMemoryIngestionStatusStore(): IngestionStatusStore;
+export declare function ingestionStatus(scope: RagScope, sourceId: string, state: IngestionStatus["state"], bytes: number, chunks: number, error?: string): IngestionStatus;

package/dist/ingestion-status.js

@@ -0,0 +1,87 @@
+import { RagScopeError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
+export async function listIngestionStatus(options) {
+    const scope = requireScope(options.scope);
+    const limit = resolveRagLimits({ ingestionStatusPageSize: options.limit }).ingestionStatusPageSize;
+    if (options.cursor !== undefined)
+        requireSourceId(options.cursor);
+    assertNotAborted(options.signal);
+    const page = await options.store.list(scope, { limit, cursor: options.cursor, signal: options.signal });
+    if (page.entries.length > limit)
+        throw new RagValidationError("ingestion status store exceeded requested page limit");
+    for (const entry of page.entries)
+        assertStatus(entry, scope);
+    if (page.nextCursor !== undefined)
+        requireSourceId(page.nextCursor);
+    return Object.freeze({ entries: Object.freeze([...page.entries]), ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}) });
+}
+export function createMemoryIngestionStatusStore() {
+    const scopes = new Map();
+    return {
+        async set(status, options = {}) {
+            assertNotAborted(options.signal);
+            const scope = requireScope(status.scope);
+            assertStatus(status, scope);
+            const key = scopeKey(scope);
+            const entries = scopes.get(key) ?? new Map();
+            entries.set(status.sourceId, Object.freeze({ ...status, scope }));
+            scopes.set(key, entries);
+        },
+        async delete(scope, sourceId, options = {}) {
+            assertNotAborted(options.signal);
+            const entries = scopes.get(scopeKey(requireScope(scope)));
+            entries?.delete(requireSourceId(sourceId));
+        },
+        async list(scope, options) {
+            assertNotAborted(options.signal);
+            const required = requireScope(scope);
+            const limit = options.limit;
+            if (!Number.isInteger(limit) || limit < 1)
+                throw new RagValidationError("ingestion status limit must be a positive integer");
+            if (options.cursor !== undefined)
+                requireSourceId(options.cursor);
+            const entries = [...(scopes.get(scopeKey(required))?.values() ?? [])]
+                .filter((entry) => entry.sourceId > (options.cursor ?? ""))
+                .sort((left, right) => left.sourceId.localeCompare(right.sourceId));
+            const page = entries.slice(0, limit);
+            const next = entries.length > page.length ? page.at(-1)?.sourceId : undefined;
+            return Object.freeze({ entries: Object.freeze(page), ...(next ? { nextCursor: next } : {}) });
+        },
+    };
+}
+export function ingestionStatus(scope, sourceId, state, bytes, chunks, error) {
+    const status = {
+        sourceId: requireSourceId(sourceId),
+        scope: requireScope(scope),
+        state,
+        bytes,
+        chunks,
+        ...(error ? { error } : {}),
+        updatedAt: new Date().toISOString(),
+    };
+    assertStatus(status, status.scope);
+    return Object.freeze(status);
+}
+function assertStatus(status, scope) {
+    requireSourceId(status.sourceId);
+    if (status.scope.tenantId !== scope.tenantId ||
+        status.scope.resourceId !== scope.resourceId ||
+        status.scope.corpusId !== scope.corpusId) {
+        throw new RagScopeError("ingestion status crossed tenant/resource/corpus boundary");
+    }
+    if (!["pending", "indexed", "failed", "partial"].includes(status.state))
+        throw new RagValidationError("ingestion status state is invalid");
+    if (!Number.isSafeInteger(status.bytes) || status.bytes < 0 || !Number.isSafeInteger(status.chunks) || status.chunks < 0) {
+        throw new RagValidationError("ingestion status bytes and chunks must be non-negative safe integers");
+    }
+    if (!Number.isFinite(Date.parse(status.updatedAt)))
+        throw new RagValidationError("ingestion status updatedAt must be an ISO timestamp");
+    if (status.error !== undefined && (typeof status.error !== "string" || byteLength(status.error) > 4_096)) {
+        throw new RagValidationError("ingestion status error must be a string <= 4096 bytes");
+    }
+}
+function scopeKey(scope) {
+    return `${scope.tenantId}\0${scope.resourceId}\0${scope.corpusId}`;
+}
+//# sourceMappingURL=ingestion-status.js.map

package/dist/limits.d.ts

@@ -4,6 +4,12 @@ export declare const DEFAULT_CHUNK_OVERLAP = 100;
 export declare const HARD_CHUNK_OVERLAP_CAP = 4096;
 export declare const DEFAULT_MAX_DOCUMENT_CHARS = 1048576;
 export declare const HARD_MAX_DOCUMENT_CHARS_CAP = 8388608;
+export declare const DEFAULT_MAX_DOCUMENT_BYTES = 1048576;
+export declare const HARD_MAX_DOCUMENT_BYTES_CAP: number;
+export declare const DEFAULT_MAX_PARSE_MS = 30000;
+export declare const HARD_MAX_PARSE_MS_CAP = 30000;
+export declare const DEFAULT_MAX_PDF_PAGES = 256;
+export declare const HARD_MAX_PDF_PAGES_CAP = 256;
 export declare const DEFAULT_MAX_CHUNKS = 2048;
 export declare const HARD_MAX_CHUNKS_CAP = 8192;
 export declare const DEFAULT_EMBED_BATCH_SIZE = 32;
@@ -19,10 +25,21 @@ export declare const HARD_MAX_CONTEXT_TOKENS_CAP = 8000;
 export declare const DEFAULT_MAX_METADATA_BYTES: number;
 export declare const HARD_MAX_METADATA_BYTES_CAP: number;
 export declare const DEFAULT_MAX_VECTOR_DIMENSIONS = 4096;
+export declare const DEFAULT_MAX_RERANK_BYTES: number;
+export declare const HARD_MAX_RERANK_BYTES_CAP: number;
+export declare const DEFAULT_MAX_RERANK_MS = 2000;
+export declare const HARD_MAX_RERANK_MS_CAP = 10000;
+export declare const DEFAULT_RERANK_CONCURRENCY = 2;
+export declare const HARD_RERANK_CONCURRENCY_CAP = 8;
+export declare const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
+export declare const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 export interface RagLimits {
     readonly chunkSize: number;
     readonly chunkOverlap: number;
     readonly maxDocumentChars: number;
+    readonly maxDocumentBytes: number;
+    readonly maxParseMs: number;
+    readonly maxPdfPages: number;
     readonly maxChunks: number;
     readonly embedBatchSize: number;
     readonly topK: number;
@@ -31,6 +48,10 @@ export interface RagLimits {
     readonly maxContextTokens: number;
     readonly maxMetadataBytes: number;
     readonly maxVectorDimensions: number;
+    readonly maxRerankBytes: number;
+    readonly maxRerankMs: number;
+    readonly rerankConcurrency: number;
+    readonly ingestionStatusPageSize: number;
 }
 export type RagLimitsInput = Partial<RagLimits>;
 export declare function resolveRagLimits(input?: RagLimitsInput): RagLimits;

package/dist/limits.js

@@ -5,6 +5,12 @@ export const DEFAULT_CHUNK_OVERLAP = 100;
 export const HARD_CHUNK_OVERLAP_CAP = 4_096;
 export const DEFAULT_MAX_DOCUMENT_CHARS = 1_048_576;
 export const HARD_MAX_DOCUMENT_CHARS_CAP = 8_388_608;
+export const DEFAULT_MAX_DOCUMENT_BYTES = 1_048_576;
+export const HARD_MAX_DOCUMENT_BYTES_CAP = 8 * 1024 * 1024;
+export const DEFAULT_MAX_PARSE_MS = 30_000;
+export const HARD_MAX_PARSE_MS_CAP = 30_000;
+export const DEFAULT_MAX_PDF_PAGES = 256;
+export const HARD_MAX_PDF_PAGES_CAP = 256;
 export const DEFAULT_MAX_CHUNKS = 2_048;
 export const HARD_MAX_CHUNKS_CAP = 8_192;
 export const DEFAULT_EMBED_BATCH_SIZE = 32;
@@ -20,6 +26,14 @@ export const HARD_MAX_CONTEXT_TOKENS_CAP = 8_000;
 export const DEFAULT_MAX_METADATA_BYTES = 16 * 1024;
 export const HARD_MAX_METADATA_BYTES_CAP = 64 * 1024;
 export const DEFAULT_MAX_VECTOR_DIMENSIONS = 4_096;
+export const DEFAULT_MAX_RERANK_BYTES = 64 * 1024;
+export const HARD_MAX_RERANK_BYTES_CAP = 256 * 1024;
+export const DEFAULT_MAX_RERANK_MS = 2_000;
+export const HARD_MAX_RERANK_MS_CAP = 10_000;
+export const DEFAULT_RERANK_CONCURRENCY = 2;
+export const HARD_RERANK_CONCURRENCY_CAP = 8;
+export const DEFAULT_INGESTION_STATUS_PAGE_SIZE = 50;
+export const HARD_INGESTION_STATUS_PAGE_SIZE_CAP = 200;
 function integer(value, fallback, cap, label, minimum = 1) {
     const resolved = value ?? fallback;
     if (!Number.isInteger(resolved) || resolved < minimum) {
@@ -42,6 +56,9 @@ export function resolveRagLimits(input = {}) {
         chunkSize,
         chunkOverlap,
         maxDocumentChars: integer(input.maxDocumentChars, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, "maxDocumentChars"),
+        maxDocumentBytes: integer(input.maxDocumentBytes, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, "maxDocumentBytes"),
+        maxParseMs: integer(input.maxParseMs, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, "maxParseMs"),
+        maxPdfPages: integer(input.maxPdfPages, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, "maxPdfPages"),
         maxChunks: integer(input.maxChunks, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, "maxChunks"),
         embedBatchSize: integer(input.embedBatchSize, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, "embedBatchSize"),
         topK,
@@ -50,6 +67,10 @@ export function resolveRagLimits(input = {}) {
         maxContextTokens: integer(input.maxContextTokens, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, "maxContextTokens"),
         maxMetadataBytes: integer(input.maxMetadataBytes, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, "maxMetadataBytes"),
         maxVectorDimensions: integer(input.maxVectorDimensions, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_VECTOR_DIMENSIONS, "maxVectorDimensions"),
+        maxRerankBytes: integer(input.maxRerankBytes, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, "maxRerankBytes"),
+        maxRerankMs: integer(input.maxRerankMs, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, "maxRerankMs"),
+        rerankConcurrency: integer(input.rerankConcurrency, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, "rerankConcurrency"),
+        ingestionStatusPageSize: integer(input.ingestionStatusPageSize, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, "ingestionStatusPageSize"),
     });
 }
 //# sourceMappingURL=limits.js.map

package/dist/loaders.d.ts

@@ -0,0 +1,24 @@
+import type { ResourceLoadContext, ResourceLoader } from "@arnilo/prism";
+import type { DocumentLoader } from "./types.js";
+export interface ResourceDocumentLoaderOptions {
+    readonly loader: ResourceLoader;
+    readonly context?: ResourceLoadContext;
+}
+export interface WebFetchDocument {
+    readonly citationId: string;
+    readonly provider: string;
+    readonly url: string;
+    readonly markdown: string;
+    readonly retrievedAt: string;
+    readonly untrusted: true;
+}
+export interface WebFetchAdapter {
+    fetch(url: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<WebFetchDocument>;
+}
+export interface WebFetchDocumentLoaderOptions {
+    readonly fetcher: WebFetchAdapter;
+}
+export declare function createResourceDocumentLoader(options: ResourceDocumentLoaderOptions): DocumentLoader;
+export declare function createWebFetchDocumentLoader(options: WebFetchDocumentLoaderOptions): DocumentLoader;

package/dist/loaders.js

@@ -0,0 +1,78 @@
+import { RagLimitError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { nonEmpty } from "./util.js";
+export function createResourceDocumentLoader(options) {
+    return {
+        async load(uri, loadOptions = {}) {
+            nonEmpty(uri, "uri");
+            const context = { ...options.context, ...(loadOptions.signal ? { signal: loadOptions.signal } : {}) };
+            const resource = await options.loader.load(uri, context);
+            if (resource.text === undefined && resource.data === undefined)
+                throw new RagValidationError("resource has no text or data");
+            const document = Object.freeze({
+                uri: resource.uri,
+                ...(resource.mediaType ? { mediaType: resource.mediaType } : {}),
+                ...(resource.text !== undefined ? { text: resource.text } : { data: new Uint8Array(resource.data) }),
+            });
+            assertDocumentBytes(document, loadOptions.maxBytes);
+            return document;
+        },
+    };
+}
+export function createWebFetchDocumentLoader(options) {
+    return {
+        async load(uri, loadOptions = {}) {
+            const url = publicWebUrl(uri);
+            const document = await options.fetcher.fetch(url, { signal: loadOptions.signal });
+            publicWebUrl(document.url);
+            if (!document.untrusted || !document.markdown)
+                throw new RagValidationError("web fetcher returned an invalid untrusted document");
+            const loaded = Object.freeze({
+                uri: document.url,
+                sourceId: document.citationId,
+                mediaType: "text/markdown",
+                text: document.markdown,
+                metadata: {
+                    web: {
+                        citationId: document.citationId,
+                        provider: document.provider,
+                        url: document.url,
+                        retrievedAt: document.retrievedAt,
+                    },
+                    untrusted: true,
+                },
+            });
+            assertDocumentBytes(loaded, loadOptions.maxBytes);
+            return loaded;
+        },
+    };
+}
+function assertDocumentBytes(document, maxBytes) {
+    const limit = resolveRagLimits({ maxDocumentBytes: maxBytes }).maxDocumentBytes;
+    const bytes = document.data ?? Buffer.from(document.text ?? "", "utf8");
+    if (bytes.byteLength > limit)
+        throw new RagLimitError(`document exceeds ${limit} bytes`);
+}
+function publicWebUrl(input) {
+    let url;
+    try {
+        url = new URL(input);
+    }
+    catch {
+        throw new RagValidationError("web document URI must be an absolute HTTP(S) URL");
+    }
+    const host = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+    if (!/^https?:$/u.test(url.protocol) || url.username || url.password || privateOrLocalHost(host)) {
+        throw new RagValidationError("web document URI must use a public hostname without credentials");
+    }
+    url.hash = "";
+    return url.toString();
+}
+function privateOrLocalHost(host) {
+    if (!host || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local"))
+        return true;
+    if (/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(host) || host.includes(":"))
+        return true; // Require fetcher DNS policy for hostnames; never fetch IP literals.
+    return false;
+}
+//# sourceMappingURL=loaders.js.map

package/dist/parsers.d.ts

@@ -0,0 +1,5 @@
+import type { Parser } from "./types.js";
+export declare const textParser: Parser;
+export declare const markdownParser: Parser;
+export declare const htmlParser: Parser;
+export declare const pdfParser: Parser;