@arnilo/prism-rag 0.0.14 → 0.0.16

package/dist/parsers.js

@@ -0,0 +1,101 @@
+import { RagLimitError, RagValidationError } from "./errors.js";
+import { resolveRagLimits } from "./limits.js";
+import { assertNotAborted } from "./util.js";
+export const textParser = { parse: (document, options) => parseText(document, options, ["text/plain"]) };
+export const markdownParser = { parse: (document, options) => parseText(document, options, ["text/markdown", "text/x-markdown"]) };
+export const htmlParser = {
+    parse: (document, options) => parseText(document, options, ["text/html", "application/xhtml+xml"], htmlToText),
+};
+export const pdfParser = { parse: parsePdf };
+async function parseText(document, options = {}, mediaTypes, transform = (text) => text) {
+    assertMediaType(document, mediaTypes);
+    const { bytes, limits, started } = boundedDocument(document, options);
+    assertNotAborted(options.signal);
+    let text;
+    try {
+        text = document.text ?? new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+    }
+    catch {
+        throw new RagValidationError("document is not valid UTF-8 text");
+    }
+    const parsed = transform(text);
+    assertParseTime(started, limits.maxParseMs);
+    assertNotAborted(options.signal);
+    return Object.freeze({ text: parsed, ...(document.metadata ? { metadata: document.metadata } : {}) });
+}
+async function parsePdf(document, options = {}) {
+    assertMediaType(document, ["application/pdf"]);
+    const { bytes, limits, started } = boundedDocument(document, options);
+    if (document.text !== undefined)
+        throw new RagValidationError("PDF parser requires binary document data");
+    const pdf = Buffer.from(bytes).toString("latin1");
+    if (!pdf.startsWith("%PDF-"))
+        throw new RagValidationError("document is not a PDF");
+    if (/\/Filter\s*\//u.test(pdf))
+        throw new RagValidationError("compressed PDFs require a host parser");
+    const pages = [...pdf.matchAll(/\/Type\s*\/Page\b/gu)].length;
+    if (pages > limits.maxPdfPages)
+        throw new RagLimitError(`PDF exceeds ${limits.maxPdfPages} pages`);
+    const blocks = pdf.match(/BT[\s\S]*?ET/gu) ?? [];
+    const text = blocks.flatMap(pdfBlockText).join("\n").trim();
+    if (!text)
+        throw new RagValidationError("PDF has no uncompressed text");
+    assertParseTime(started, limits.maxParseMs);
+    assertNotAborted(options.signal);
+    return Object.freeze({ text, ...(document.metadata ? { metadata: document.metadata } : {}) });
+}
+function boundedDocument(document, options) {
+    assertNotAborted(options.signal);
+    const limits = resolveRagLimits({
+        maxDocumentBytes: options.maxBytes,
+        maxParseMs: options.maxParseMs,
+        maxPdfPages: options.maxPages,
+    });
+    const bytes = document.data ?? Buffer.from(document.text ?? "", "utf8");
+    if (bytes.byteLength > limits.maxDocumentBytes)
+        throw new RagLimitError(`document exceeds ${limits.maxDocumentBytes} bytes`);
+    return { bytes, limits, started: Date.now() };
+}
+function assertMediaType(document, expected) {
+    if (document.mediaType && !expected.includes(document.mediaType.toLowerCase())) {
+        throw new RagValidationError(`parser does not accept ${document.mediaType}`);
+    }
+}
+function assertParseTime(started, maxParseMs) {
+    if (Date.now() - started > maxParseMs)
+        throw new RagLimitError(`document parsing exceeded ${maxParseMs}ms`);
+}
+function htmlToText(html) {
+    return decodeEntities(html
+        .replace(/<!--[\s\S]*?-->/gu, "")
+        .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/giu, "")
+        .replace(/<(?:br|p|div|li|h[1-6]|tr)\b[^>]*>/giu, "\n")
+        .replace(/<[^>]*>/gu, " "))
+        .replace(/[ \t]{2,}/gu, " ")
+        .replace(/[ \t]+\n/gu, "\n")
+        .replace(/\n[ \t]+/gu, "\n")
+        .replace(/\n{3,}/gu, "\n\n")
+        .trim();
+}
+function decodeEntities(text) {
+    const entities = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", "#39": "'" };
+    return text.replace(/&([a-z]+|#39);/giu, (_match, name) => entities[name.toLowerCase()] ?? `&${name};`);
+}
+function pdfBlockText(block) {
+    const strings = [];
+    for (const match of block.matchAll(/\((?:\\.|[^\\)])*\)\s*(?:Tj|['"])/gu))
+        strings.push(decodePdfString(match[0].replace(/\s*(?:Tj|['"])$/u, "")));
+    for (const match of block.matchAll(/\[([\s\S]*?)\]\s*TJ/gu)) {
+        for (const value of match[1].matchAll(/\((?:\\.|[^\\)])*\)/gu))
+            strings.push(decodePdfString(value[0]));
+    }
+    return strings.filter(Boolean);
+}
+function decodePdfString(value) {
+    return value.slice(1, -1).replace(/\\([0-7]{1,3}|[nrtbf()\\])/gu, (_match, escape) => {
+        if (/^[0-7]+$/u.test(escape))
+            return String.fromCharCode(Number.parseInt(escape, 8));
+        return { n: "\n", r: "\r", t: "\t", b: "\b", f: "\f", "(": "(", ")": ")", "\\": "\\" }[escape];
+    });
+}
+//# sourceMappingURL=parsers.js.map

package/dist/rerank.d.ts

@@ -0,0 +1,11 @@
+import { resolveRedactor } from "@arnilo/prism";
+import type { RagHit, Reranker } from "./types.js";
+export declare function rerankHits(query: string, hits: readonly RagHit[], options: {
+    readonly reranker: Reranker;
+    readonly maxBytes: number;
+    readonly maxMs: number;
+    readonly concurrency: number;
+    readonly signal?: AbortSignal;
+    readonly redactor?: Parameters<typeof resolveRedactor>[0];
+    readonly secrets?: Parameters<typeof resolveRedactor>[1];
+}): Promise<readonly RagHit[]>;

package/dist/rerank.js

@@ -0,0 +1,62 @@
+import { resolveRedactor } from "@arnilo/prism";
+import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
+import { assertNotAborted, byteLength } from "./util.js";
+const active = new WeakMap();
+export async function rerankHits(query, hits, options) {
+    assertNotAborted(options.signal);
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const safeQuery = redactor?.redact(query) ?? query;
+    const safeHits = hits.map((hit) => redactor?.redact(hit) ?? hit);
+    if (byteLength({ query: safeQuery, hits: safeHits }) > options.maxBytes) {
+        throw new RagLimitError(`reranker input exceeds ${options.maxBytes} bytes`);
+    }
+    const running = active.get(options.reranker) ?? 0;
+    if (running >= options.concurrency)
+        throw new RagLimitError(`reranker concurrency exceeds ${options.concurrency}`);
+    active.set(options.reranker, running + 1);
+    const release = () => {
+        const remaining = (active.get(options.reranker) ?? 1) - 1;
+        if (remaining)
+            active.set(options.reranker, remaining);
+        else
+            active.delete(options.reranker);
+    };
+    try {
+        const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => {
+            void operation.then(release, release);
+        }, options.maxMs, options.signal);
+        const originals = new Map(hits.map((hit) => [hit.id, hit]));
+        if (ordered.length !== hits.length || new Set(ordered.map((hit) => hit.id)).size !== hits.length) {
+            throw new RagValidationError("reranker must return each retrieved hit exactly once");
+        }
+        const output = ordered.map((hit) => originals.get(hit.id));
+        if (output.some((hit) => !hit))
+            throw new RagValidationError("reranker returned an unknown hit");
+        return Object.freeze(output);
+    }
+    catch (error) {
+        if (error instanceof RagAbortError || error instanceof RagLimitError || error instanceof RagValidationError)
+            throw error;
+        const message = error instanceof Error ? error.message : "reranker failed";
+        throw new RagValidationError(`reranker failed: ${redactor?.redact(message) ?? message}`);
+    }
+}
+async function boundedRerank(run, onStart, maxMs, signal) {
+    const controller = new AbortController();
+    const onAbort = () => controller.abort();
+    signal?.addEventListener("abort", onAbort, { once: true });
+    const timeout = setTimeout(() => controller.abort(), maxMs);
+    const aborted = new Promise((_, reject) => controller.signal.addEventListener("abort", () => {
+        reject(signal?.aborted ? new RagAbortError() : new RagLimitError(`reranker exceeded ${maxMs}ms`));
+    }, { once: true }));
+    const operation = Promise.resolve().then(() => run(controller.signal));
+    onStart(operation);
+    try {
+        return await Promise.race([operation, aborted]);
+    }
+    finally {
+        clearTimeout(timeout);
+        signal?.removeEventListener("abort", onAbort);
+    }
+}
+//# sourceMappingURL=rerank.js.map

package/dist/retrieve.js

@@ -1,11 +1,13 @@
-import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { resolveRedactor } from "@arnilo/prism";
 import { RagScopeError, RagValidationError } from "./errors.js";
-import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, resolveRedactor, truncateUtf8, } from "./util.js";
+import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { rerankHits } from "./rerank.js";
+import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
+const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
 export async function retrieveContext(query, options) {
     nonEmpty(query, "query");
-    if (query.length > HARD_CHUNK_SIZE_CAP) {
+    if (query.length > HARD_CHUNK_SIZE_CAP)
         throw new RagValidationError(`query exceeds ${HARD_CHUNK_SIZE_CAP} characters`);
-    }
     const scope = requireScope(options.scope);
     const limits = resolveRagLimits({
         topK: options.topK,
@@ -14,10 +16,13 @@ export async function retrieveContext(query, options) {
         maxContextTokens: options.maxContextTokens,
         maxMetadataBytes: options.maxMetadataBytes,
         maxVectorDimensions: options.maxVectorDimensions,
+        maxRerankBytes: options.maxRerankBytes,
+        maxRerankMs: options.maxRerankMs,
+        rerankConcurrency: options.rerankConcurrency,
     });
-    if (!Number.isInteger(options.embedder.dimensions)
-        || options.embedder.dimensions <= 0
-        || options.embedder.dimensions > limits.maxVectorDimensions) {
+    if (!Number.isInteger(options.embedder.dimensions) ||
+        options.embedder.dimensions <= 0 ||
+        options.embedder.dimensions > limits.maxVectorDimensions) {
         throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
     }
     if (options.filter)
@@ -27,10 +32,10 @@ export async function retrieveContext(query, options) {
     assertNotAborted(options.signal);
     const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
     const embedding = vectors[0];
-    if (vectors.length !== 1
-        || !embedding
-        || embedding.length !== options.embedder.dimensions
-        || embedding.some((value) => !Number.isFinite(value))) {
+    if (vectors.length !== 1 ||
+        !embedding ||
+        embedding.length !== options.embedder.dimensions ||
+        embedding.some((value) => !Number.isFinite(value))) {
         throw new RagValidationError("embedder returned invalid query vector");
     }
     const candidates = await options.store.query({
@@ -42,6 +47,26 @@ export async function retrieveContext(query, options) {
         signal: options.signal,
     });
     assertNotAborted(options.signal);
+    const retrievedAt = new Date().toISOString();
+    const retrieved = [];
+    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
+        assertScope(scope, candidate);
+        const parsed = parseHit(candidate, retrieved.length, retrievedAt);
+        if (!matchesFilter(parsed.metadata, options.filter))
+            continue;
+        retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
+    }
+    const ranked = options.reranker
+        ? await rerankHits(safeQuery, retrieved, {
+            reranker: options.reranker,
+            maxBytes: limits.maxRerankBytes,
+            maxMs: limits.maxRerankMs,
+            concurrency: limits.rerankConcurrency,
+            signal: options.signal,
+            redactor: options.redactor,
+            secrets: options.secrets,
+        })
+        : retrieved;
     const hits = [];
     const citations = [];
     const rendered = [];
@@ -49,15 +74,10 @@ export async function retrieveContext(query, options) {
     let usedBytes = 0;
     let usedChars = 0;
     let truncated = false;
-    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
-        assertScope(scope, candidate);
-        const parsed = parseHit(candidate);
-        if (!matchesFilter(parsed.metadata, options.filter))
-            continue;
+    for (const hit of ranked) {
         if (hits.length >= limits.topK)
             break;
-        const safe = redactor?.redact(parsed) ?? parsed;
-        const prefix = `[${parsed.citationId}] `;
+        const prefix = `[${hit.citationId}] `;
         const separator = rendered.length ? "\n\n" : "";
         const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
         const availableChars = maxChars - usedChars - separator.length - prefix.length;
@@ -65,61 +85,66 @@ export async function retrieveContext(query, options) {
             truncated = true;
             break;
         }
-        let text = safe.text.slice(0, availableChars);
+        let text = hit.text.slice(0, availableChars);
         text = truncateUtf8(text, availableBytes);
         if (!text) {
             truncated = true;
             break;
         }
-        if (text.length < safe.text.length)
+        if (text.length < hit.text.length)
             truncated = true;
-        const hit = Object.freeze({ ...safe, text });
+        const renderedHit = Object.freeze({ ...hit, text });
         const citation = Object.freeze({
-            id: hit.citationId,
-            sourceId: hit.sourceId,
-            chunkId: hit.id,
-            ...(hit.metadata ? { metadata: hit.metadata } : {}),
+            id: renderedHit.citationId,
+            sourceId: renderedHit.sourceId,
+            chunkId: renderedHit.id,
+            provenance: renderedHit.provenance,
+            trust: renderedHit.trust,
+            ...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
         });
         const block = `${separator}${prefix}${text}`;
         rendered.push(block);
         usedBytes += byteLength(block);
         usedChars += block.length;
-        hits.push(hit);
+        hits.push(renderedHit);
         citations.push(citation);
         if (truncated)
             break;
     }
     return Object.freeze({
         query: safeQuery,
+        trust: RETRIEVED_CONTENT_TRUST,
         text: rendered.join(""),
         hits: Object.freeze(hits),
         citations: Object.freeze(citations),
         truncated,
     });
 }
-function parseHit(hit) {
+function parseHit(hit, retrievalRank, retrievedAt) {
     const metadata = hit.metadata;
     const rag = metadata?._rag;
     if (!isJsonObject(rag))
         throw new RagScopeError("vector hit is missing RAG source metadata");
     const sourceId = requireSourceId(rag.sourceId);
     const citationId = nonEmpty(rag.citationId, "metadata._rag.citationId");
-    if (!Number.isInteger(rag.chunkIndex)
-        || Number(rag.chunkIndex) < 0
-        || !Number.isInteger(rag.start)
-        || Number(rag.start) < 0
-        || !Number.isInteger(rag.end)
-        || Number(rag.end) < Number(rag.start)
-        || !Number.isFinite(hit.score)) {
+    if (!Number.isInteger(rag.chunkIndex) ||
+        Number(rag.chunkIndex) < 0 ||
+        !Number.isInteger(rag.start) ||
+        Number(rag.start) < 0 ||
+        !Number.isInteger(rag.end) ||
+        Number(rag.end) < Number(rag.start) ||
+        !Number.isFinite(hit.score)) {
         throw new RagValidationError("vector hit has invalid RAG offsets");
     }
-    if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`)) {
+    if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`))
         throw new RagValidationError("vector hit has inconsistent citation identity");
-    }
     const userMetadata = {};
     for (const [key, value] of Object.entries(metadata ?? {}))
         if (key !== "_rag")
             userMetadata[key] = value;
+    const web = isJsonObject(userMetadata.web) ? userMetadata.web : undefined;
+    const provider = typeof web?.provider === "string" && web.provider.trim() ? web.provider : "host";
+    const provenance = Object.freeze({ sourceId, chunkId: hit.id, citationId, provider, retrieval: "vector", retrievedAt });
     return {
         id: hit.id,
         citationId,
@@ -129,6 +154,9 @@ function parseHit(hit) {
         end: rag.end,
         text: hit.text,
         score: hit.score,
+        retrievalRank,
+        provenance,
+        trust: RETRIEVED_CONTENT_TRUST,
         ...(Object.keys(userMetadata).length ? { metadata: userMetadata } : {}),
     };
 }

package/dist/sources.d.ts

@@ -0,0 +1,9 @@
+import type { DeleteSourceOptions, ReplaceDocumentOptions, ReplaceSourceOptions } from "./types.js";
+export interface SourceMutationResult {
+    readonly sourceId: string;
+    readonly deleted: number;
+    readonly indexed: number;
+}
+export declare function replaceSource(options: ReplaceSourceOptions): Promise<SourceMutationResult>;
+export declare function deleteSource(options: DeleteSourceOptions): Promise<SourceMutationResult>;
+export declare function replaceDocument(options: ReplaceDocumentOptions): Promise<SourceMutationResult>;

package/dist/sources.js

@@ -0,0 +1,118 @@
+import { resolveRedactor } from "@arnilo/prism";
+import { chunkText } from "./chunk.js";
+import { RagScopeError, RagValidationError } from "./errors.js";
+import { indexChunkBatches } from "./indexing.js";
+import { ingestionStatus } from "./ingestion-status.js";
+import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
+export async function replaceSource(options) {
+    const sourceId = requireSourceId(options.sourceId);
+    const scope = requireScope(options.scope);
+    assertTransactionalStore(options.store);
+    if (options.chunks.some((chunk) => chunk.sourceId !== sourceId)) {
+        throw new RagValidationError("replaceSource chunks must all belong to sourceId");
+    }
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const totalBytes = options.chunks.reduce((total, chunk) => total + byteLength(redactor?.redact(chunk.text) ?? chunk.text), 0);
+    const setStatus = async (state, error) => {
+        if (!options.statusStore)
+            return;
+        const message = error instanceof Error ? error.message : error === undefined ? undefined : "source replacement failed";
+        await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? (redactor?.redact(message) ?? message) : undefined));
+    };
+    await setStatus("pending");
+    try {
+        const staged = [];
+        const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
+            staged.push(...records);
+        });
+        assertNotAborted(options.signal);
+        const result = await options.store.transaction(async (store) => {
+            const previous = await sourceRecords(store, sourceId, scope, options.signal);
+            assertNotAborted(options.signal);
+            if (previous.length) {
+                await store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: previous.map((record) => record.id) }, { signal: options.signal });
+            }
+            if (staged.length)
+                await store.upsert(staged, { signal: options.signal });
+            return Object.freeze({ sourceId, deleted: previous.length, indexed: indexed.indexed });
+        }, { signal: options.signal });
+        await setStatus("indexed");
+        return result;
+    }
+    catch (error) {
+        await setStatus("failed", error);
+        throw error;
+    }
+}
+export async function deleteSource(options) {
+    const sourceId = requireSourceId(options.sourceId);
+    const scope = requireScope(options.scope);
+    assertSourceStore(options.store);
+    const records = await sourceRecords(options.store, sourceId, scope, options.signal);
+    assertNotAborted(options.signal);
+    const deleted = records.length
+        ? await options.store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: records.map((record) => record.id) }, { signal: options.signal })
+        : 0;
+    await options.statusStore?.delete(scope, sourceId);
+    return Object.freeze({ sourceId, deleted, indexed: 0 });
+}
+export async function replaceDocument(options) {
+    const loaded = await options.loader.load(options.uri, {
+        ...options.loaderOptions,
+        signal: options.signal ?? options.loaderOptions?.signal,
+    });
+    assertNotAborted(options.signal);
+    const parsed = await options.parser.parse(loaded, {
+        ...options.parserOptions,
+        signal: options.signal ?? options.parserOptions?.signal,
+    });
+    assertNotAborted(options.signal);
+    const sourceId = options.sourceId ?? loaded.sourceId;
+    if (!sourceId)
+        throw new RagValidationError("replaceDocument requires sourceId when its loader does not provide one");
+    const { metadata, ...chunk } = options.chunk ?? {};
+    const mergedMetadata = mergeMetadata(parsed.metadata, metadata);
+    const chunks = (options.chunker ?? chunkText)(parsed.text, {
+        ...chunk,
+        sourceId,
+        ...(mergedMetadata ? { metadata: mergedMetadata } : {}),
+    });
+    return replaceSource({
+        ...options,
+        sourceId,
+        chunks,
+    });
+}
+function mergeMetadata(parser, supplied) {
+    if (!parser && !supplied)
+        return undefined;
+    return { ...supplied, ...parser }; // Loader trust metadata wins over caller-supplied metadata.
+}
+async function sourceRecords(store, sourceId, scope, signal) {
+    assertNotAborted(signal);
+    const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, {
+        signal,
+    });
+    for (const record of records) {
+        if (record.tenantId !== scope.tenantId || record.resourceId !== scope.resourceId || record.threadId !== scope.corpusId) {
+            throw new RagScopeError("source lookup crossed tenant/resource/corpus boundary");
+        }
+        const rag = record.metadata?._rag;
+        if (typeof rag !== "object" || rag === null || Array.isArray(rag) || rag.sourceId !== sourceId) {
+            throw new RagScopeError("source lookup returned a different source");
+        }
+    }
+    return records;
+}
+function assertSourceStore(store) {
+    if (!store || typeof store !== "object" || typeof store.getBySource !== "function") {
+        throw new RagValidationError("source deletion requires a scoped source-aware vector store");
+    }
+}
+function assertTransactionalStore(store) {
+    if (!store || typeof store !== "object" || typeof store.transaction !== "function") {
+        throw new RagValidationError("atomic source replacement requires a transactional vector store");
+    }
+    assertSourceStore(store);
+}
+//# sourceMappingURL=sources.js.map

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
-import type { Embedder, VectorStore } from "@arnilo/prism-memory";
+import type { Embedder, MemoryVectorRecord, VectorStore } from "@arnilo/prism-memory";
 export interface RagScope {
     readonly tenantId: string;
     readonly resourceId: string;
@@ -23,6 +23,80 @@ export interface ChunkOptions {
     readonly maxDocumentChars?: number;
     readonly maxChunks?: number;
 }
+export interface LoadedDocument {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly mediaType?: string;
+    readonly text?: string;
+    readonly data?: Uint8Array;
+    readonly metadata?: JsonObject;
+}
+export interface ParsedDocument {
+    readonly text: string;
+    readonly metadata?: JsonObject;
+}
+export interface DocumentLoadOptions {
+    readonly signal?: AbortSignal;
+    readonly maxBytes?: number;
+}
+export interface DocumentParseOptions extends DocumentLoadOptions {
+    readonly maxParseMs?: number;
+    readonly maxPages?: number;
+}
+export interface DocumentLoader {
+    load(uri: string, options?: DocumentLoadOptions): Promise<LoadedDocument>;
+}
+export interface Parser {
+    parse(document: LoadedDocument, options?: DocumentParseOptions): Promise<ParsedDocument>;
+}
+export type Chunker = (text: string, options: ChunkOptions) => readonly RagChunk[];
+export interface SourceVectorStore extends VectorStore {
+    getBySource(scope: {
+        readonly tenantId: string;
+        readonly resourceId: string;
+        readonly threadId: string;
+    }, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<readonly MemoryVectorRecord[]>;
+}
+export interface TransactionalVectorStore extends SourceVectorStore {
+    transaction<T>(operation: (store: SourceVectorStore) => Promise<T>, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<T>;
+}
+export type IngestionState = "pending" | "indexed" | "failed" | "partial";
+export interface IngestionStatus {
+    readonly sourceId: string;
+    readonly scope: RagScope;
+    readonly state: IngestionState;
+    readonly bytes: number;
+    readonly chunks: number;
+    readonly error?: string;
+    readonly updatedAt: string;
+}
+export interface IngestionStatusStore {
+    set(status: IngestionStatus, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    delete(scope: RagScope, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    list(scope: RagScope, options: {
+        readonly limit: number;
+        readonly cursor?: string;
+        readonly signal?: AbortSignal;
+    }): Promise<{
+        readonly entries: readonly IngestionStatus[];
+        readonly nextCursor?: string;
+    }>;
+}
+export interface IngestionStatusQuery {
+    readonly store: IngestionStatusStore;
+    readonly scope: RagScope;
+    readonly limit?: number;
+    readonly cursor?: string;
+    readonly signal?: AbortSignal;
+}
 export interface IndexChunksOptions {
     readonly chunks: readonly RagChunk[];
     readonly embedder: Embedder;
@@ -35,20 +109,68 @@ export interface IndexChunksOptions {
     readonly maxMetadataBytes?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
+    readonly statusStore?: IngestionStatusStore;
     readonly signal?: AbortSignal;
 }
 export interface IndexChunksResult {
     readonly indexed: number;
     readonly sourceIds: readonly string[];
 }
+export interface ReplaceSourceOptions extends Omit<IndexChunksOptions, "chunks" | "store"> {
+    readonly sourceId: string;
+    readonly chunks: readonly RagChunk[];
+    readonly store: TransactionalVectorStore;
+}
+export interface DeleteSourceOptions {
+    readonly sourceId: string;
+    readonly store: SourceVectorStore;
+    readonly scope: RagScope;
+    readonly statusStore?: IngestionStatusStore;
+    readonly signal?: AbortSignal;
+}
+export interface ReplaceDocumentOptions extends Omit<ReplaceSourceOptions, "sourceId" | "chunks"> {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly loader: DocumentLoader;
+    readonly parser: Parser;
+    readonly chunker?: Chunker;
+    readonly chunk?: Omit<ChunkOptions, "sourceId">;
+    readonly loaderOptions?: DocumentLoadOptions;
+    readonly parserOptions?: DocumentParseOptions;
+}
+export interface RagProvenance {
+    readonly sourceId: string;
+    readonly chunkId: string;
+    readonly citationId: string;
+    readonly provider: string;
+    readonly retrieval: "vector";
+    readonly retrievedAt: string;
+}
+export interface RagContentTrust {
+    readonly untrusted: true;
+    readonly inert: true;
+    readonly injectionCapable: true;
+}
 export interface RagCitation {
     readonly id: string;
     readonly sourceId: string;
     readonly chunkId: string;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
     readonly metadata?: JsonObject;
 }
 export interface RagHit extends RagChunk {
     readonly score: number;
+    readonly retrievalRank: number;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
+}
+export interface Reranker {
+    rerank(input: {
+        readonly query: string;
+        readonly hits: readonly RagHit[];
+        readonly signal?: AbortSignal;
+    }): Promise<readonly RagHit[]>;
 }
 export interface RetrieveContextOptions {
     readonly embedder: Embedder;
@@ -61,12 +183,17 @@ export interface RetrieveContextOptions {
     readonly maxContextTokens?: number;
     readonly maxMetadataBytes?: number;
     readonly maxVectorDimensions?: number;
+    readonly reranker?: Reranker;
+    readonly maxRerankBytes?: number;
+    readonly maxRerankMs?: number;
+    readonly rerankConcurrency?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
     readonly signal?: AbortSignal;
 }
 export interface RagContextResult {
     readonly query: string;
+    readonly trust: RagContentTrust;
     readonly text: string;
     readonly hits: readonly RagHit[];
     readonly citations: readonly RagCitation[];