@arnilo/prism-rag 0.0.5

package/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## [Unreleased]
+
+## [0.0.5] - 2026-07-16
+
+- Added optional bounded text/Markdown chunking, vector indexing/retrieval, and stable citations.
+
+
+## [0.0.4] - 2026-07-14
+
+- Initial release: bounded deterministic text/Markdown chunking, batch vector indexing, filtered retrieval, stable citations, and ContextProvider integration.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Prism contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,38 @@
+# @arnilo/prism-rag
+
+Optional bounded text/Markdown retrieval-augmented generation primitives for Prism. Reuses `Embedder` and `VectorStore` from `@arnilo/prism-memory`; no document framework, network loader, or core activation.
+
+## Install
+
+```bash
+npm install @arnilo/prism-rag @arnilo/prism-memory @arnilo/prism
+```
+
+## Usage
+
+```ts
+import { createHashEmbedder, createMemoryVectorStore } from "@arnilo/prism-memory";
+import { chunkMarkdown, createRagContextProvider, indexChunks } from "@arnilo/prism-rag";
+
+const embedder = createHashEmbedder(); // demo/test only
+const store = createMemoryVectorStore();
+const scope = { tenantId: "t1", resourceId: "docs", corpusId: "handbook" };
+const chunks = chunkMarkdown("# Approval\n\nRecheck policy before side effects.", {
+  sourceId: "security-guide",
+});
+await indexChunks({ chunks, embedder, store, scope });
+const context = createRagContextProvider({ embedder, store, scope, topK: 4 });
+```
+
+## API
+
+- `chunkText()` / `chunkMarkdown()` — deterministic boundary-aware character chunks with overlap and stable citations.
+- `indexChunks()` — bounded batch embedding and scoped vector upsert.
+- `retrieveContext()` — bounded candidate query, shallow metadata filter, top-K hits, and citation rendering.
+- `createRagContextProvider()` — explicit inert context injection through Prism's existing seam.
+
+## Security
+
+Every operation requires tenant/resource/corpus scope. Configure `redactor` or `secrets` before external embedding/persistence. Package performs no I/O; load remote/local sources through host-owned resource/media policies. Retrieved text is untrusted context and grants no tools or permissions.
+
+See [RAG](../../docs/rag.md).

package/dist/chunk.d.ts

@@ -0,0 +1,3 @@
+import type { RagChunk, ChunkOptions } from "./types.js";
+export declare function chunkText(text: string, options: ChunkOptions): readonly RagChunk[];
+export declare function chunkMarkdown(markdown: string, options: ChunkOptions): readonly RagChunk[];

package/dist/chunk.js

@@ -0,0 +1,67 @@
+import { resolveRagLimits } from "./limits.js";
+import { RagLimitError } from "./errors.js";
+import { assertBytes, requireSourceId } from "./util.js";
+export function chunkText(text, options) {
+    return chunkDocument(text, options, false);
+}
+export function chunkMarkdown(markdown, options) {
+    return chunkDocument(markdown, options, true);
+}
+function chunkDocument(text, options, markdown) {
+    const sourceId = requireSourceId(options.sourceId);
+    const limits = resolveRagLimits({
+        chunkSize: options.size,
+        chunkOverlap: options.overlap,
+        maxDocumentChars: options.maxDocumentChars,
+        maxChunks: options.maxChunks,
+    });
+    if (text.length > limits.maxDocumentChars) {
+        throw new RagLimitError(`document exceeds ${limits.maxDocumentChars} characters`);
+    }
+    if (options.metadata)
+        assertBytes(options.metadata, 64 * 1024, "chunk metadata");
+    if (!text.trim())
+        return Object.freeze([]);
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        while (start < text.length && /\s/.test(text[start]))
+            start += 1;
+        if (start >= text.length)
+            break;
+        const ceiling = Math.min(start + limits.chunkSize, text.length);
+        const end = ceiling === text.length ? ceiling : preferredEnd(text, start, ceiling, markdown);
+        const raw = text.slice(start, end).trimEnd();
+        if (raw) {
+            const index = chunks.length;
+            const citationId = `${sourceId}#${String(index + 1).padStart(4, "0")}`;
+            chunks.push(Object.freeze({
+                id: citationId,
+                citationId,
+                sourceId,
+                index,
+                start,
+                end: start + raw.length,
+                text: raw,
+                ...(options.metadata ? { metadata: Object.freeze({ ...options.metadata }) } : {}),
+            }));
+            if (chunks.length > limits.maxChunks)
+                throw new RagLimitError(`chunk count exceeds ${limits.maxChunks}`);
+        }
+        if (end >= text.length)
+            break;
+        start = Math.max(start + 1, end - limits.chunkOverlap);
+    }
+    return Object.freeze(chunks);
+}
+function preferredEnd(text, start, ceiling, markdown) {
+    const floor = start + Math.floor((ceiling - start) / 2);
+    const candidates = markdown ? ["\n#", "\n\n", "\n", " "] : ["\n\n", "\n", " "];
+    for (const separator of candidates) {
+        const found = text.lastIndexOf(separator, ceiling);
+        if (found >= floor)
+            return separator === "\n#" ? found : found + separator.length;
+    }
+    return ceiling;
+}
+//# sourceMappingURL=chunk.js.map

package/dist/context.d.ts

@@ -0,0 +1,3 @@
+import type { ContextProvider } from "@arnilo/prism";
+import type { RagContextProviderOptions } from "./types.js";
+export declare function createRagContextProvider(options: RagContextProviderOptions): ContextProvider;

package/dist/context.js

@@ -0,0 +1,25 @@
+import { retrieveContext } from "./retrieve.js";
+import { latestUserText } from "./util.js";
+export function createRagContextProvider(options) {
+    return {
+        name: options.name ?? "rag",
+        async resolve(context) {
+            context.signal?.throwIfAborted();
+            const query = typeof options.query === "function"
+                ? options.query({ messages: context.messages })
+                : options.query ?? latestUserText(context.messages);
+            if (!query?.trim())
+                return [];
+            const result = await retrieveContext(query, { ...options, signal: context.signal });
+            if (!result.text)
+                return [];
+            return [{
+                    id: `${options.name ?? "rag"}:context`,
+                    title: options.title ?? "Retrieved context",
+                    content: result.text,
+                    metadata: { citations: result.citations, inert: true },
+                }];
+        },
+    };
+}
+//# sourceMappingURL=context.js.map

package/dist/errors.d.ts

@@ -0,0 +1,16 @@
+export declare class RagError extends Error {
+    readonly code: string;
+    constructor(message: string, code?: string);
+}
+export declare class RagValidationError extends RagError {
+    constructor(message: string);
+}
+export declare class RagLimitError extends RagError {
+    constructor(message: string);
+}
+export declare class RagScopeError extends RagError {
+    constructor(message: string);
+}
+export declare class RagAbortError extends RagError {
+    constructor();
+}

package/dist/errors.js

@@ -0,0 +1,33 @@
+export class RagError extends Error {
+    code;
+    constructor(message, code = "ERR_PRISM_RAG") {
+        super(message);
+        this.name = "RagError";
+        this.code = code;
+    }
+}
+export class RagValidationError extends RagError {
+    constructor(message) {
+        super(message, "ERR_PRISM_RAG_VALIDATION");
+        this.name = "RagValidationError";
+    }
+}
+export class RagLimitError extends RagError {
+    constructor(message) {
+        super(message, "ERR_PRISM_RAG_LIMIT");
+        this.name = "RagLimitError";
+    }
+}
+export class RagScopeError extends RagError {
+    constructor(message) {
+        super(message, "ERR_PRISM_RAG_SCOPE");
+        this.name = "RagScopeError";
+    }
+}
+export class RagAbortError extends RagError {
+    constructor() {
+        super("RAG operation aborted", "ERR_PRISM_RAG_ABORTED");
+        this.name = "AbortError";
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
+export type { RagLimits, RagLimitsInput } from "./limits.js";
+export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export { chunkMarkdown, chunkText } from "./chunk.js";
+export { indexChunks } from "./indexing.js";
+export { retrieveContext } from "./retrieve.js";
+export { createRagContextProvider } from "./context.js";
+export type { ChunkOptions, IndexChunksOptions, IndexChunksResult, RagChunk, RagCitation, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagScope, RetrieveContextOptions, } from "./types.js";
+export declare const packageName = "@arnilo/prism-rag";

package/dist/index.js

@@ -0,0 +1,8 @@
+export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, resolveRagLimits, } from "./limits.js";
+export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export { chunkMarkdown, chunkText } from "./chunk.js";
+export { indexChunks } from "./indexing.js";
+export { retrieveContext } from "./retrieve.js";
+export { createRagContextProvider } from "./context.js";
+export const packageName = "@arnilo/prism-rag";
+//# sourceMappingURL=index.js.map

package/dist/indexing.d.ts

@@ -0,0 +1,2 @@
+import type { IndexChunksOptions, IndexChunksResult } from "./types.js";
+export declare function indexChunks(options: IndexChunksOptions): Promise<IndexChunksResult>;

package/dist/indexing.js

@@ -0,0 +1,89 @@
+import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { RagValidationError } from "./errors.js";
+import { assertBytes, assertNotAborted, nonEmpty, requireScope, requireSourceId, resolveRedactor } from "./util.js";
+// ponytail: stable IDs make identical retries idempotent; replacing a source with
+// fewer chunks requires host deletion of stale source IDs before this generic upsert.
+export async function indexChunks(options) {
+    const scope = requireScope(options.scope);
+    const limits = resolveRagLimits({
+        embedBatchSize: options.batchSize,
+        maxChunks: options.maxChunks,
+        chunkSize: options.maxChunkChars ?? HARD_CHUNK_SIZE_CAP,
+        maxVectorDimensions: options.maxVectorDimensions,
+        maxMetadataBytes: options.maxMetadataBytes,
+    });
+    if (!Number.isInteger(options.embedder.dimensions)
+        || options.embedder.dimensions <= 0
+        || options.embedder.dimensions > limits.maxVectorDimensions) {
+        throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
+    }
+    if (options.chunks.length > limits.maxChunks) {
+        throw new RagValidationError(`chunk count exceeds ${limits.maxChunks}`);
+    }
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const sourceIds = new Set();
+    const chunkIds = new Set();
+    for (const chunk of options.chunks) {
+        nonEmpty(chunk.id, "chunk.id");
+        requireSourceId(chunk.sourceId);
+        if (chunk.id !== chunk.citationId || !chunk.citationId.startsWith(`${chunk.sourceId}#`)) {
+            throw new RagValidationError("chunk has inconsistent citation identity");
+        }
+        if (chunkIds.has(chunk.id))
+            throw new RagValidationError(`duplicate chunk id: ${chunk.id}`);
+        if (!Number.isInteger(chunk.index)
+            || chunk.index < 0
+            || !Number.isInteger(chunk.start)
+            || chunk.start < 0
+            || !Number.isInteger(chunk.end)
+            || chunk.end < chunk.start) {
+            throw new RagValidationError("chunk has invalid index or offsets");
+        }
+        if (chunk.text.length > limits.chunkSize)
+            throw new RagValidationError(`chunk text exceeds ${limits.chunkSize} characters`);
+        chunkIds.add(chunk.id);
+    }
+    for (let offset = 0; offset < options.chunks.length; offset += limits.embedBatchSize) {
+        assertNotAborted(options.signal);
+        const batch = options.chunks.slice(offset, offset + limits.embedBatchSize);
+        const texts = batch.map((chunk) => redactor?.redact(chunk.text) ?? chunk.text);
+        const vectors = await options.embedder.embed(texts, { signal: options.signal });
+        if (vectors.length !== batch.length)
+            throw new RagValidationError("embedder returned unexpected vector count");
+        const records = batch.map((chunk, index) => {
+            const embedding = vectors[index];
+            if (embedding.length !== options.embedder.dimensions
+                || embedding.some((value) => !Number.isFinite(value))) {
+                throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
+            }
+            const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? (chunk.metadata ?? {});
+            const metadata = {
+                ...safeMetadata,
+                _rag: {
+                    sourceId: chunk.sourceId,
+                    citationId: chunk.citationId,
+                    chunkIndex: chunk.index,
+                    start: chunk.start,
+                    end: chunk.end,
+                },
+            };
+            assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
+            sourceIds.add(chunk.sourceId);
+            return {
+                id: chunk.id,
+                tenantId: scope.tenantId,
+                resourceId: scope.resourceId,
+                threadId: scope.corpusId,
+                text: texts[index],
+                embedding,
+                sequence: chunk.index,
+                metadata: metadata,
+                createdAt: new Date(0).toISOString(),
+            };
+        });
+        assertNotAborted(options.signal);
+        await options.store.upsert(records, { signal: options.signal });
+    }
+    return Object.freeze({ indexed: options.chunks.length, sourceIds: Object.freeze([...sourceIds].sort()) });
+}
+//# sourceMappingURL=indexing.js.map

package/dist/limits.d.ts

@@ -0,0 +1,36 @@
+export declare const DEFAULT_CHUNK_SIZE = 1000;
+export declare const HARD_CHUNK_SIZE_CAP = 16384;
+export declare const DEFAULT_CHUNK_OVERLAP = 100;
+export declare const HARD_CHUNK_OVERLAP_CAP = 4096;
+export declare const DEFAULT_MAX_DOCUMENT_CHARS = 1048576;
+export declare const HARD_MAX_DOCUMENT_CHARS_CAP = 8388608;
+export declare const DEFAULT_MAX_CHUNKS = 2048;
+export declare const HARD_MAX_CHUNKS_CAP = 8192;
+export declare const DEFAULT_EMBED_BATCH_SIZE = 32;
+export declare const HARD_EMBED_BATCH_SIZE_CAP = 128;
+export declare const DEFAULT_TOP_K = 5;
+export declare const HARD_TOP_K_CAP = 32;
+export declare const DEFAULT_QUERY_CANDIDATES = 20;
+export declare const HARD_QUERY_CANDIDATES_CAP = 128;
+export declare const DEFAULT_MAX_RESULT_BYTES: number;
+export declare const HARD_MAX_RESULT_BYTES_CAP: number;
+export declare const DEFAULT_MAX_CONTEXT_TOKENS = 2000;
+export declare const HARD_MAX_CONTEXT_TOKENS_CAP = 8000;
+export declare const DEFAULT_MAX_METADATA_BYTES: number;
+export declare const HARD_MAX_METADATA_BYTES_CAP: number;
+export declare const DEFAULT_MAX_VECTOR_DIMENSIONS = 4096;
+export interface RagLimits {
+    readonly chunkSize: number;
+    readonly chunkOverlap: number;
+    readonly maxDocumentChars: number;
+    readonly maxChunks: number;
+    readonly embedBatchSize: number;
+    readonly topK: number;
+    readonly queryCandidates: number;
+    readonly maxResultBytes: number;
+    readonly maxContextTokens: number;
+    readonly maxMetadataBytes: number;
+    readonly maxVectorDimensions: number;
+}
+export type RagLimitsInput = Partial<RagLimits>;
+export declare function resolveRagLimits(input?: RagLimitsInput): RagLimits;

package/dist/limits.js

@@ -0,0 +1,55 @@
+import { RagLimitError } from "./errors.js";
+export const DEFAULT_CHUNK_SIZE = 1_000;
+export const HARD_CHUNK_SIZE_CAP = 16_384;
+export const DEFAULT_CHUNK_OVERLAP = 100;
+export const HARD_CHUNK_OVERLAP_CAP = 4_096;
+export const DEFAULT_MAX_DOCUMENT_CHARS = 1_048_576;
+export const HARD_MAX_DOCUMENT_CHARS_CAP = 8_388_608;
+export const DEFAULT_MAX_CHUNKS = 2_048;
+export const HARD_MAX_CHUNKS_CAP = 8_192;
+export const DEFAULT_EMBED_BATCH_SIZE = 32;
+export const HARD_EMBED_BATCH_SIZE_CAP = 128;
+export const DEFAULT_TOP_K = 5;
+export const HARD_TOP_K_CAP = 32;
+export const DEFAULT_QUERY_CANDIDATES = 20;
+export const HARD_QUERY_CANDIDATES_CAP = 128;
+export const DEFAULT_MAX_RESULT_BYTES = 64 * 1024;
+export const HARD_MAX_RESULT_BYTES_CAP = 512 * 1024;
+export const DEFAULT_MAX_CONTEXT_TOKENS = 2_000;
+export const HARD_MAX_CONTEXT_TOKENS_CAP = 8_000;
+export const DEFAULT_MAX_METADATA_BYTES = 16 * 1024;
+export const HARD_MAX_METADATA_BYTES_CAP = 64 * 1024;
+export const DEFAULT_MAX_VECTOR_DIMENSIONS = 4_096;
+function integer(value, fallback, cap, label, minimum = 1) {
+    const resolved = value ?? fallback;
+    if (!Number.isInteger(resolved) || resolved < minimum) {
+        throw new RagLimitError(`${label} must be an integer >= ${minimum}`);
+    }
+    if (resolved > cap)
+        throw new RagLimitError(`${label} exceeds hard cap ${cap}`);
+    return resolved;
+}
+export function resolveRagLimits(input = {}) {
+    const chunkSize = integer(input.chunkSize, DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, "chunkSize");
+    const chunkOverlap = integer(input.chunkOverlap, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, "chunkOverlap", 0);
+    if (chunkOverlap >= chunkSize)
+        throw new RagLimitError("chunkOverlap must be smaller than chunkSize");
+    const topK = integer(input.topK, DEFAULT_TOP_K, HARD_TOP_K_CAP, "topK");
+    const queryCandidates = integer(input.queryCandidates, Math.max(DEFAULT_QUERY_CANDIDATES, topK), HARD_QUERY_CANDIDATES_CAP, "queryCandidates");
+    if (queryCandidates < topK)
+        throw new RagLimitError("queryCandidates must be >= topK");
+    return Object.freeze({
+        chunkSize,
+        chunkOverlap,
+        maxDocumentChars: integer(input.maxDocumentChars, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, "maxDocumentChars"),
+        maxChunks: integer(input.maxChunks, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, "maxChunks"),
+        embedBatchSize: integer(input.embedBatchSize, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, "embedBatchSize"),
+        topK,
+        queryCandidates,
+        maxResultBytes: integer(input.maxResultBytes, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, "maxResultBytes"),
+        maxContextTokens: integer(input.maxContextTokens, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, "maxContextTokens"),
+        maxMetadataBytes: integer(input.maxMetadataBytes, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, "maxMetadataBytes"),
+        maxVectorDimensions: integer(input.maxVectorDimensions, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_VECTOR_DIMENSIONS, "maxVectorDimensions"),
+    });
+}
+//# sourceMappingURL=limits.js.map

package/dist/retrieve.d.ts

@@ -0,0 +1,2 @@
+import type { RagContextResult, RetrieveContextOptions } from "./types.js";
+export declare function retrieveContext(query: string, options: RetrieveContextOptions): Promise<RagContextResult>;

package/dist/retrieve.js

@@ -0,0 +1,135 @@
+import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
+import { RagScopeError, RagValidationError } from "./errors.js";
+import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, resolveRedactor, truncateUtf8, } from "./util.js";
+export async function retrieveContext(query, options) {
+    nonEmpty(query, "query");
+    if (query.length > HARD_CHUNK_SIZE_CAP) {
+        throw new RagValidationError(`query exceeds ${HARD_CHUNK_SIZE_CAP} characters`);
+    }
+    const scope = requireScope(options.scope);
+    const limits = resolveRagLimits({
+        topK: options.topK,
+        queryCandidates: options.queryCandidates,
+        maxResultBytes: options.maxResultBytes,
+        maxContextTokens: options.maxContextTokens,
+        maxMetadataBytes: options.maxMetadataBytes,
+        maxVectorDimensions: options.maxVectorDimensions,
+    });
+    if (!Number.isInteger(options.embedder.dimensions)
+        || options.embedder.dimensions <= 0
+        || options.embedder.dimensions > limits.maxVectorDimensions) {
+        throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
+    }
+    if (options.filter)
+        assertBytes(options.filter, limits.maxMetadataBytes, "metadata filter");
+    const redactor = resolveRedactor(options.redactor, options.secrets);
+    const safeQuery = redactor?.redact(query) ?? query;
+    assertNotAborted(options.signal);
+    const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
+    const embedding = vectors[0];
+    if (vectors.length !== 1
+        || !embedding
+        || embedding.length !== options.embedder.dimensions
+        || embedding.some((value) => !Number.isFinite(value))) {
+        throw new RagValidationError("embedder returned invalid query vector");
+    }
+    const candidates = await options.store.query({
+        tenantId: scope.tenantId,
+        resourceId: scope.resourceId,
+        threadId: scope.corpusId,
+        embedding,
+        topK: limits.queryCandidates,
+        signal: options.signal,
+    });
+    assertNotAborted(options.signal);
+    const hits = [];
+    const citations = [];
+    const rendered = [];
+    const maxChars = limits.maxContextTokens * 4;
+    let usedBytes = 0;
+    let usedChars = 0;
+    let truncated = false;
+    for (const candidate of candidates.slice(0, limits.queryCandidates)) {
+        assertScope(scope, candidate);
+        const parsed = parseHit(candidate);
+        if (!matchesFilter(parsed.metadata, options.filter))
+            continue;
+        if (hits.length >= limits.topK)
+            break;
+        const safe = redactor?.redact(parsed) ?? parsed;
+        const prefix = `[${parsed.citationId}] `;
+        const separator = rendered.length ? "\n\n" : "";
+        const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
+        const availableChars = maxChars - usedChars - separator.length - prefix.length;
+        if (availableBytes <= 0 || availableChars <= 0) {
+            truncated = true;
+            break;
+        }
+        let text = safe.text.slice(0, availableChars);
+        text = truncateUtf8(text, availableBytes);
+        if (!text) {
+            truncated = true;
+            break;
+        }
+        if (text.length < safe.text.length)
+            truncated = true;
+        const hit = Object.freeze({ ...safe, text });
+        const citation = Object.freeze({
+            id: hit.citationId,
+            sourceId: hit.sourceId,
+            chunkId: hit.id,
+            ...(hit.metadata ? { metadata: hit.metadata } : {}),
+        });
+        const block = `${separator}${prefix}${text}`;
+        rendered.push(block);
+        usedBytes += byteLength(block);
+        usedChars += block.length;
+        hits.push(hit);
+        citations.push(citation);
+        if (truncated)
+            break;
+    }
+    return Object.freeze({
+        query: safeQuery,
+        text: rendered.join(""),
+        hits: Object.freeze(hits),
+        citations: Object.freeze(citations),
+        truncated,
+    });
+}
+function parseHit(hit) {
+    const metadata = hit.metadata;
+    const rag = metadata?._rag;
+    if (!isJsonObject(rag))
+        throw new RagScopeError("vector hit is missing RAG source metadata");
+    const sourceId = requireSourceId(rag.sourceId);
+    const citationId = nonEmpty(rag.citationId, "metadata._rag.citationId");
+    if (!Number.isInteger(rag.chunkIndex)
+        || Number(rag.chunkIndex) < 0
+        || !Number.isInteger(rag.start)
+        || Number(rag.start) < 0
+        || !Number.isInteger(rag.end)
+        || Number(rag.end) < Number(rag.start)
+        || !Number.isFinite(hit.score)) {
+        throw new RagValidationError("vector hit has invalid RAG offsets");
+    }
+    if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`)) {
+        throw new RagValidationError("vector hit has inconsistent citation identity");
+    }
+    const userMetadata = {};
+    for (const [key, value] of Object.entries(metadata ?? {}))
+        if (key !== "_rag")
+            userMetadata[key] = value;
+    return {
+        id: hit.id,
+        citationId,
+        sourceId,
+        index: rag.chunkIndex,
+        start: rag.start,
+        end: rag.end,
+        text: hit.text,
+        score: hit.score,
+        ...(Object.keys(userMetadata).length ? { metadata: userMetadata } : {}),
+    };
+}
+//# sourceMappingURL=retrieve.js.map

package/dist/types.d.ts

@@ -0,0 +1,82 @@
+import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
+import type { Embedder, VectorStore } from "@arnilo/prism-memory";
+export interface RagScope {
+    readonly tenantId: string;
+    readonly resourceId: string;
+    readonly corpusId: string;
+}
+export interface RagChunk {
+    readonly id: string;
+    readonly citationId: string;
+    readonly sourceId: string;
+    readonly index: number;
+    readonly start: number;
+    readonly end: number;
+    readonly text: string;
+    readonly metadata?: JsonObject;
+}
+export interface ChunkOptions {
+    readonly sourceId: string;
+    readonly metadata?: JsonObject;
+    readonly size?: number;
+    readonly overlap?: number;
+    readonly maxDocumentChars?: number;
+    readonly maxChunks?: number;
+}
+export interface IndexChunksOptions {
+    readonly chunks: readonly RagChunk[];
+    readonly embedder: Embedder;
+    readonly store: VectorStore;
+    readonly scope: RagScope;
+    readonly batchSize?: number;
+    readonly maxChunks?: number;
+    readonly maxChunkChars?: number;
+    readonly maxVectorDimensions?: number;
+    readonly maxMetadataBytes?: number;
+    readonly redactor?: SecretRedactor;
+    readonly secrets?: readonly (string | undefined)[];
+    readonly signal?: AbortSignal;
+}
+export interface IndexChunksResult {
+    readonly indexed: number;
+    readonly sourceIds: readonly string[];
+}
+export interface RagCitation {
+    readonly id: string;
+    readonly sourceId: string;
+    readonly chunkId: string;
+    readonly metadata?: JsonObject;
+}
+export interface RagHit extends RagChunk {
+    readonly score: number;
+}
+export interface RetrieveContextOptions {
+    readonly embedder: Embedder;
+    readonly store: VectorStore;
+    readonly scope: RagScope;
+    readonly topK?: number;
+    readonly queryCandidates?: number;
+    readonly filter?: JsonObject;
+    readonly maxResultBytes?: number;
+    readonly maxContextTokens?: number;
+    readonly maxMetadataBytes?: number;
+    readonly maxVectorDimensions?: number;
+    readonly redactor?: SecretRedactor;
+    readonly secrets?: readonly (string | undefined)[];
+    readonly signal?: AbortSignal;
+}
+export interface RagContextResult {
+    readonly query: string;
+    readonly text: string;
+    readonly hits: readonly RagHit[];
+    readonly citations: readonly RagCitation[];
+    readonly truncated: boolean;
+}
+export interface RagContextProviderOptions extends Omit<RetrieveContextOptions, "signal"> {
+    readonly name?: string;
+    readonly title?: string;
+    readonly query?: string | ((context: {
+        readonly messages: readonly Message[];
+    }) => string | undefined);
+}
+export type RagContextProvider = ContextProvider;

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/util.d.ts

@@ -0,0 +1,19 @@
+import { type JsonObject, type JsonValue, type Message, type SecretRedactor } from "@arnilo/prism";
+import type { RagScope } from "./types.js";
+export declare function assertNotAborted(signal?: AbortSignal): void;
+export declare function nonEmpty(value: unknown, label: string): string;
+export declare function requireSourceId(value: unknown): string;
+export declare function requireScope(scope: RagScope): RagScope;
+export declare function assertScope(expected: RagScope, actual: {
+    tenantId: string;
+    resourceId: string;
+    threadId: string;
+}): void;
+export declare function resolveRedactor(redactor?: SecretRedactor, secrets?: readonly (string | undefined)[]): SecretRedactor | undefined;
+export declare function byteLength(value: unknown): number;
+export declare function assertBytes(value: unknown, limit: number, label: string): void;
+export declare function latestUserText(messages: readonly Message[]): string | undefined;
+export declare function jsonEqual(left: JsonValue | undefined, right: JsonValue | undefined): boolean;
+export declare function matchesFilter(metadata: JsonObject | undefined, filter: JsonObject | undefined): boolean;
+export declare function isJsonObject(value: unknown): value is JsonObject;
+export declare function truncateUtf8(text: string, maxBytes: number): string;

package/dist/util.js

@@ -0,0 +1,94 @@
+import { createSecretRedactor } from "@arnilo/prism";
+import { RagAbortError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
+export function assertNotAborted(signal) {
+    if (signal?.aborted)
+        throw new RagAbortError();
+}
+export function nonEmpty(value, label) {
+    if (typeof value !== "string" || !value.trim())
+        throw new RagValidationError(`${label} must be a non-empty string`);
+    return value;
+}
+export function requireSourceId(value) {
+    const sourceId = nonEmpty(value, "sourceId");
+    if (sourceId.length > 256 || !/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(sourceId)) {
+        throw new RagValidationError("sourceId must be <= 256 URL-safe identifier characters");
+    }
+    return sourceId;
+}
+export function requireScope(scope) {
+    return {
+        tenantId: nonEmpty(scope.tenantId, "tenantId"),
+        resourceId: nonEmpty(scope.resourceId, "resourceId"),
+        corpusId: nonEmpty(scope.corpusId, "corpusId"),
+    };
+}
+export function assertScope(expected, actual) {
+    if (actual.tenantId !== expected.tenantId
+        || actual.resourceId !== expected.resourceId
+        || actual.threadId !== expected.corpusId) {
+        throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
+    }
+}
+export function resolveRedactor(redactor, secrets) {
+    return redactor ?? (secrets?.length ? createSecretRedactor(secrets) : undefined);
+}
+export function byteLength(value) {
+    return Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf8");
+}
+export function assertBytes(value, limit, label) {
+    const bytes = byteLength(value);
+    if (bytes > limit)
+        throw new RagLimitError(`${label} exceeds ${limit} bytes (${bytes})`);
+}
+export function latestUserText(messages) {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+        const message = messages[index];
+        if (message.role !== "user")
+            continue;
+        const text = message.content
+            .filter((block) => block.type === "text")
+            .map((block) => block.text)
+            .join("\n");
+        if (text.trim())
+            return text;
+    }
+    return undefined;
+}
+export function jsonEqual(left, right) {
+    if (left === right)
+        return true;
+    if (Array.isArray(left) && Array.isArray(right)) {
+        return left.length === right.length && left.every((item, index) => jsonEqual(item, right[index]));
+    }
+    if (isJsonObject(left) && isJsonObject(right)) {
+        const keys = Object.keys(left);
+        return keys.length === Object.keys(right).length && keys.every((key) => jsonEqual(left[key], right[key]));
+    }
+    return false;
+}
+export function matchesFilter(metadata, filter) {
+    if (!filter)
+        return true;
+    if (!metadata)
+        return false;
+    return Object.entries(filter).every(([key, value]) => jsonEqual(metadata[key], value));
+}
+export function isJsonObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export function truncateUtf8(text, maxBytes) {
+    if (byteLength(text) <= maxBytes)
+        return text;
+    let output = "";
+    let bytes = 0;
+    for (const character of text) {
+        const size = byteLength(character);
+        if (bytes + size > maxBytes)
+            break;
+        output += character;
+        bytes += size;
+    }
+    return output;
+}
+//# sourceMappingURL=util.js.map

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@arnilo/prism-rag",
+  "version": "0.0.5",
+  "description": "Optional bounded text and Markdown RAG primitives for Prism.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!dist/__tests__",
+    "!dist/**/*.map",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "node --test dist/__tests__/rag.test.js",
+    "pack:dry-run": "npm pack --dry-run"
+  },
+  "peerDependencies": {
+    "@arnilo/prism": "0.0.5",
+    "@arnilo/prism-memory": "0.0.5"
+  },
+  "devDependencies": {
+    "@arnilo/prism": "file:../..",
+    "@arnilo/prism-memory": "file:../memory"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ashiqrniloy/prism.git",
+    "directory": "packages/rag"
+  },
+  "bugs": {
+    "url": "https://github.com/ashiqrniloy/prism/issues"
+  },
+  "homepage": "https://github.com/ashiqrniloy/prism/tree/main/packages/rag#readme",
+  "keywords": [
+    "prism",
+    "rag",
+    "retrieval",
+    "chunking",
+    "citations",
+    "vector-store"
+  ],
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  }
+}