@arnilo/prism-rag 0.3.0 → 0.3.2

package/dist/tei-reranker.js

@@ -0,0 +1,138 @@
+import { MediaContentError, pinnedFetch } from "@arnilo/prism";
+import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
+const DEFAULT_TIMEOUT_MS = 2000;
+const DEFAULT_MAX_RESPONSE_BYTES = 64 * 1024;
+export function createTeiReranker(options) {
+    const { baseUrl, model, timeoutMs = DEFAULT_TIMEOUT_MS, maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES, ssrf, allowLoopback } = options;
+    if (!baseUrl.trim())
+        throw new RagValidationError("TEI reranker baseUrl is required");
+    let url;
+    try {
+        url = new URL(baseUrl.trim());
+    }
+    catch {
+        throw new RagValidationError("TEI reranker baseUrl must be an absolute URL");
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new RagValidationError(`TEI reranker baseUrl must use http(s) (got ${url.protocol})`);
+    }
+    if (url.username || url.password)
+        throw new RagValidationError("TEI reranker baseUrl must not embed credentials");
+    if (url.hash)
+        throw new RagValidationError("TEI reranker baseUrl must not contain a fragment");
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)
+        throw new RagValidationError("TEI reranker timeoutMs must be a positive integer");
+    if (!Number.isSafeInteger(maxResponseBytes) || maxResponseBytes < 1) {
+        throw new RagValidationError("TEI reranker maxResponseBytes must be a positive integer");
+    }
+    // Join without doubling a trailing slash.
+    const endpoint = `${url.toString().replace(/\/+$/, "")}/rerank`;
+    const pinnedRerankFetch = (input, init) => pinnedFetch(input instanceof URL ? input : new URL(String(input)), init, {
+        errorPrefix: "TEI rerank",
+        hostnameErrorPrefix: "TEI rerank",
+        ssrf,
+        allowLoopback,
+        maxResponseBytes,
+    });
+    const transport = options.fetch ?? pinnedRerankFetch;
+    return {
+        async rerank({ query, hits, signal }) {
+            const payload = JSON.stringify({ query, texts: hits.map((hit) => hit.text), raw_scores: false, ...(model ? { model } : {}) });
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeoutMs);
+            const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+            let response;
+            try {
+                response = await transport(endpoint, {
+                    method: "POST",
+                    headers: { "content-type": "application/json", accept: "application/json" },
+                    body: payload,
+                    signal: combined,
+                    redirect: "manual",
+                });
+            }
+            catch (error) {
+                if (error instanceof MediaContentError) {
+                    throw new RagValidationError(`TEI rerank request denied: ${error.message}`);
+                }
+                if (signal?.aborted)
+                    throw new RagAbortError();
+                throw new RagLimitError(`TEI rerank exceeded ${timeoutMs}ms or failed to reach ${url.host}`);
+            }
+            finally {
+                clearTimeout(timer);
+            }
+            if (!response.ok)
+                throw new RagValidationError(`TEI rerank endpoint returned HTTP ${response.status}`);
+            const text = await readBoundedBody(response, maxResponseBytes);
+            let parsed;
+            try {
+                parsed = JSON.parse(text);
+            }
+            catch {
+                throw new RagValidationError("TEI rerank response is not valid JSON");
+            }
+            return orderByScores(hits, parsed);
+        },
+    };
+}
+/** Strictly parse `{results: [{index, score}]}` and reorder hits by score desc. */
+function orderByScores(hits, parsed) {
+    if (typeof parsed !== "object" || parsed === null || !Array.isArray(parsed.results)) {
+        throw new RagValidationError("TEI rerank response missing results array");
+    }
+    const results = parsed.results;
+    if (results.length !== hits.length) {
+        throw new RagValidationError(`TEI rerank returned ${results.length} scores for ${hits.length} hits`);
+    }
+    const scores = new Array(hits.length);
+    const seen = new Set();
+    for (const item of results) {
+        if (typeof item !== "object" || item === null)
+            throw new RagValidationError("TEI rerank result must be an object");
+        const index = item.index;
+        const score = item.score;
+        if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index >= hits.length) {
+            throw new RagValidationError("TEI rerank returned an out-of-range index");
+        }
+        if (seen.has(index))
+            throw new RagValidationError("TEI rerank returned a duplicate index");
+        if (typeof score !== "number" || !Number.isFinite(score))
+            throw new RagValidationError("TEI rerank returned a non-finite score");
+        seen.add(index);
+        scores[index] = score;
+    }
+    // Stable sort by descending score; ties keep original array order.
+    const ordered = hits
+        .map((hit, i) => ({ hit, score: scores[i] }))
+        .sort((a, b) => b.score - a.score)
+        .map((entry) => entry.hit);
+    return Object.freeze(ordered);
+}
+async function readBoundedBody(response, maxBytes) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return "";
+    const decoder = new TextDecoder();
+    let out = "";
+    let total = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            total += value.byteLength;
+            if (total > maxBytes)
+                throw new RagLimitError(`TEI rerank response exceeds ${maxBytes} bytes`);
+            out += decoder.decode(value, { stream: true });
+        }
+        return out + decoder.decode();
+    }
+    catch (error) {
+        // The pinned transport may already have errorred the stream (MediaContentError).
+        if (error instanceof RagLimitError)
+            throw error;
+        throw new RagLimitError(`TEI rerank response exceeds ${maxBytes} bytes`);
+    }
+}
+//# sourceMappingURL=tei-reranker.js.map

package/dist/telemetry.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Dependency-free telemetry seam for RAG operations. When `telemetry` is omitted every
+ * call site short-circuits on an optional chain — zero allocations, zero per-span work.
+ * Implementations own timing/status; the seam only carries structure and attributes.
+ */
+export type RagTelemetryAttributeValue = string | number | boolean;
+export interface RagTelemetrySpan {
+    setAttribute(name: string, value: RagTelemetryAttributeValue): void;
+    addEvent(name: string, attributes?: Readonly<Record<string, RagTelemetryAttributeValue>>): void;
+    /** Flags the span as failed without carrying any error text (no user data leaves the host). */
+    recordError(): void;
+    end(): void;
+}
+export interface RagTelemetry {
+    startSpan(name: string, attributes?: Readonly<Record<string, RagTelemetryAttributeValue>>, parent?: RagTelemetrySpan): RagTelemetrySpan;
+}

package/dist/telemetry.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=telemetry.js.map

package/dist/types.d.ts

@@ -1,5 +1,6 @@
 import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
 import type { Embedder, MemoryVectorRecord, VectorStore } from "@arnilo/prism-memory";
+import type { RagTelemetry, RagTelemetrySpan } from "./telemetry.js";
 export interface RagScope {
     readonly tenantId: string;
     readonly resourceId: string;
@@ -15,6 +16,11 @@ export interface RagChunk {
     readonly text: string;
     readonly metadata?: JsonObject;
 }
+/** Stored embedding of a previous record offered for reuse when its text is unchanged. */
+export interface ReusableEmbedding {
+    readonly text: string;
+    readonly embedding: readonly number[];
+}
 export interface ChunkOptions {
     readonly sourceId: string;
     readonly metadata?: JsonObject;
@@ -111,6 +117,13 @@ export interface IndexChunksOptions {
     readonly secrets?: readonly (string | undefined)[];
     readonly statusStore?: IngestionStatusStore;
     readonly signal?: AbortSignal;
+    /** Host-supplied document digest stamped into each record's `_rag.contentHash`. */
+    readonly contentHash?: string;
+    /** Chunk id → previous text+embedding; embeddings are reused (no embed call) when texts match. */
+    readonly reuseEmbeddings?: ReadonlyMap<string, ReusableEmbedding>;
+    /** Optional telemetry seam; spans nest under `telemetryParent` when supplied. */
+    readonly telemetry?: RagTelemetry;
+    readonly telemetryParent?: RagTelemetrySpan;
 }
 export interface IndexChunksResult {
     readonly indexed: number;
@@ -120,6 +133,10 @@ export interface ReplaceSourceOptions extends Omit<IndexChunksOptions, "chunks"
     readonly sourceId: string;
     readonly chunks: readonly RagChunk[];
     readonly store: TransactionalVectorStore;
+    /** Host-computed document digest; enables the unchanged-source skip. */
+    readonly contentHash?: string;
+    /** Skip re-indexing when the stored document hash matches. Default true when contentHash is present. */
+    readonly skipIfUnchanged?: boolean;
 }
 export interface DeleteSourceOptions {
     readonly sourceId: string;
@@ -143,7 +160,11 @@ export interface RagProvenance {
     readonly chunkId: string;
     readonly citationId: string;
     readonly provider: string;
-    readonly retrieval: "vector";
+    readonly tenantId: string;
+    readonly resourceId: string;
+    readonly corpusId: string;
+    /** Which retrieval leg(s) surfaced this hit after RRF fusion. */
+    readonly retrieval: "vector" | "lexical" | "hybrid";
     readonly retrievedAt: string;
 }
 export interface RagContentTrust {
@@ -175,9 +196,21 @@ export interface Reranker {
 export interface RetrieveContextOptions {
     readonly embedder: Embedder;
     readonly store: VectorStore;
-    readonly scope: RagScope;
+    /** Single exact scope. Provide `scope` or `scopes`, never both. */
+    readonly scope?: RagScope;
+    /** One or more exact scopes (empty = no hits, no embed). */
+    readonly scopes?: readonly RagScope[];
+    /** Optional telemetry seam; when omitted, instrumentation costs nothing. */
+    readonly telemetry?: RagTelemetry;
     readonly topK?: number;
     readonly queryCandidates?: number;
+    /** Lexical leg mode. Default runs fts when the store supports it and silently skips otherwise;
+     * explicitly requesting "fts"/"bm25" on an unsupported store fails closed. */
+    readonly lexical?: "fts" | "bm25" | "off";
+    /** Only reciprocal-rank fusion is supported. */
+    readonly fusion?: "rrf";
+    /** RRF smoothing constant (default 60, capped). */
+    readonly rrfK?: number;
     readonly filter?: JsonObject;
     readonly maxResultBytes?: number;
     readonly maxContextTokens?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arnilo/prism-rag",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Optional bounded text and Markdown RAG primitives for Prism.",
   "type": "module",
   "main": "./dist/index.js",
@@ -33,7 +33,7 @@
     "pack:dry-run": "npm pack --dry-run"
   },
   "peerDependencies": {
-    "@arnilo/prism": "^0.3.0",
+    "@arnilo/prism": "^0.3.1",
     "@arnilo/prism-memory": "^0.3.0"
   },
   "devDependencies": {