@arnilo/prism-rag 0.0.96 → 0.1.1

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { ContextProvider, JsonObject, Message, SecretRedactor } from "@arnilo/prism";
-import type { Embedder, VectorStore } from "@arnilo/prism-memory";
+import type { Embedder, MemoryVectorRecord, VectorStore } from "@arnilo/prism-memory";
 export interface RagScope {
     readonly tenantId: string;
     readonly resourceId: string;
@@ -23,6 +23,80 @@ export interface ChunkOptions {
     readonly maxDocumentChars?: number;
     readonly maxChunks?: number;
 }
+export interface LoadedDocument {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly mediaType?: string;
+    readonly text?: string;
+    readonly data?: Uint8Array;
+    readonly metadata?: JsonObject;
+}
+export interface ParsedDocument {
+    readonly text: string;
+    readonly metadata?: JsonObject;
+}
+export interface DocumentLoadOptions {
+    readonly signal?: AbortSignal;
+    readonly maxBytes?: number;
+}
+export interface DocumentParseOptions extends DocumentLoadOptions {
+    readonly maxParseMs?: number;
+    readonly maxPages?: number;
+}
+export interface DocumentLoader {
+    load(uri: string, options?: DocumentLoadOptions): Promise<LoadedDocument>;
+}
+export interface Parser {
+    parse(document: LoadedDocument, options?: DocumentParseOptions): Promise<ParsedDocument>;
+}
+export type Chunker = (text: string, options: ChunkOptions) => readonly RagChunk[];
+export interface SourceVectorStore extends VectorStore {
+    getBySource(scope: {
+        readonly tenantId: string;
+        readonly resourceId: string;
+        readonly threadId: string;
+    }, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<readonly MemoryVectorRecord[]>;
+}
+export interface TransactionalVectorStore extends SourceVectorStore {
+    transaction<T>(operation: (store: SourceVectorStore) => Promise<T>, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<T>;
+}
+export type IngestionState = "pending" | "indexed" | "failed" | "partial";
+export interface IngestionStatus {
+    readonly sourceId: string;
+    readonly scope: RagScope;
+    readonly state: IngestionState;
+    readonly bytes: number;
+    readonly chunks: number;
+    readonly error?: string;
+    readonly updatedAt: string;
+}
+export interface IngestionStatusStore {
+    set(status: IngestionStatus, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    delete(scope: RagScope, sourceId: string, options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<void>;
+    list(scope: RagScope, options: {
+        readonly limit: number;
+        readonly cursor?: string;
+        readonly signal?: AbortSignal;
+    }): Promise<{
+        readonly entries: readonly IngestionStatus[];
+        readonly nextCursor?: string;
+    }>;
+}
+export interface IngestionStatusQuery {
+    readonly store: IngestionStatusStore;
+    readonly scope: RagScope;
+    readonly limit?: number;
+    readonly cursor?: string;
+    readonly signal?: AbortSignal;
+}
 export interface IndexChunksOptions {
     readonly chunks: readonly RagChunk[];
     readonly embedder: Embedder;
@@ -35,20 +109,68 @@ export interface IndexChunksOptions {
     readonly maxMetadataBytes?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
+    readonly statusStore?: IngestionStatusStore;
     readonly signal?: AbortSignal;
 }
 export interface IndexChunksResult {
     readonly indexed: number;
     readonly sourceIds: readonly string[];
 }
+export interface ReplaceSourceOptions extends Omit<IndexChunksOptions, "chunks" | "store"> {
+    readonly sourceId: string;
+    readonly chunks: readonly RagChunk[];
+    readonly store: TransactionalVectorStore;
+}
+export interface DeleteSourceOptions {
+    readonly sourceId: string;
+    readonly store: SourceVectorStore;
+    readonly scope: RagScope;
+    readonly statusStore?: IngestionStatusStore;
+    readonly signal?: AbortSignal;
+}
+export interface ReplaceDocumentOptions extends Omit<ReplaceSourceOptions, "sourceId" | "chunks"> {
+    readonly uri: string;
+    readonly sourceId?: string;
+    readonly loader: DocumentLoader;
+    readonly parser: Parser;
+    readonly chunker?: Chunker;
+    readonly chunk?: Omit<ChunkOptions, "sourceId">;
+    readonly loaderOptions?: DocumentLoadOptions;
+    readonly parserOptions?: DocumentParseOptions;
+}
+export interface RagProvenance {
+    readonly sourceId: string;
+    readonly chunkId: string;
+    readonly citationId: string;
+    readonly provider: string;
+    readonly retrieval: "vector";
+    readonly retrievedAt: string;
+}
+export interface RagContentTrust {
+    readonly untrusted: true;
+    readonly inert: true;
+    readonly injectionCapable: true;
+}
 export interface RagCitation {
     readonly id: string;
     readonly sourceId: string;
     readonly chunkId: string;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
     readonly metadata?: JsonObject;
 }
 export interface RagHit extends RagChunk {
     readonly score: number;
+    readonly retrievalRank: number;
+    readonly provenance: RagProvenance;
+    readonly trust: RagContentTrust;
+}
+export interface Reranker {
+    rerank(input: {
+        readonly query: string;
+        readonly hits: readonly RagHit[];
+        readonly signal?: AbortSignal;
+    }): Promise<readonly RagHit[]>;
 }
 export interface RetrieveContextOptions {
     readonly embedder: Embedder;
@@ -61,12 +183,17 @@ export interface RetrieveContextOptions {
     readonly maxContextTokens?: number;
     readonly maxMetadataBytes?: number;
     readonly maxVectorDimensions?: number;
+    readonly reranker?: Reranker;
+    readonly maxRerankBytes?: number;
+    readonly maxRerankMs?: number;
+    readonly rerankConcurrency?: number;
     readonly redactor?: SecretRedactor;
     readonly secrets?: readonly (string | undefined)[];
     readonly signal?: AbortSignal;
 }
 export interface RagContextResult {
     readonly query: string;
+    readonly trust: RagContentTrust;
     readonly text: string;
     readonly hits: readonly RagHit[];
     readonly citations: readonly RagCitation[];

package/dist/util.d.ts

@@ -1,4 +1,4 @@
-import { type JsonObject, type JsonValue, type Message, type SecretRedactor } from "@arnilo/prism";
+import { type JsonObject, type JsonValue, type Message } from "@arnilo/prism";
 import type { RagScope } from "./types.js";
 export declare function assertNotAborted(signal?: AbortSignal): void;
 export declare function nonEmpty(value: unknown, label: string): string;
@@ -9,7 +9,6 @@ export declare function assertScope(expected: RagScope, actual: {
     resourceId: string;
     threadId: string;
 }): void;
-export declare function resolveRedactor(redactor?: SecretRedactor, secrets?: readonly (string | undefined)[]): SecretRedactor | undefined;
 export declare function byteLength(value: unknown): number;
 export declare function assertBytes(value: unknown, limit: number, label: string): void;
 export declare function latestUserText(messages: readonly Message[]): string | undefined;

package/dist/util.js

@@ -1,4 +1,3 @@
-import { createSecretRedactor } from "@arnilo/prism";
 import { RagAbortError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
 export function assertNotAborted(signal) {
     if (signal?.aborted)
@@ -24,15 +23,10 @@ export function requireScope(scope) {
     };
 }
 export function assertScope(expected, actual) {
-    if (actual.tenantId !== expected.tenantId
-        || actual.resourceId !== expected.resourceId
-        || actual.threadId !== expected.corpusId) {
+    if (actual.tenantId !== expected.tenantId || actual.resourceId !== expected.resourceId || actual.threadId !== expected.corpusId) {
         throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
     }
 }
-export function resolveRedactor(redactor, secrets) {
-    return redactor ?? (secrets?.length ? createSecretRedactor(secrets) : undefined);
-}
 export function byteLength(value) {
     return Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf8");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arnilo/prism-rag",
-  "version": "0.0.96",
+  "version": "0.1.1",
   "description": "Optional bounded text and Markdown RAG primitives for Prism.",
   "type": "module",
   "main": "./dist/index.js",
@@ -9,6 +9,14 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./loaders": {
+      "types": "./dist/loaders.d.ts",
+      "default": "./dist/loaders.js"
+    },
+    "./parsers": {
+      "types": "./dist/parsers.d.ts",
+      "default": "./dist/parsers.js"
     }
   },
   "files": [
@@ -25,8 +33,8 @@
     "pack:dry-run": "npm pack --dry-run"
   },
   "peerDependencies": {
-    "@arnilo/prism": "0.0.96",
-    "@arnilo/prism-memory": "0.0.96"
+    "@arnilo/prism": "0.1.1",
+    "@arnilo/prism-memory": "0.1.1"
   },
   "devDependencies": {
     "@arnilo/prism": "file:../..",