@absolutejs/rag 0.6.1 → 0.7.0

package/README.md

@@ -1,6 +1,59 @@
 # @absolutejs/rag
 
-Standalone RAG runtime extracted from AbsoluteJS.
+A standalone RAG runtime for Bun and Elysia applications covering document ingestion, chunking, embeddings, hybrid retrieval, reranking, source synchronization, evaluation, client primitives, and framework bindings.
 
-This package currently contains retrieval, ingestion, sync, connector, and evaluation code.
-A later cleanup pass should extract the remaining shared AI substrate into @absolutejs/ai.
+## Installation
+
+```sh
+bun add @absolutejs/rag
+```
+
+## Quick start
+
+```ts
+import {
+	createInMemoryRAGStore,
+	createRAGCollection,
+	ingestRAGDocuments,
+	openaiEmbeddings,
+	searchDocuments
+} from '@absolutejs/rag';
+
+const collection = createRAGCollection({
+	embedding: openaiEmbeddings({
+		apiKey: process.env.OPENAI_API_KEY ?? '',
+		defaultModel: 'text-embedding-3-small'
+	}),
+	store: createInMemoryRAGStore()
+});
+
+await ingestRAGDocuments(collection, {
+	documents: [{ id: 'intro', text: 'AbsoluteJS ships typed Bun primitives.' }]
+});
+
+const results = await searchDocuments(collection, {
+	query: 'What does AbsoluteJS ship?',
+	topK: 3
+});
+```
+
+## Retrieval and storage
+
+The built-in memory store supports development and tests. Published adapters provide PostgreSQL with pgvector, SQLite with optional vec0 acceleration, and Pinecone behind the same `RAGVectorStore` contract. Lexical and vector results can be fused, transformed, and reranked with provider or heuristic rerankers.
+
+## Ingestion and source sync
+
+The ingestion pipeline handles files, directories, uploads, URLs, PDFs, office documents, archives, images, and media transcripts. Scheduled connectors can keep collections synchronized from email, GitHub, sitemaps, feeds, directories, and S3-compatible storage.
+
+## Quality and evaluation
+
+`@absolutejs/rag/quality` evaluates retrieval relevance and answer grounding, compares strategies and rerankers, and records runs against a baseline so retrieval changes can be tested before release.
+
+## Client and framework entry points
+
+- `@absolutejs/rag/client` and `/client/ui` provide browser-side primitives.
+- `@absolutejs/rag/react`, `/vue`, `/svelte`, and `/angular` provide framework bindings.
+- `@absolutejs/rag/adapter-kit` exposes the contracts used by vector-store adapters.
+- `@absolutejs/rag/ui` exposes presentation-neutral UI contracts.
+
+Pair the retrieval runtime with `@absolutejs/ai` when retrieved context should feed a model or streaming assistant.

package/dist/adapter-kit/index.js

@@ -13403,6 +13403,7 @@ var resolveRAGEmbeddingProvider = (providerLike, fallbackEmbed, defaultModel) =>
   }
   const resolvedDefaultModel = provider.defaultModel ?? defaultModel;
   return {
+    cacheNamespace: provider.cacheNamespace,
     defaultModel: resolvedDefaultModel,
     dimensions: provider.dimensions,
     embed: (input) => provider.embed({
@@ -20893,6 +20894,30 @@ var createRAGCollection = (options) => {
     validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
     return vector;
   };
+  const throwIfAborted = (signal) => {
+    if (signal?.aborted) {
+      throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+    }
+  };
+  const mapConcurrentSettled = async (values, concurrency, mapper) => {
+    const results = new Array(values.length);
+    let nextIndex = 0;
+    const workers = Array.from({ length: Math.min(values.length, Math.max(1, Math.floor(concurrency))) }, async () => {
+      while (nextIndex < values.length) {
+        const index = nextIndex++;
+        try {
+          results[index] = {
+            status: "fulfilled",
+            value: await mapper(values[index], index)
+          };
+        } catch (reason) {
+          results[index] = { reason, status: "rejected" };
+        }
+      }
+    });
+    await Promise.all(workers);
+    return results;
+  };
   const searchWithTrace = async (input) => {
     const model = input.model ?? options.defaultModel;
     const topK = input.topK ?? defaultTopK;
@@ -20953,17 +20978,20 @@ var createRAGCollection = (options) => {
         stage: "input"
       }
     ];
-    const queryVector = runVector ? await embed({
-      model,
-      signal: input.signal,
-      text: input.query
-    }, "query") : [];
+    const queryVectors = new Map;
+    if (runVector) {
+      for (const query of searchQueries) {
+        queryVectors.set(query, embed({ model, signal: input.signal, text: query }, "query"));
+      }
+    }
+    const primarySearchQuery = searchQueries[0] ?? input.query;
+    const queryVector = runVector ? await queryVectors.get(primarySearchQuery) : [];
     if (runVector) {
       steps.push({
         label: "Embedded primary query",
         metadata: {
           dimensions: queryVector.length,
-          query: input.query
+          query: primarySearchQuery
         },
         stage: "embed"
       });
@@ -21022,11 +21050,7 @@ var createRAGCollection = (options) => {
     }
     const resultGroups = await Promise.all(searchQueries.map(async (query, queryIndex) => {
       const [vectorResults2, lexicalResults2] = await Promise.all([
-        runVector ? embed({
-          model,
-          signal: input.signal,
-          text: query
-        }, "query").then((nextQueryVector) => options.store.query({
+        runVector ? queryVectors.get(query).then((nextQueryVector) => options.store.query({
           filter: input.filter,
           candidateLimit: input.nativeCandidateLimit ?? nativeQueryProfile?.candidateLimit,
           fillPolicy: input.nativeFillPolicy ?? nativeQueryProfile?.fillPolicy,
@@ -21322,32 +21346,46 @@ var createRAGCollection = (options) => {
     return result.results;
   };
   const ingest = async (input) => {
-    const chunks = (await Promise.all(input.chunks.map(async (chunk) => {
-      const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
-        model: options.defaultModel,
-        text: chunk.text
-      }, "chunk");
-      const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
-        const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
+    const batchSize = Math.max(1, Math.floor(input.upsertBatchSize ?? 32));
+    const concurrency = Math.max(1, Math.floor(input.embeddingConcurrency ?? 4));
+    for (let start = 0;start < input.chunks.length; start += batchSize) {
+      throwIfAborted(input.signal);
+      const batch = input.chunks.slice(start, start + batchSize);
+      const settled = await mapConcurrentSettled(batch, concurrency, async (chunk) => {
+        throwIfAborted(input.signal);
+        const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
           model: options.defaultModel,
-          text: variant.text ?? chunk.text
+          signal: input.signal,
+          text: chunk.text
         }, "chunk");
-        return {
-          ...variant,
-          embedding
-        };
-      })) : undefined;
-      return expandChunkForMultivectorStorage({
-        ...chunk,
-        embedding: normalizedEmbedding,
-        embeddingVariants: normalizedVariants,
-        metadata: {
-          ...chunk.metadata ?? {},
-          [MULTIVECTOR_PRIMARY]: true
-        }
+        const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
+          const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
+            model: options.defaultModel,
+            signal: input.signal,
+            text: variant.text ?? chunk.text
+          }, "chunk");
+          return {
+            ...variant,
+            embedding
+          };
+        })) : undefined;
+        return expandChunkForMultivectorStorage({
+          ...chunk,
+          embedding: normalizedEmbedding,
+          embeddingVariants: normalizedVariants,
+          metadata: {
+            ...chunk.metadata ?? {},
+            [MULTIVECTOR_PRIMARY]: true
+          }
+        });
       });
-    }))).flat();
-    await options.store.upsert({ chunks });
+      const chunks = settled.filter((result) => result.status === "fulfilled").flatMap((result) => result.value);
+      if (chunks.length > 0)
+        await options.store.upsert({ chunks });
+      const failed = settled.find((result) => result.status === "rejected");
+      if (failed)
+        throw failed.reason;
+    }
   };
   const buildSourceUpsertInput = async (sourceId, input) => {
     const sharedMetadata = input.metadata;
@@ -21425,33 +21463,34 @@ var createRAGCollection = (options) => {
     if (!sourceId) {
       throw new Error("ingestSource requires a non-empty sourceId.");
     }
-    if (input.replace !== false) {
-      await removeSource({
-        chunkCount: input.previousChunkCount,
-        sourceId
-      });
-    }
     const built = await buildSourceUpsertInput(sourceId, input);
     const embedKind = input.embedKind ?? "passage";
-    const chunks = await Promise.all(built.chunks.map(async (chunk, index) => {
+    const chunks = built.chunks.map((chunk, index) => {
       const chunkId = `${sourceId}#${index}`;
-      const embedding = chunk.embedding ?? await embed({
-        kind: embedKind,
-        model: options.defaultModel,
-        text: chunk.text
-      }, "chunk");
       return {
         ...chunk,
         chunkId,
-        embedding,
         metadata: {
           ...chunk.metadata ?? {},
+          embeddingKind: embedKind,
           sourceId
         },
         source: chunk.source ?? sourceId
       };
-    }));
-    await ingest({ chunks });
+    });
+    await ingest({
+      chunks,
+      embeddingConcurrency: input.embeddingConcurrency,
+      signal: input.signal,
+      upsertBatchSize: input.upsertBatchSize
+    });
+    if (input.replace !== false && typeof input.previousChunkCount === "number" && input.previousChunkCount > chunks.length) {
+      await removeSource({
+        chunkIds: Array.from({ length: input.previousChunkCount - chunks.length }, (_unused, index) => `${sourceId}#${chunks.length + index}`),
+        filterDelete: false,
+        sourceId
+      });
+    }
     return {
       chunkCount: chunks.length,
       chunkIds: chunks.map((chunk) => chunk.chunkId),
@@ -31365,5 +31404,5 @@ export {
   RAG_NATIVE_QUERY_CANDIDATE_LIMIT
 };
 
-//# debugId=7097CF082CBBCC6664756E2164756E21
+//# debugId=F1E8AAD3FF52845464756E2164756E21
 //# sourceMappingURL=index.js.map