@absolutejs/rag 0.0.27 → 0.0.29

package/dist/index.js

@@ -1,4 +1,19 @@
 // @bun
+var __defProp = Object.defineProperty;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+
 // src/quality/quality.ts
 import { mkdir, readFile } from "fs/promises";
 
@@ -21092,7 +21107,10 @@ var createRAGCollection = (options) => {
   const embeddingProvider = resolveRAGEmbeddingProvider(options.embedding, options.store.embed, options.defaultModel);
   const getExpectedDimensions = () => embeddingProvider.dimensions ?? getStatus?.()?.dimensions;
   const embed = async (input, context) => {
-    const vector = await embeddingProvider.embed(input);
+    const vector = await embeddingProvider.embed({
+      ...input,
+      kind: input.kind ?? (context === "query" ? "query" : "passage")
+    });
     validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
     return vector;
   };
@@ -21552,6 +21570,115 @@ var createRAGCollection = (options) => {
     }))).flat();
     await options.store.upsert({ chunks });
   };
+  const buildSourceUpsertInput = async (sourceId, input) => {
+    const sharedMetadata = input.metadata;
+    if (input.document) {
+      return buildRAGUpsertInputFromDocuments({
+        chunkingRegistry: input.chunkingRegistry,
+        defaultChunking: input.chunking,
+        documents: [
+          {
+            ...input.document,
+            chunking: input.document.chunking ?? input.chunking,
+            metadata: { ...input.document.metadata ?? {}, ...sharedMetadata },
+            source: input.document.source ?? sourceId
+          }
+        ]
+      });
+    }
+    if (input.upload) {
+      return buildRAGUpsertInputFromUploads({
+        chunkingRegistry: input.chunkingRegistry,
+        extractorRegistry: input.extractorRegistry,
+        extractors: input.extractors,
+        uploads: [
+          {
+            ...input.upload,
+            chunking: input.upload.chunking ?? input.chunking,
+            metadata: { ...input.upload.metadata ?? {}, ...sharedMetadata },
+            source: input.upload.source ?? sourceId
+          }
+        ]
+      });
+    }
+    if (input.url) {
+      return buildRAGUpsertInputFromURLs({
+        chunkingRegistry: input.chunkingRegistry,
+        extractorRegistry: input.extractorRegistry,
+        extractors: input.extractors,
+        urls: [
+          {
+            ...input.url,
+            chunking: input.url.chunking ?? input.chunking,
+            extractorRegistry: input.url.extractorRegistry ?? input.extractorRegistry,
+            extractors: input.url.extractors ?? input.extractors,
+            metadata: { ...input.url.metadata ?? {}, ...sharedMetadata },
+            source: input.url.source ?? sourceId
+          }
+        ]
+      });
+    }
+    throw new Error("ingestSource requires one of upload, url, or document to be provided.");
+  };
+  const removeSource = async (input) => {
+    const sourceId = input.sourceId?.trim();
+    if (!sourceId) {
+      throw new Error("removeSource requires a non-empty sourceId.");
+    }
+    const deleteFromStore = options.store.delete;
+    if (typeof deleteFromStore !== "function") {
+      throw new Error("removeSource requires a store that implements delete().");
+    }
+    let deleted = 0;
+    const chunkIds = input.chunkIds ?? (typeof input.chunkCount === "number" && input.chunkCount > 0 ? Array.from({ length: input.chunkCount }, (_unused, index) => `${sourceId}#${index}`) : undefined);
+    if (chunkIds && chunkIds.length > 0) {
+      deleted += await deleteFromStore({ chunkIds });
+    }
+    if (input.filterDelete !== false) {
+      try {
+        deleted += await deleteFromStore({ filter: { sourceId } });
+      } catch {}
+    }
+    return { deleted, sourceId };
+  };
+  const ingestSource = async (input) => {
+    const sourceId = input.sourceId?.trim();
+    if (!sourceId) {
+      throw new Error("ingestSource requires a non-empty sourceId.");
+    }
+    if (input.replace !== false) {
+      await removeSource({
+        chunkCount: input.previousChunkCount,
+        sourceId
+      });
+    }
+    const built = await buildSourceUpsertInput(sourceId, input);
+    const embedKind = input.embedKind ?? "passage";
+    const chunks = await Promise.all(built.chunks.map(async (chunk, index) => {
+      const chunkId = `${sourceId}#${index}`;
+      const embedding = chunk.embedding ?? await embed({
+        kind: embedKind,
+        model: options.defaultModel,
+        text: chunk.text
+      }, "chunk");
+      return {
+        ...chunk,
+        chunkId,
+        embedding,
+        metadata: {
+          ...chunk.metadata ?? {},
+          sourceId
+        },
+        source: chunk.source ?? sourceId
+      };
+    }));
+    await ingest({ chunks });
+    return {
+      chunkCount: chunks.length,
+      chunkIds: chunks.map((chunk) => chunk.chunkId),
+      sourceId
+    };
+  };
   return {
     clear: typeof options.store.clear === "function" ? () => options.store.clear?.() : undefined,
     getCapabilities: typeof getCapabilities === "function" ? () => getCapabilities() : undefined,
@@ -21559,12 +21686,16 @@ var createRAGCollection = (options) => {
     searchWithTrace,
     search,
     store: options.store,
-    ingest
+    ingest,
+    ingestSource,
+    removeSource
   };
 };
 var ingestDocuments = async (collection, input) => collection.ingest(input);
 var ingestRAGDocuments = async (collection, input) => collection.ingest(buildRAGUpsertInputFromDocuments(input));
 var searchDocuments = async (collection, input) => collection.search(input);
+var ingestRAGSource = async (collection, input) => collection.ingestSource(input);
+var removeRAGSource = async (collection, input) => collection.removeSource(input);
 
 // src/presentation/htmxWorkflowRenderers.ts
 var escapeHtml2 = (text) => text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
@@ -35882,6 +36013,7 @@ export {
   resolveRAGHybridSearchOptions,
   resolveRAGEmbeddingProvider,
   reorderRAGEvaluationSuiteCases,
+  removeRAGSource,
   removeRAGEvaluationSuiteCaseHardNegative,
   removeRAGEvaluationSuiteCase,
   ragChat as ragPlugin,
@@ -35953,6 +36085,7 @@ export {
   loadRAGAnswerGroundingEvaluationHistory,
   loadRAGAnswerGroundingCaseDifficultyHistory,
   inspectRAGSQLiteStoreMigrations,
+  ingestRAGSource,
   ingestRAGDocuments,
   ingestDocuments,
   googleEmbeddings,
@@ -36153,5 +36286,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=3C1CA4A1C596290364756E2164756E21
+//# debugId=CB9ED8A4182DD67364756E2164756E21
 //# sourceMappingURL=index.js.map