@absolutejs/rag 0.5.0 → 0.6.1

package/dist/index.js

@@ -36099,6 +36099,53 @@ var createSyncRAGStore = (options = {}) => {
     upsert
   };
 };
+// src/retrieval/corpus.ts
+var corpusTextHash = async (text) => {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+  return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
+};
+var planRAGCorpus = async (store, owner, desired) => {
+  const wanted = new Map;
+  for (const doc of desired) {
+    if (!wanted.has(doc.chunkId))
+      wanted.set(doc.chunkId, doc);
+  }
+  const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
+  const embed = [];
+  let unchanged = 0;
+  for (const [chunkId, doc] of wanted) {
+    const hash = await corpusTextHash(doc.text);
+    if (stored.get(chunkId) === hash)
+      unchanged += 1;
+    else
+      embed.push(doc);
+  }
+  return {
+    embed,
+    remove: [...stored.keys()].filter((chunkId) => !wanted.has(chunkId)),
+    unchanged
+  };
+};
+var reconcileRAGCorpus = async (store, owner, desired, apply) => {
+  const plan = await planRAGCorpus(store, owner, desired);
+  if (plan.remove.length > 0) {
+    await apply.remove(plan.remove, owner);
+    await store.forget(owner, plan.remove);
+  }
+  let embedded = 0;
+  if (plan.embed.length > 0) {
+    const outcome = await apply.embed(plan.embed, owner);
+    const written = typeof outcome === "number" ? plan.embed.slice(0, Math.max(0, Math.min(outcome, plan.embed.length))) : plan.embed.filter((doc) => new Set(outcome).has(doc.chunkId));
+    embedded = written.length;
+    if (written.length > 0) {
+      await store.remember(owner, await Promise.all(written.map(async (doc) => ({
+        chunkId: doc.chunkId,
+        textHash: await corpusTextHash(doc.text)
+      }))));
+    }
+  }
+  return { embedded, removed: plan.remove.length, unchanged: plan.unchanged };
+};
 export {
   xaiEmbeddings,
   withEmbeddingBudget,
@@ -36123,6 +36170,7 @@ export {
   removeRAGSource,
   removeRAGEvaluationSuiteCaseHardNegative,
   removeRAGEvaluationSuiteCase,
+  reconcileRAGCorpus,
   ragChat as ragPlugin,
   ragChat,
   querySimilarity,
@@ -36134,6 +36182,7 @@ export {
   prepareRAGDocumentFile,
   prepareRAGDocument,
   prepareRAGDirectoryDocuments,
+  planRAGCorpus,
   persistRAGSearchTraceRecord,
   persistRAGSearchTracePruneRun,
   persistRAGRetrievalReleaseLanePolicyHistory,
@@ -36313,6 +36362,7 @@ export {
   createEPUBExtractor,
   createCohereRAGReranker,
   createBuiltinArchiveExpander,
+  corpusTextHash,
   compareRAGRetrievalTraceSummaries,
   compareRAGRetrievalStrategies,
   compareRAGRerankers,
@@ -36396,5 +36446,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=935E9CE2ED17498064756E2164756E21
+//# debugId=6D0F2872DF78972C64756E2164756E21
 //# sourceMappingURL=index.js.map