@absolutejs/rag 0.1.2 → 0.2.0

package/dist/index.js

@@ -19308,7 +19308,10 @@ var sourceAwareUnits = (document, format, normalizedText) => {
   }
 };
 var overlapTail = (value, overlap) => {
-  if (overlap <= 0 || value.length <= overlap) {
+  if (overlap <= 0) {
+    return "";
+  }
+  if (value.length <= overlap) {
     return value;
   }
   const candidate = value.slice(-overlap);
@@ -19323,11 +19326,16 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
   };
   const mergeSmallChunk = (merged2, chunk) => {
     const last = merged2[merged2.length - 1];
-    if (!(last && chunk.length < minChunkLength)) {
+    if (last === undefined || chunk.length >= minChunkLength) {
+      merged2.push(chunk);
+      return;
+    }
+    const candidate = normalizeWhitespace(`${last} ${chunk}`);
+    if (candidate.length > maxChunkLength) {
       merged2.push(chunk);
       return;
     }
-    merged2[merged2.length - 1] = normalizeWhitespace(`${last} ${chunk}`);
+    merged2[merged2.length - 1] = candidate;
   };
   const appendUnitToChunk = (trimmed) => {
     if (!current) {
@@ -19348,11 +19356,21 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
     const carry = overlapTail(current, chunkOverlap);
     current = carry.length > 0 ? `${carry} ${trimmed}`.trim() : trimmed;
   };
+  const explodeOversized = (unit) => {
+    if (unit.length <= maxChunkLength)
+      return [unit];
+    const sentences = sentenceUnits(unit);
+    if (sentences.length > 1)
+      return sentences.flatMap(explodeOversized);
+    return fixedUnits(unit, maxChunkLength);
+  };
   for (const unit of units) {
-    const trimmed = unit.trim();
-    if (!trimmed)
-      continue;
-    appendUnitToChunk(trimmed);
+    for (const piece of explodeOversized(unit.trim())) {
+      const trimmed = piece.trim();
+      if (!trimmed)
+        continue;
+      appendUnitToChunk(trimmed);
+    }
   }
   if (current) {
     appendChunk(current);
@@ -31555,6 +31573,54 @@ var ragChat = (config) => {
 // src/presentation/htmxConfig.ts
 var createRAGHTMXConfig = (config) => config;
 var createRAGHTMXWorkflowRenderConfig = (config) => config;
+// src/retrieval/embeddingBudget.ts
+var embeddingCacheKey = async (text, model, kind) => {
+  const payload = `${model ?? "default"}:${kind ?? "passage"}:${text}`;
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(payload));
+  return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
+};
+var QUOTA_PATTERN = /RESOURCE_EXHAUSTED|token limit|quota|insufficient_quota|billing hard limit/i;
+var isEmbeddingQuotaError = (error) => {
+  const message = error instanceof Error ? error.message : typeof error === "string" ? error : "";
+  return QUOTA_PATTERN.test(message);
+};
+var withEmbeddingBudget = (provider, options = {}) => {
+  const base = typeof provider === "function" ? { embed: provider } : provider;
+  const { cache, onQuotaExhausted } = options;
+  let lastError = null;
+  let quotaExhaustedAt = null;
+  let reused = 0;
+  let embedded = 0;
+  return {
+    ...base,
+    embed: async (input) => {
+      const key = cache ? await embeddingCacheKey(input.text, input.model, input.kind) : null;
+      if (key && cache) {
+        const hit = await cache.get(key);
+        if (hit && hit.length > 0) {
+          reused += 1;
+          return hit;
+        }
+      }
+      const vector = await base.embed(input).catch((error) => {
+        lastError = error instanceof Error ? error.message : String(error);
+        if (isEmbeddingQuotaError(error)) {
+          const first = quotaExhaustedAt === null;
+          quotaExhaustedAt = new Date;
+          if (first)
+            onQuotaExhausted?.(error);
+        }
+        throw error;
+      });
+      lastError = null;
+      embedded += 1;
+      if (key && cache)
+        await cache.set(key, vector);
+      return vector;
+    },
+    health: () => ({ embedded, lastError, quotaExhaustedAt, reused })
+  };
+};
 // src/providers/rerankerProviders.ts
 var rowScore = (row) => row.relevance_score ?? row.relevanceScore ?? row.score ?? 0;
 var applyRanking = (candidates, rows, topK) => {
@@ -35995,6 +36061,7 @@ var createSyncRAGStore = (options = {}) => {
 };
 export {
   xaiEmbeddings,
+  withEmbeddingBudget,
   validateRAGEmbeddingDimensions,
   updateRAGEvaluationSuiteCase,
   summarizeRAGSearchTraceStore,
@@ -36084,6 +36151,7 @@ export {
   loadRAGDocumentFile,
   loadRAGAnswerGroundingEvaluationHistory,
   loadRAGAnswerGroundingCaseDifficultyHistory,
+  isEmbeddingQuotaError,
   inspectRAGSQLiteStoreMigrations,
   ingestRAGSource,
   ingestRAGDocuments,
@@ -36097,6 +36165,7 @@ export {
   evaluateRAGCollection,
   evaluateRAGAnswerGroundingCase,
   evaluateRAGAnswerGrounding,
+  embeddingCacheKey,
   deepseekEmbeddings,
   createVoyageRAGReranker,
   createTextFileExtractor,
@@ -36286,5 +36355,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=CB9ED8A4182DD67364756E2164756E21
+//# debugId=A910166B20BDB76264756E2164756E21
 //# sourceMappingURL=index.js.map