@absolutejs/rag 0.1.3 → 0.2.0

package/dist/index.js

@@ -19308,7 +19308,10 @@ var sourceAwareUnits = (document, format, normalizedText) => {
   }
 };
 var overlapTail = (value, overlap) => {
-  if (overlap <= 0 || value.length <= overlap) {
+  if (overlap <= 0) {
+    return "";
+  }
+  if (value.length <= overlap) {
     return value;
   }
   const candidate = value.slice(-overlap);
@@ -19323,11 +19326,16 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
   };
   const mergeSmallChunk = (merged2, chunk) => {
     const last = merged2[merged2.length - 1];
-    if (!(last && chunk.length < minChunkLength)) {
+    if (last === undefined || chunk.length >= minChunkLength) {
+      merged2.push(chunk);
+      return;
+    }
+    const candidate = normalizeWhitespace(`${last} ${chunk}`);
+    if (candidate.length > maxChunkLength) {
       merged2.push(chunk);
       return;
     }
-    merged2[merged2.length - 1] = normalizeWhitespace(`${last} ${chunk}`);
+    merged2[merged2.length - 1] = candidate;
   };
   const appendUnitToChunk = (trimmed) => {
     if (!current) {
@@ -19348,11 +19356,21 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
     const carry = overlapTail(current, chunkOverlap);
     current = carry.length > 0 ? `${carry} ${trimmed}`.trim() : trimmed;
   };
+  const explodeOversized = (unit) => {
+    if (unit.length <= maxChunkLength)
+      return [unit];
+    const sentences = sentenceUnits(unit);
+    if (sentences.length > 1)
+      return sentences.flatMap(explodeOversized);
+    return fixedUnits(unit, maxChunkLength);
+  };
   for (const unit of units) {
-    const trimmed = unit.trim();
-    if (!trimmed)
-      continue;
-    appendUnitToChunk(trimmed);
+    for (const piece of explodeOversized(unit.trim())) {
+      const trimmed = piece.trim();
+      if (!trimmed)
+        continue;
+      appendUnitToChunk(trimmed);
+    }
   }
   if (current) {
     appendChunk(current);
@@ -36337,5 +36355,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=E7024598E84C592764756E2164756E21
+//# debugId=A910166B20BDB76264756E2164756E21
 //# sourceMappingURL=index.js.map