@absolutejs/rag 0.1.3 → 0.3.0

package/dist/index.js

@@ -15103,6 +15103,8 @@ var emailStructureUnits = (value, metadata) => {
         return "Forwarded Headers";
       case "quoted_history":
         return "Quoted History";
+      case "signature":
+        return "Signature";
     }
   };
   const familyKeyBase = threadTopic || from || "Email Message";
@@ -17311,15 +17313,30 @@ var EMAIL_FORWARDED_HEADER_PATTERN = /^(?:from|sent|to|cc|bcc|subject|date|reply
 var EMAIL_FORWARDED_SEPARATOR_PATTERN = /^-{2,}\s*forwarded message\s*-{2,}$/i;
 var EMAIL_HTML_QUOTE_OPEN_TOKEN = "[[ABS_EMAIL_QUOTE_OPEN]]";
 var EMAIL_HTML_QUOTE_CLOSE_TOKEN = "[[ABS_EMAIL_QUOTE_CLOSE]]";
+var EMAIL_ATTRIBUTION_PATTERNS = [
+  /^on\s.+\bwrote:$/iu,
+  /^on\s.+\bat\s.+\bwrote:$/iu,
+  /^-+\s*original message\s*-+$/iu,
+  /^_{5,}$/u,
+  /^el\s.+\bescribi\u00F3:$/iu,
+  /^le\s.+\ba \u00E9crit\s*:$/iu
+];
+var EMAIL_SIGNATURE_DELIMITER = /^--\s?$/u;
 var classifyEmailBodyLine = (line, currentKind) => {
   const trimmed = line.trim();
   if (!trimmed) {
     return currentKind;
   }
+  if (currentKind === "signature") {
+    return "signature";
+  }
+  if (EMAIL_SIGNATURE_DELIMITER.test(line)) {
+    return "signature";
+  }
   if (/^>+/.test(trimmed)) {
     return "quoted_history";
   }
-  if (/^on .+wrote:$/i.test(trimmed)) {
+  if (EMAIL_ATTRIBUTION_PATTERNS.some((pattern) => pattern.test(trimmed))) {
     return "quoted_history";
   }
   if (EMAIL_FORWARDED_SEPARATOR_PATTERN.test(trimmed)) {
@@ -17507,16 +17524,8 @@ var scoreEmailBodyCandidate = (candidate) => {
     return Number.NEGATIVE_INFINITY;
   }
   const sections = parseEmailBodySections(candidate);
-  const structuralScore = sections.reduce((score, section) => {
-    if (section.kind === "forwarded_headers") {
-      return score + 8;
-    }
-    if (section.kind === "quoted_history") {
-      return score + 6 + (section.quotedDepth ?? 0);
-    }
-    return score + 1;
-  }, 0);
-  return structuralScore * 10 + Math.min(candidate.length, 5000) / 100;
+  const authoredChars = sections.filter((section) => section.kind === "authored_text").reduce((total, section) => total + section.text.trim().length, 0);
+  return authoredChars * 10 + Math.min(candidate.length, 5000) / 1000;
 };
 var choosePreferredEmailBodyText = (htmlText, plainText) => {
   const htmlScore = scoreEmailBodyCandidate(htmlText);
@@ -17613,6 +17622,12 @@ var parseEmailMimeParts = (body, contentType) => {
     bodyText
   };
 };
+var authoredEmailText = (text) => {
+  const authored = parseEmailBodySections(text).filter((section) => section.kind === "authored_text").map((section) => section.text.trim()).filter(Boolean).join(`
+
+`).trim();
+  return authored || text;
+};
 var extractEmailText = (raw) => {
   const { body, headerBlock } = splitEmailMessage(raw);
   const headers = parseHeaderBlock(headerBlock);
@@ -19308,7 +19323,10 @@ var sourceAwareUnits = (document, format, normalizedText) => {
   }
 };
 var overlapTail = (value, overlap) => {
-  if (overlap <= 0 || value.length <= overlap) {
+  if (overlap <= 0) {
+    return "";
+  }
+  if (value.length <= overlap) {
     return value;
   }
   const candidate = value.slice(-overlap);
@@ -19323,11 +19341,16 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
   };
   const mergeSmallChunk = (merged2, chunk) => {
     const last = merged2[merged2.length - 1];
-    if (!(last && chunk.length < minChunkLength)) {
+    if (last === undefined || chunk.length >= minChunkLength) {
+      merged2.push(chunk);
+      return;
+    }
+    const candidate = normalizeWhitespace(`${last} ${chunk}`);
+    if (candidate.length > maxChunkLength) {
       merged2.push(chunk);
       return;
     }
-    merged2[merged2.length - 1] = normalizeWhitespace(`${last} ${chunk}`);
+    merged2[merged2.length - 1] = candidate;
   };
   const appendUnitToChunk = (trimmed) => {
     if (!current) {
@@ -19348,11 +19371,21 @@ var chunkFromUnits = (units, maxChunkLength, chunkOverlap, minChunkLength) => {
     const carry = overlapTail(current, chunkOverlap);
     current = carry.length > 0 ? `${carry} ${trimmed}`.trim() : trimmed;
   };
+  const explodeOversized = (unit) => {
+    if (unit.length <= maxChunkLength)
+      return [unit];
+    const sentences = sentenceUnits(unit);
+    if (sentences.length > 1)
+      return sentences.flatMap(explodeOversized);
+    return fixedUnits(unit, maxChunkLength);
+  };
   for (const unit of units) {
-    const trimmed = unit.trim();
-    if (!trimmed)
-      continue;
-    appendUnitToChunk(trimmed);
+    for (const piece of explodeOversized(unit.trim())) {
+      const trimmed = piece.trim();
+      if (!trimmed)
+        continue;
+      appendUnitToChunk(trimmed);
+    }
   }
   if (current) {
     appendChunk(current);
@@ -36328,6 +36361,7 @@ export {
   buildRAGAdminJobPresentation,
   buildRAGAdminActionPresentations,
   buildRAGAdminActionPresentation,
+  authoredEmailText,
   applyRAGSQLiteStoreMigrations,
   applyRAGReranking,
   applyRAGQueryTransform,
@@ -36337,5 +36371,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=E7024598E84C592764756E2164756E21
+//# debugId=176571A7F3C6808164756E2164756E21
 //# sourceMappingURL=index.js.map