@absolutejs/rag 0.2.0 → 0.4.0

package/dist/index.js

@@ -15103,6 +15103,8 @@ var emailStructureUnits = (value, metadata) => {
         return "Forwarded Headers";
       case "quoted_history":
         return "Quoted History";
+      case "signature":
+        return "Signature";
     }
   };
   const familyKeyBase = threadTopic || from || "Email Message";
@@ -17124,6 +17126,16 @@ var parseMaildirMetadata = (source) => {
     ...maildirKey ? { emailMailboxKey: maildirKey } : {}
   };
 };
+var RFC2047_WORD = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g;
+var decodeEncodedWord = (charset, encoding, text) => {
+  const bytes = encoding.toUpperCase() === "B" ? Buffer.from(text, "base64") : Buffer.from(decodeQuotedPrintable(text.replace(/_/g, " ")), "latin1");
+  try {
+    return new TextDecoder(charset.trim().toLowerCase()).decode(bytes);
+  } catch {
+    return bytes.toString("utf8");
+  }
+};
+var decodeRfc2047 = (value) => value.includes("=?") ? value.replace(/\?=\s+=\?/g, "?==?").replace(RFC2047_WORD, (_match, charset, enc, text) => decodeEncodedWord(charset, enc, text)) : value;
 var parseHeaderBlock = (headerBlock) => {
   const unfolded = headerBlock.replace(/\n[ \t]+/g, " ");
   const headers = new Map;
@@ -17133,7 +17145,7 @@ var parseHeaderBlock = (headerBlock) => {
     if (separator < 0) {
       continue;
     }
-    headers.set(line.slice(0, separator).trim().toLowerCase(), line.slice(separator + 1).trim());
+    headers.set(line.slice(0, separator).trim().toLowerCase(), decodeRfc2047(line.slice(separator + 1).trim()));
   }
   return headers;
 };
@@ -17145,7 +17157,7 @@ var decodeEmailPartBody = (body, encoding) => {
     return new Uint8Array(Buffer.from(trimmed.replace(/\s+/g, ""), "base64"));
   }
   if (normalizedEncoding === "quoted-printable") {
-    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "utf8"));
+    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
   }
   return new Uint8Array(Buffer.from(body, "utf8"));
 };
@@ -17311,15 +17323,30 @@ var EMAIL_FORWARDED_HEADER_PATTERN = /^(?:from|sent|to|cc|bcc|subject|date|reply
 var EMAIL_FORWARDED_SEPARATOR_PATTERN = /^-{2,}\s*forwarded message\s*-{2,}$/i;
 var EMAIL_HTML_QUOTE_OPEN_TOKEN = "[[ABS_EMAIL_QUOTE_OPEN]]";
 var EMAIL_HTML_QUOTE_CLOSE_TOKEN = "[[ABS_EMAIL_QUOTE_CLOSE]]";
+var EMAIL_ATTRIBUTION_PATTERNS = [
+  /^on\s.+\bwrote:$/iu,
+  /^on\s.+\bat\s.+\bwrote:$/iu,
+  /^-+\s*original message\s*-+$/iu,
+  /^_{5,}$/u,
+  /^el\s.+\bescribi\u00F3:$/iu,
+  /^le\s.+\ba \u00E9crit\s*:$/iu
+];
+var EMAIL_SIGNATURE_DELIMITER = /^--\s?$/u;
 var classifyEmailBodyLine = (line, currentKind) => {
   const trimmed = line.trim();
   if (!trimmed) {
     return currentKind;
   }
+  if (currentKind === "signature") {
+    return "signature";
+  }
+  if (EMAIL_SIGNATURE_DELIMITER.test(line)) {
+    return "signature";
+  }
   if (/^>+/.test(trimmed)) {
     return "quoted_history";
   }
-  if (/^on .+wrote:$/i.test(trimmed)) {
+  if (EMAIL_ATTRIBUTION_PATTERNS.some((pattern) => pattern.test(trimmed))) {
     return "quoted_history";
   }
   if (EMAIL_FORWARDED_SEPARATOR_PATTERN.test(trimmed)) {
@@ -17507,16 +17534,8 @@ var scoreEmailBodyCandidate = (candidate) => {
     return Number.NEGATIVE_INFINITY;
   }
   const sections = parseEmailBodySections(candidate);
-  const structuralScore = sections.reduce((score, section) => {
-    if (section.kind === "forwarded_headers") {
-      return score + 8;
-    }
-    if (section.kind === "quoted_history") {
-      return score + 6 + (section.quotedDepth ?? 0);
-    }
-    return score + 1;
-  }, 0);
-  return structuralScore * 10 + Math.min(candidate.length, 5000) / 100;
+  const authoredChars = sections.filter((section) => section.kind === "authored_text").reduce((total, section) => total + section.text.trim().length, 0);
+  return authoredChars * 10 + Math.min(candidate.length, 5000) / 1000;
 };
 var choosePreferredEmailBodyText = (htmlText, plainText) => {
   const htmlScore = scoreEmailBodyCandidate(htmlText);
@@ -17532,7 +17551,7 @@ var chooseEmailBodyCandidate = (current, candidate) => {
   }
   return candidate.length > current.length ? candidate : current;
 };
-var parseEmailMimeParts = (body, contentType) => {
+var parseEmailMimeParts = (body, contentType, transferEncoding) => {
   const attachments = [];
   let bodyText;
   let bodyHtml;
@@ -17554,11 +17573,11 @@ var parseEmailMimeParts = (body, contentType) => {
       const nestedContentType = headers.get("content-type");
       const disposition = headers.get("content-disposition");
       const dispositionType = disposition?.split(";")[0]?.trim().toLowerCase();
-      const transferEncoding = headers.get("content-transfer-encoding");
+      const transferEncoding2 = headers.get("content-transfer-encoding");
       const contentId = normalizeEmailContentId(headers.get("content-id"));
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
-      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding);
+      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
       const decodedText = Buffer.from(decodedBytes).toString("utf8");
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
@@ -17606,17 +17625,24 @@ var parseEmailMimeParts = (body, contentType) => {
       }
     }
   };
-  collectMimeParts(body, contentType);
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeUtf8(decodeEmailPartBody(body, transferEncoding)) : body;
+  collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
     bodyHtml,
     bodyText
   };
 };
+var authoredEmailText = (text) => {
+  const authored = parseEmailBodySections(text).filter((section) => section.kind === "authored_text").map((section) => section.text.trim()).filter(Boolean).join(`
+
+`).trim();
+  return authored || text;
+};
 var extractEmailText = (raw) => {
   const { body, headerBlock } = splitEmailMessage(raw);
   const headers = parseHeaderBlock(headerBlock);
-  const parsed = parseEmailMimeParts(body, headers.get("content-type"));
+  const parsed = parseEmailMimeParts(body, headers.get("content-type"), headers.get("content-transfer-encoding"));
   const htmlText = parsed.bodyHtml ? stripEmailHtml(parsed.bodyHtml) : undefined;
   const plainText = parsed.bodyText ? normalizeWhitespace(parsed.bodyText) : undefined;
   const preferredBodyText = choosePreferredEmailBodyText(htmlText, plainText);
@@ -17673,6 +17699,7 @@ var parseEmailHeaders = (raw) => {
     ccAddressEntries: ccParsed.entries,
     ccAddresses: ccParsed.addresses,
     contentType: getHeader("Content-Type"),
+    contentTransferEncoding: getHeader("Content-Transfer-Encoding"),
     from,
     fromAddress: fromParsed.addresses[0],
     fromAddressEntries: fromParsed.entries,
@@ -17697,7 +17724,7 @@ var parseEmailHeaders = (raw) => {
 var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   const headers = parseEmailHeaders(raw);
   const { body } = splitEmailMessage(raw);
-  const parsed = parseEmailMimeParts(body, headers.contentType);
+  const parsed = parseEmailMimeParts(body, headers.contentType, headers.contentTransferEncoding);
   const source = options?.source ?? input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
   const maildirMetadata = parseMaildirMetadata(source);
   const mergedMetadata = {
@@ -17900,7 +17927,7 @@ var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   return [messageDocument, ...attachmentDocuments.flat()];
 };
 var normalizeEmailThreadKey = (value) => {
-  const normalized = normalizeWhitespace(value?.replace(/^(re|fw|fwd)\s*:\s*/gi, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
+  const normalized = normalizeWhitespace(value?.replace(/^(?:(?:re|fw|fwd|aw|sv|vs|antw)\s*:\s*)+/i, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
   return normalized || undefined;
 };
 var normalizeEmailMessageId = (value) => {
@@ -36346,6 +36373,7 @@ export {
   buildRAGAdminJobPresentation,
   buildRAGAdminActionPresentations,
   buildRAGAdminActionPresentation,
+  authoredEmailText,
   applyRAGSQLiteStoreMigrations,
   applyRAGReranking,
   applyRAGQueryTransform,
@@ -36355,5 +36383,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=A910166B20BDB76264756E2164756E21
+//# debugId=E2463B33C7EF84EC64756E2164756E21
 //# sourceMappingURL=index.js.map