@absolutejs/rag 0.2.0 → 0.4.0

package/dist/adapter-kit/index.js

@@ -14824,6 +14824,8 @@ var emailStructureUnits = (value, metadata) => {
         return "Forwarded Headers";
       case "quoted_history":
         return "Quoted History";
+      case "signature":
+        return "Signature";
     }
   };
   const familyKeyBase = threadTopic || from || "Email Message";
@@ -16845,6 +16847,16 @@ var parseMaildirMetadata = (source) => {
     ...maildirKey ? { emailMailboxKey: maildirKey } : {}
   };
 };
+var RFC2047_WORD = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g;
+var decodeEncodedWord = (charset, encoding, text) => {
+  const bytes = encoding.toUpperCase() === "B" ? Buffer.from(text, "base64") : Buffer.from(decodeQuotedPrintable(text.replace(/_/g, " ")), "latin1");
+  try {
+    return new TextDecoder(charset.trim().toLowerCase()).decode(bytes);
+  } catch {
+    return bytes.toString("utf8");
+  }
+};
+var decodeRfc2047 = (value) => value.includes("=?") ? value.replace(/\?=\s+=\?/g, "?==?").replace(RFC2047_WORD, (_match, charset, enc, text) => decodeEncodedWord(charset, enc, text)) : value;
 var parseHeaderBlock = (headerBlock) => {
   const unfolded = headerBlock.replace(/\n[ \t]+/g, " ");
   const headers = new Map;
@@ -16854,7 +16866,7 @@ var parseHeaderBlock = (headerBlock) => {
     if (separator < 0) {
       continue;
     }
-    headers.set(line.slice(0, separator).trim().toLowerCase(), line.slice(separator + 1).trim());
+    headers.set(line.slice(0, separator).trim().toLowerCase(), decodeRfc2047(line.slice(separator + 1).trim()));
   }
   return headers;
 };
@@ -16866,7 +16878,7 @@ var decodeEmailPartBody = (body, encoding) => {
     return new Uint8Array(Buffer.from(trimmed.replace(/\s+/g, ""), "base64"));
   }
   if (normalizedEncoding === "quoted-printable") {
-    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "utf8"));
+    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
   }
   return new Uint8Array(Buffer.from(body, "utf8"));
 };
@@ -17032,15 +17044,30 @@ var EMAIL_FORWARDED_HEADER_PATTERN = /^(?:from|sent|to|cc|bcc|subject|date|reply
 var EMAIL_FORWARDED_SEPARATOR_PATTERN = /^-{2,}\s*forwarded message\s*-{2,}$/i;
 var EMAIL_HTML_QUOTE_OPEN_TOKEN = "[[ABS_EMAIL_QUOTE_OPEN]]";
 var EMAIL_HTML_QUOTE_CLOSE_TOKEN = "[[ABS_EMAIL_QUOTE_CLOSE]]";
+var EMAIL_ATTRIBUTION_PATTERNS = [
+  /^on\s.+\bwrote:$/iu,
+  /^on\s.+\bat\s.+\bwrote:$/iu,
+  /^-+\s*original message\s*-+$/iu,
+  /^_{5,}$/u,
+  /^el\s.+\bescribi\u00F3:$/iu,
+  /^le\s.+\ba \u00E9crit\s*:$/iu
+];
+var EMAIL_SIGNATURE_DELIMITER = /^--\s?$/u;
 var classifyEmailBodyLine = (line, currentKind) => {
   const trimmed = line.trim();
   if (!trimmed) {
     return currentKind;
   }
+  if (currentKind === "signature") {
+    return "signature";
+  }
+  if (EMAIL_SIGNATURE_DELIMITER.test(line)) {
+    return "signature";
+  }
   if (/^>+/.test(trimmed)) {
     return "quoted_history";
   }
-  if (/^on .+wrote:$/i.test(trimmed)) {
+  if (EMAIL_ATTRIBUTION_PATTERNS.some((pattern) => pattern.test(trimmed))) {
     return "quoted_history";
   }
   if (EMAIL_FORWARDED_SEPARATOR_PATTERN.test(trimmed)) {
@@ -17228,16 +17255,8 @@ var scoreEmailBodyCandidate = (candidate) => {
     return Number.NEGATIVE_INFINITY;
   }
   const sections = parseEmailBodySections(candidate);
-  const structuralScore = sections.reduce((score, section) => {
-    if (section.kind === "forwarded_headers") {
-      return score + 8;
-    }
-    if (section.kind === "quoted_history") {
-      return score + 6 + (section.quotedDepth ?? 0);
-    }
-    return score + 1;
-  }, 0);
-  return structuralScore * 10 + Math.min(candidate.length, 5000) / 100;
+  const authoredChars = sections.filter((section) => section.kind === "authored_text").reduce((total, section) => total + section.text.trim().length, 0);
+  return authoredChars * 10 + Math.min(candidate.length, 5000) / 1000;
 };
 var choosePreferredEmailBodyText = (htmlText, plainText) => {
   const htmlScore = scoreEmailBodyCandidate(htmlText);
@@ -17253,7 +17272,7 @@ var chooseEmailBodyCandidate = (current, candidate) => {
   }
   return candidate.length > current.length ? candidate : current;
 };
-var parseEmailMimeParts = (body, contentType) => {
+var parseEmailMimeParts = (body, contentType, transferEncoding) => {
   const attachments = [];
   let bodyText;
   let bodyHtml;
@@ -17275,11 +17294,11 @@ var parseEmailMimeParts = (body, contentType) => {
       const nestedContentType = headers.get("content-type");
       const disposition = headers.get("content-disposition");
       const dispositionType = disposition?.split(";")[0]?.trim().toLowerCase();
-      const transferEncoding = headers.get("content-transfer-encoding");
+      const transferEncoding2 = headers.get("content-transfer-encoding");
       const contentId = normalizeEmailContentId(headers.get("content-id"));
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
-      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding);
+      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
       const decodedText = Buffer.from(decodedBytes).toString("utf8");
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
@@ -17327,17 +17346,24 @@ var parseEmailMimeParts = (body, contentType) => {
       }
     }
   };
-  collectMimeParts(body, contentType);
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeUtf8(decodeEmailPartBody(body, transferEncoding)) : body;
+  collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
     bodyHtml,
     bodyText
   };
 };
+var authoredEmailText = (text) => {
+  const authored = parseEmailBodySections(text).filter((section) => section.kind === "authored_text").map((section) => section.text.trim()).filter(Boolean).join(`
+
+`).trim();
+  return authored || text;
+};
 var extractEmailText = (raw) => {
   const { body, headerBlock } = splitEmailMessage(raw);
   const headers = parseHeaderBlock(headerBlock);
-  const parsed = parseEmailMimeParts(body, headers.get("content-type"));
+  const parsed = parseEmailMimeParts(body, headers.get("content-type"), headers.get("content-transfer-encoding"));
   const htmlText = parsed.bodyHtml ? stripEmailHtml(parsed.bodyHtml) : undefined;
   const plainText = parsed.bodyText ? normalizeWhitespace(parsed.bodyText) : undefined;
   const preferredBodyText = choosePreferredEmailBodyText(htmlText, plainText);
@@ -17394,6 +17420,7 @@ var parseEmailHeaders = (raw) => {
     ccAddressEntries: ccParsed.entries,
     ccAddresses: ccParsed.addresses,
     contentType: getHeader("Content-Type"),
+    contentTransferEncoding: getHeader("Content-Transfer-Encoding"),
     from,
     fromAddress: fromParsed.addresses[0],
     fromAddressEntries: fromParsed.entries,
@@ -17418,7 +17445,7 @@ var parseEmailHeaders = (raw) => {
 var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   const headers = parseEmailHeaders(raw);
   const { body } = splitEmailMessage(raw);
-  const parsed = parseEmailMimeParts(body, headers.contentType);
+  const parsed = parseEmailMimeParts(body, headers.contentType, headers.contentTransferEncoding);
   const source = options?.source ?? input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
   const maildirMetadata = parseMaildirMetadata(source);
   const mergedMetadata = {
@@ -17621,7 +17648,7 @@ var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   return [messageDocument, ...attachmentDocuments.flat()];
 };
 var normalizeEmailThreadKey = (value) => {
-  const normalized = normalizeWhitespace(value?.replace(/^(re|fw|fwd)\s*:\s*/gi, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
+  const normalized = normalizeWhitespace(value?.replace(/^(?:(?:re|fw|fwd|aw|sv|vs|antw)\s*:\s*)+/i, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
   return normalized || undefined;
 };
 var normalizeEmailMessageId = (value) => {
@@ -31325,5 +31352,5 @@ export {
   RAG_NATIVE_QUERY_CANDIDATE_LIMIT
 };
 
-//# debugId=86AE9E92B8164CAE64756E2164756E21
+//# debugId=EDE8F57AE9AB8A5064756E2164756E21
 //# sourceMappingURL=index.js.map