@absolutejs/rag 0.3.0 → 0.4.0

package/dist/index.js

@@ -17126,6 +17126,16 @@ var parseMaildirMetadata = (source) => {
     ...maildirKey ? { emailMailboxKey: maildirKey } : {}
   };
 };
+var RFC2047_WORD = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g;
+var decodeEncodedWord = (charset, encoding, text) => {
+  const bytes = encoding.toUpperCase() === "B" ? Buffer.from(text, "base64") : Buffer.from(decodeQuotedPrintable(text.replace(/_/g, " ")), "latin1");
+  try {
+    return new TextDecoder(charset.trim().toLowerCase()).decode(bytes);
+  } catch {
+    return bytes.toString("utf8");
+  }
+};
+var decodeRfc2047 = (value) => value.includes("=?") ? value.replace(/\?=\s+=\?/g, "?==?").replace(RFC2047_WORD, (_match, charset, enc, text) => decodeEncodedWord(charset, enc, text)) : value;
 var parseHeaderBlock = (headerBlock) => {
   const unfolded = headerBlock.replace(/\n[ \t]+/g, " ");
   const headers = new Map;
@@ -17135,7 +17145,7 @@ var parseHeaderBlock = (headerBlock) => {
     if (separator < 0) {
       continue;
     }
-    headers.set(line.slice(0, separator).trim().toLowerCase(), line.slice(separator + 1).trim());
+    headers.set(line.slice(0, separator).trim().toLowerCase(), decodeRfc2047(line.slice(separator + 1).trim()));
   }
   return headers;
 };
@@ -17147,7 +17157,7 @@ var decodeEmailPartBody = (body, encoding) => {
     return new Uint8Array(Buffer.from(trimmed.replace(/\s+/g, ""), "base64"));
   }
   if (normalizedEncoding === "quoted-printable") {
-    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "utf8"));
+    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
   }
   return new Uint8Array(Buffer.from(body, "utf8"));
 };
@@ -17541,7 +17551,7 @@ var chooseEmailBodyCandidate = (current, candidate) => {
   }
   return candidate.length > current.length ? candidate : current;
 };
-var parseEmailMimeParts = (body, contentType) => {
+var parseEmailMimeParts = (body, contentType, transferEncoding) => {
   const attachments = [];
   let bodyText;
   let bodyHtml;
@@ -17563,11 +17573,11 @@ var parseEmailMimeParts = (body, contentType) => {
       const nestedContentType = headers.get("content-type");
       const disposition = headers.get("content-disposition");
       const dispositionType = disposition?.split(";")[0]?.trim().toLowerCase();
-      const transferEncoding = headers.get("content-transfer-encoding");
+      const transferEncoding2 = headers.get("content-transfer-encoding");
       const contentId = normalizeEmailContentId(headers.get("content-id"));
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
-      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding);
+      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
       const decodedText = Buffer.from(decodedBytes).toString("utf8");
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
@@ -17615,7 +17625,8 @@ var parseEmailMimeParts = (body, contentType) => {
       }
     }
   };
-  collectMimeParts(body, contentType);
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeUtf8(decodeEmailPartBody(body, transferEncoding)) : body;
+  collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
     bodyHtml,
@@ -17631,7 +17642,7 @@ var authoredEmailText = (text) => {
 var extractEmailText = (raw) => {
   const { body, headerBlock } = splitEmailMessage(raw);
   const headers = parseHeaderBlock(headerBlock);
-  const parsed = parseEmailMimeParts(body, headers.get("content-type"));
+  const parsed = parseEmailMimeParts(body, headers.get("content-type"), headers.get("content-transfer-encoding"));
   const htmlText = parsed.bodyHtml ? stripEmailHtml(parsed.bodyHtml) : undefined;
   const plainText = parsed.bodyText ? normalizeWhitespace(parsed.bodyText) : undefined;
   const preferredBodyText = choosePreferredEmailBodyText(htmlText, plainText);
@@ -17688,6 +17699,7 @@ var parseEmailHeaders = (raw) => {
     ccAddressEntries: ccParsed.entries,
     ccAddresses: ccParsed.addresses,
     contentType: getHeader("Content-Type"),
+    contentTransferEncoding: getHeader("Content-Transfer-Encoding"),
     from,
     fromAddress: fromParsed.addresses[0],
     fromAddressEntries: fromParsed.entries,
@@ -17712,7 +17724,7 @@ var parseEmailHeaders = (raw) => {
 var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   const headers = parseEmailHeaders(raw);
   const { body } = splitEmailMessage(raw);
-  const parsed = parseEmailMimeParts(body, headers.contentType);
+  const parsed = parseEmailMimeParts(body, headers.contentType, headers.contentTransferEncoding);
   const source = options?.source ?? input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
   const maildirMetadata = parseMaildirMetadata(source);
   const mergedMetadata = {
@@ -17915,7 +17927,7 @@ var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   return [messageDocument, ...attachmentDocuments.flat()];
 };
 var normalizeEmailThreadKey = (value) => {
-  const normalized = normalizeWhitespace(value?.replace(/^(re|fw|fwd)\s*:\s*/gi, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
+  const normalized = normalizeWhitespace(value?.replace(/^(?:(?:re|fw|fwd|aw|sv|vs|antw)\s*:\s*)+/i, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
   return normalized || undefined;
 };
 var normalizeEmailMessageId = (value) => {
@@ -36371,5 +36383,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=176571A7F3C6808164756E2164756E21
+//# debugId=E2463B33C7EF84EC64756E2164756E21
 //# sourceMappingURL=index.js.map