@absolutejs/rag 0.3.0 → 0.5.0

package/dist/index.js

@@ -15975,6 +15975,7 @@ var extractNativePDFText = (data) => {
 var readUInt16LE = (data, offset) => data[offset] | data[offset + 1] << 8;
 var readUInt32LE = (data, offset) => (data[offset] | data[offset + 1] << 8 | data[offset + 2] << 16 | data[offset + 3] << 24) >>> 0;
 var decodeUtf8 = (data) => Buffer.from(data).toString("utf8");
+var decodeLatin1 = (data) => Buffer.from(data).toString("latin1");
 var isZipData = (data) => data.length >= 4 && data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
 var unzipEntries = (data) => {
   const entries = [];
@@ -17126,6 +17127,16 @@ var parseMaildirMetadata = (source) => {
     ...maildirKey ? { emailMailboxKey: maildirKey } : {}
   };
 };
+var RFC2047_WORD = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g;
+var decodeEncodedWord = (charset, encoding, text) => {
+  const bytes = encoding.toUpperCase() === "B" ? Buffer.from(text, "base64") : Buffer.from(decodeQuotedPrintable(text.replace(/_/g, " ")), "latin1");
+  try {
+    return new TextDecoder(charset.trim().toLowerCase()).decode(bytes);
+  } catch {
+    return bytes.toString("utf8");
+  }
+};
+var decodeRfc2047 = (value) => value.includes("=?") ? value.replace(/\?=\s+=\?/g, "?==?").replace(RFC2047_WORD, (_match, charset, enc, text) => decodeEncodedWord(charset, enc, text)) : value;
 var parseHeaderBlock = (headerBlock) => {
   const unfolded = headerBlock.replace(/\n[ \t]+/g, " ");
   const headers = new Map;
@@ -17135,7 +17146,7 @@ var parseHeaderBlock = (headerBlock) => {
     if (separator < 0) {
       continue;
     }
-    headers.set(line.slice(0, separator).trim().toLowerCase(), line.slice(separator + 1).trim());
+    headers.set(line.slice(0, separator).trim().toLowerCase(), decodeRfc2047(line.slice(separator + 1).trim()));
   }
   return headers;
 };
@@ -17147,9 +17158,21 @@ var decodeEmailPartBody = (body, encoding) => {
     return new Uint8Array(Buffer.from(trimmed.replace(/\s+/g, ""), "base64"));
   }
   if (normalizedEncoding === "quoted-printable") {
-    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "utf8"));
+    return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
+  }
+  return new Uint8Array(Buffer.from(body, "latin1"));
+};
+var parseMimeCharset = (contentType) => contentType?.match(/charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
+var decodeWithCharset = (data, contentType) => {
+  const charset = parseMimeCharset(contentType)?.trim().toLowerCase();
+  if (!charset || charset === "utf-8" || charset === "utf8") {
+    return decodeUtf8(data);
+  }
+  try {
+    return new TextDecoder(charset).decode(data);
+  } catch {
+    return decodeUtf8(data);
   }
-  return new Uint8Array(Buffer.from(body, "utf8"));
 };
 var parseMimeBoundary = (contentType) => {
   const match = contentType?.match(/boundary="?([^";]+)"?/i);
@@ -17541,7 +17564,7 @@ var chooseEmailBodyCandidate = (current, candidate) => {
   }
   return candidate.length > current.length ? candidate : current;
 };
-var parseEmailMimeParts = (body, contentType) => {
+var parseEmailMimeParts = (body, contentType, transferEncoding) => {
   const attachments = [];
   let bodyText;
   let bodyHtml;
@@ -17563,12 +17586,12 @@ var parseEmailMimeParts = (body, contentType) => {
       const nestedContentType = headers.get("content-type");
       const disposition = headers.get("content-disposition");
       const dispositionType = disposition?.split(";")[0]?.trim().toLowerCase();
-      const transferEncoding = headers.get("content-transfer-encoding");
+      const transferEncoding2 = headers.get("content-transfer-encoding");
       const contentId = normalizeEmailContentId(headers.get("content-id"));
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
-      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding);
-      const decodedText = Buffer.from(decodedBytes).toString("utf8");
+      const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
+      const decodedText = decodeWithCharset(decodedBytes, nestedContentType);
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
       const isHtml = normalizedContentType.includes("text/html");
@@ -17615,7 +17638,8 @@ var parseEmailMimeParts = (body, contentType) => {
       }
     }
   };
-  collectMimeParts(body, contentType);
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeWithCharset(decodeEmailPartBody(body, transferEncoding), contentType) : body;
+  collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
     bodyHtml,
@@ -17631,7 +17655,7 @@ var authoredEmailText = (text) => {
 var extractEmailText = (raw) => {
   const { body, headerBlock } = splitEmailMessage(raw);
   const headers = parseHeaderBlock(headerBlock);
-  const parsed = parseEmailMimeParts(body, headers.get("content-type"));
+  const parsed = parseEmailMimeParts(body, headers.get("content-type"), headers.get("content-transfer-encoding"));
   const htmlText = parsed.bodyHtml ? stripEmailHtml(parsed.bodyHtml) : undefined;
   const plainText = parsed.bodyText ? normalizeWhitespace(parsed.bodyText) : undefined;
   const preferredBodyText = choosePreferredEmailBodyText(htmlText, plainText);
@@ -17688,6 +17712,7 @@ var parseEmailHeaders = (raw) => {
     ccAddressEntries: ccParsed.entries,
     ccAddresses: ccParsed.addresses,
     contentType: getHeader("Content-Type"),
+    contentTransferEncoding: getHeader("Content-Transfer-Encoding"),
     from,
     fromAddress: fromParsed.addresses[0],
     fromAddressEntries: fromParsed.entries,
@@ -17712,7 +17737,7 @@ var parseEmailHeaders = (raw) => {
 var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   const headers = parseEmailHeaders(raw);
   const { body } = splitEmailMessage(raw);
-  const parsed = parseEmailMimeParts(body, headers.contentType);
+  const parsed = parseEmailMimeParts(body, headers.contentType, headers.contentTransferEncoding);
   const source = options?.source ?? input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
   const maildirMetadata = parseMaildirMetadata(source);
   const mergedMetadata = {
@@ -17915,7 +17940,7 @@ var extractEmailDocumentsFromRawMessage = async (input, raw, options) => {
   return [messageDocument, ...attachmentDocuments.flat()];
 };
 var normalizeEmailThreadKey = (value) => {
-  const normalized = normalizeWhitespace(value?.replace(/^(re|fw|fwd)\s*:\s*/gi, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
+  const normalized = normalizeWhitespace(value?.replace(/^(?:(?:re|fw|fwd|aw|sv|vs|antw)\s*:\s*)+/i, "")?.replace(/[<>]/g, "")?.toLowerCase() ?? "");
   return normalized || undefined;
 };
 var normalizeEmailMessageId = (value) => {
@@ -18263,7 +18288,7 @@ var createEmailExtractor = () => ({
     const source = input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
     const extension = inferExtensionFromInput(input);
     const emlx = extension === ".emlx" ? decodeEmlxMessageData(input.data) : undefined;
-    const raw = emlx?.raw ?? decodeUtf8(input.data);
+    const raw = emlx?.raw ?? decodeLatin1(input.data);
     if (extension === ".emlx") {
       return extractEmailDocumentsFromRawMessage(input, raw, {
         metadata: {
@@ -36371,5 +36396,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=176571A7F3C6808164756E2164756E21
+//# debugId=935E9CE2ED17498064756E2164756E21
 //# sourceMappingURL=index.js.map