@absolutejs/rag 0.4.0 → 0.5.0

package/dist/index.js

@@ -15975,6 +15975,7 @@ var extractNativePDFText = (data) => {
 var readUInt16LE = (data, offset) => data[offset] | data[offset + 1] << 8;
 var readUInt32LE = (data, offset) => (data[offset] | data[offset + 1] << 8 | data[offset + 2] << 16 | data[offset + 3] << 24) >>> 0;
 var decodeUtf8 = (data) => Buffer.from(data).toString("utf8");
+var decodeLatin1 = (data) => Buffer.from(data).toString("latin1");
 var isZipData = (data) => data.length >= 4 && data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
 var unzipEntries = (data) => {
   const entries = [];
@@ -17159,7 +17160,19 @@ var decodeEmailPartBody = (body, encoding) => {
   if (normalizedEncoding === "quoted-printable") {
     return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
   }
-  return new Uint8Array(Buffer.from(body, "utf8"));
+  return new Uint8Array(Buffer.from(body, "latin1"));
+};
+var parseMimeCharset = (contentType) => contentType?.match(/charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
+var decodeWithCharset = (data, contentType) => {
+  const charset = parseMimeCharset(contentType)?.trim().toLowerCase();
+  if (!charset || charset === "utf-8" || charset === "utf8") {
+    return decodeUtf8(data);
+  }
+  try {
+    return new TextDecoder(charset).decode(data);
+  } catch {
+    return decodeUtf8(data);
+  }
 };
 var parseMimeBoundary = (contentType) => {
   const match = contentType?.match(/boundary="?([^";]+)"?/i);
@@ -17578,7 +17591,7 @@ var parseEmailMimeParts = (body, contentType, transferEncoding) => {
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
       const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
-      const decodedText = Buffer.from(decodedBytes).toString("utf8");
+      const decodedText = decodeWithCharset(decodedBytes, nestedContentType);
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
       const isHtml = normalizedContentType.includes("text/html");
@@ -17625,7 +17638,7 @@ var parseEmailMimeParts = (body, contentType, transferEncoding) => {
       }
     }
   };
-  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeUtf8(decodeEmailPartBody(body, transferEncoding)) : body;
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeWithCharset(decodeEmailPartBody(body, transferEncoding), contentType) : body;
   collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
@@ -18275,7 +18288,7 @@ var createEmailExtractor = () => ({
     const source = input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
     const extension = inferExtensionFromInput(input);
     const emlx = extension === ".emlx" ? decodeEmlxMessageData(input.data) : undefined;
-    const raw = emlx?.raw ?? decodeUtf8(input.data);
+    const raw = emlx?.raw ?? decodeLatin1(input.data);
     if (extension === ".emlx") {
       return extractEmailDocumentsFromRawMessage(input, raw, {
         metadata: {
@@ -36383,5 +36396,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=E2463B33C7EF84EC64756E2164756E21
+//# debugId=935E9CE2ED17498064756E2164756E21
 //# sourceMappingURL=index.js.map