@absolutejs/rag 0.4.0 → 0.6.0

package/dist/index.js

@@ -15975,6 +15975,7 @@ var extractNativePDFText = (data) => {
 var readUInt16LE = (data, offset) => data[offset] | data[offset + 1] << 8;
 var readUInt32LE = (data, offset) => (data[offset] | data[offset + 1] << 8 | data[offset + 2] << 16 | data[offset + 3] << 24) >>> 0;
 var decodeUtf8 = (data) => Buffer.from(data).toString("utf8");
+var decodeLatin1 = (data) => Buffer.from(data).toString("latin1");
 var isZipData = (data) => data.length >= 4 && data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
 var unzipEntries = (data) => {
   const entries = [];
@@ -17159,7 +17160,19 @@ var decodeEmailPartBody = (body, encoding) => {
   if (normalizedEncoding === "quoted-printable") {
     return new Uint8Array(Buffer.from(decodeQuotedPrintable(body), "latin1"));
   }
-  return new Uint8Array(Buffer.from(body, "utf8"));
+  return new Uint8Array(Buffer.from(body, "latin1"));
+};
+var parseMimeCharset = (contentType) => contentType?.match(/charset\s*=\s*"?([^";\s]+)"?/i)?.[1];
+var decodeWithCharset = (data, contentType) => {
+  const charset = parseMimeCharset(contentType)?.trim().toLowerCase();
+  if (!charset || charset === "utf-8" || charset === "utf8") {
+    return decodeUtf8(data);
+  }
+  try {
+    return new TextDecoder(charset).decode(data);
+  } catch {
+    return decodeUtf8(data);
+  }
 };
 var parseMimeBoundary = (contentType) => {
   const match = contentType?.match(/boundary="?([^";]+)"?/i);
@@ -17578,7 +17591,7 @@ var parseEmailMimeParts = (body, contentType, transferEncoding) => {
       const contentLocation = normalizeWhitespace(headers.get("content-location") ?? "");
       const filename = disposition?.match(/filename="?([^";]+)"?/i)?.[1] ?? nestedContentType?.match(/name="?([^";]+)"?/i)?.[1];
       const decodedBytes = decodeEmailPartBody(nestedBody, transferEncoding2);
-      const decodedText = Buffer.from(decodedBytes).toString("utf8");
+      const decodedText = decodeWithCharset(decodedBytes, nestedContentType);
       const normalizedContentType = nestedContentType?.toLowerCase() ?? "";
       const isMultipart = normalizedContentType.startsWith("multipart/");
       const isHtml = normalizedContentType.includes("text/html");
@@ -17625,7 +17638,7 @@ var parseEmailMimeParts = (body, contentType, transferEncoding) => {
       }
     }
   };
-  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeUtf8(decodeEmailPartBody(body, transferEncoding)) : body;
+  const topLevelBody = transferEncoding && !parseMimeBoundary(contentType) ? decodeWithCharset(decodeEmailPartBody(body, transferEncoding), contentType) : body;
   collectMimeParts(topLevelBody, contentType);
   return {
     attachments,
@@ -18275,7 +18288,7 @@ var createEmailExtractor = () => ({
     const source = input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.eml`;
     const extension = inferExtensionFromInput(input);
     const emlx = extension === ".emlx" ? decodeEmlxMessageData(input.data) : undefined;
-    const raw = emlx?.raw ?? decodeUtf8(input.data);
+    const raw = emlx?.raw ?? decodeLatin1(input.data);
     if (extension === ".emlx") {
       return extractEmailDocumentsFromRawMessage(input, raw, {
         metadata: {
@@ -36086,6 +36099,49 @@ var createSyncRAGStore = (options = {}) => {
     upsert
   };
 };
+// src/retrieval/corpus.ts
+var corpusTextHash = async (text) => {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+  return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
+};
+var planRAGCorpus = async (store, owner, desired) => {
+  const wanted = new Map;
+  for (const doc of desired) {
+    if (!wanted.has(doc.chunkId))
+      wanted.set(doc.chunkId, doc);
+  }
+  const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
+  const embed = [];
+  let unchanged = 0;
+  for (const [chunkId, doc] of wanted) {
+    const hash = await corpusTextHash(doc.text);
+    if (stored.get(chunkId) === hash)
+      unchanged += 1;
+    else
+      embed.push(doc);
+  }
+  return {
+    embed,
+    remove: [...stored.keys()].filter((chunkId) => !wanted.has(chunkId)),
+    unchanged
+  };
+};
+var reconcileRAGCorpus = async (store, owner, desired, apply) => {
+  const plan = await planRAGCorpus(store, owner, desired);
+  if (plan.remove.length > 0) {
+    await apply.remove(plan.remove, owner);
+    await store.forget(owner, plan.remove);
+  }
+  let embedded = 0;
+  if (plan.embed.length > 0) {
+    embedded = await apply.embed(plan.embed, owner);
+    await store.remember(owner, await Promise.all(plan.embed.map(async (doc) => ({
+      chunkId: doc.chunkId,
+      textHash: await corpusTextHash(doc.text)
+    }))));
+  }
+  return { embedded, removed: plan.remove.length, unchanged: plan.unchanged };
+};
 export {
   xaiEmbeddings,
   withEmbeddingBudget,
@@ -36110,6 +36166,7 @@ export {
   removeRAGSource,
   removeRAGEvaluationSuiteCaseHardNegative,
   removeRAGEvaluationSuiteCase,
+  reconcileRAGCorpus,
   ragChat as ragPlugin,
   ragChat,
   querySimilarity,
@@ -36121,6 +36178,7 @@ export {
   prepareRAGDocumentFile,
   prepareRAGDocument,
   prepareRAGDirectoryDocuments,
+  planRAGCorpus,
   persistRAGSearchTraceRecord,
   persistRAGSearchTracePruneRun,
   persistRAGRetrievalReleaseLanePolicyHistory,
@@ -36300,6 +36358,7 @@ export {
   createEPUBExtractor,
   createCohereRAGReranker,
   createBuiltinArchiveExpander,
+  corpusTextHash,
   compareRAGRetrievalTraceSummaries,
   compareRAGRetrievalStrategies,
   compareRAGRerankers,
@@ -36383,5 +36442,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=E2463B33C7EF84EC64756E2164756E21
+//# debugId=8567BF91160369B364756E2164756E21
 //# sourceMappingURL=index.js.map