@absolutejs/absolute 0.19.0-beta.502 → 0.19.0-beta.504

package/dist/ai/index.js

@@ -6833,15 +6833,342 @@ var openaiTranscriber = (config) => {
     }
   });
 };
+// src/ai/rag/emailProviders.ts
+import { connect as connectTls } from "tls";
+var defaultFetch = (...args) => fetch(...args);
+var stripHtml2 = (value) => value.replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/\s+/g, " ").trim();
+var decodeBase64Url = (value) => {
+  const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
+  const padding = normalized.length % 4 === 0 ? "" : "=".repeat(4 - normalized.length % 4);
+  return Buffer.from(normalized + padding, "base64");
+};
+var firstHeader = (headers, name) => headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value;
+var collectGmailPayloadParts = (payload) => {
+  if (!payload) {
+    return [];
+  }
+  return [
+    payload,
+    ...(payload.parts ?? []).flatMap((part) => collectGmailPayloadParts(part))
+  ];
+};
+var extractGmailBodyText = (payload) => {
+  const parts = collectGmailPayloadParts(payload);
+  const plain = parts.find((part) => part.mimeType?.toLowerCase() === "text/plain" && part.body?.data);
+  if (plain?.body?.data) {
+    return decodeBase64Url(plain.body.data).toString("utf8");
+  }
+  const html = parts.find((part) => part.mimeType?.toLowerCase() === "text/html" && part.body?.data);
+  if (html?.body?.data) {
+    return stripHtml2(decodeBase64Url(html.body.data).toString("utf8"));
+  }
+  if (payload?.body?.data) {
+    return decodeBase64Url(payload.body.data).toString("utf8");
+  }
+  return "";
+};
+var toGraphAddressList = (value) => value?.map((entry) => entry.emailAddress?.address).filter(Boolean);
+var toGraphAttachment = (attachment) => attachment.name && attachment.contentBytes ? {
+  content: attachment.contentBytes,
+  contentType: attachment.contentType,
+  encoding: "base64",
+  id: attachment.id,
+  name: attachment.name
+} : null;
+var buildGraphPath = (config) => {
+  const base = config.userId && config.userId !== "me" ? `/users/${encodeURIComponent(config.userId)}` : "/me";
+  return config.folderId ? `${base}/mailFolders/${encodeURIComponent(config.folderId)}/messages` : `${base}/messages`;
+};
+var parseRawEmail = (raw) => {
+  const [headerBlock, ...bodyBlocks] = raw.split(/\r?\n\r?\n/);
+  const body = bodyBlocks.join(`
+
+`).trim();
+  const lines = (headerBlock ?? "").split(/\r?\n/);
+  const headers = new Map;
+  let current = "";
+  for (const line of lines) {
+    if (/^\s/.test(line) && current) {
+      headers.set(current, `${headers.get(current) ?? ""} ${line.trim()}`.trim());
+      continue;
+    }
+    const separator = line.indexOf(":");
+    if (separator < 0) {
+      continue;
+    }
+    current = line.slice(0, separator).trim().toLowerCase();
+    headers.set(current, line.slice(separator + 1).trim());
+  }
+  const toList = (value) => value?.split(",").map((entry) => entry.trim()).filter(Boolean);
+  return {
+    bodyText: body,
+    cc: toList(headers.get("cc")),
+    from: headers.get("from"),
+    id: headers.get("message-id") ?? headers.get("subject") ?? crypto.randomUUID(),
+    receivedAt: headers.get("date"),
+    subject: headers.get("subject"),
+    threadId: headers.get("thread-topic") ?? headers.get("references") ?? headers.get("subject"),
+    to: toList(headers.get("to"))
+  };
+};
+var createRAGGmailEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const fetchImpl = config.fetch ?? defaultFetch;
+    const userId = config.userId ?? "me";
+    const listUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages`);
+    listUrl.searchParams.set("maxResults", String(input?.maxResults ?? config.maxResults ?? 100));
+    if (input?.cursor) {
+      listUrl.searchParams.set("pageToken", input.cursor);
+    }
+    if (config.query) {
+      listUrl.searchParams.set("q", config.query);
+    }
+    for (const labelId of config.labelIds ?? []) {
+      listUrl.searchParams.append("labelIds", labelId);
+    }
+    if (config.includeSpamTrash) {
+      listUrl.searchParams.set("includeSpamTrash", "true");
+    }
+    const listResponse = await fetchImpl(listUrl, {
+      headers: { Authorization: `Bearer ${config.accessToken}` }
+    });
+    if (!listResponse.ok) {
+      throw new Error(`Gmail list failed: ${listResponse.status} ${listResponse.statusText}`);
+    }
+    const listJson = await listResponse.json();
+    const messages = await Promise.all((listJson.messages ?? []).map(async (messageRef) => {
+      const getUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages/${encodeURIComponent(messageRef.id)}`);
+      getUrl.searchParams.set("format", "full");
+      const response = await fetchImpl(getUrl, {
+        headers: { Authorization: `Bearer ${config.accessToken}` }
+      });
+      if (!response.ok) {
+        throw new Error(`Gmail get failed for ${messageRef.id}: ${response.status} ${response.statusText}`);
+      }
+      const json = await response.json();
+      const attachments = await Promise.all(collectGmailPayloadParts(json.payload).filter((part) => !!part.filename && !!part.body?.attachmentId).map(async (part) => {
+        const attachmentUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages/${encodeURIComponent(json.id)}/attachments/${encodeURIComponent(part.body?.attachmentId ?? "")}`);
+        const attachmentResponse = await fetchImpl(attachmentUrl, {
+          headers: {
+            Authorization: `Bearer ${config.accessToken}`
+          }
+        });
+        if (!attachmentResponse.ok) {
+          return null;
+        }
+        const attachmentJson = await attachmentResponse.json();
+        if (!attachmentJson.data || !part.filename) {
+          return null;
+        }
+        return {
+          content: decodeBase64Url(attachmentJson.data).toString("base64"),
+          contentType: part.mimeType,
+          encoding: "base64",
+          id: part.body?.attachmentId,
+          name: part.filename
+        };
+      }));
+      return {
+        attachments: attachments.filter(Boolean),
+        bodyText: extractGmailBodyText(json.payload) || json.snippet || "",
+        cc: firstHeader(json.payload?.headers, "Cc")?.split(",").map((entry) => entry.trim()).filter(Boolean),
+        from: firstHeader(json.payload?.headers, "From"),
+        id: json.id,
+        metadata: {
+          gmailLabelIds: json.labelIds,
+          provider: "gmail"
+        },
+        receivedAt: json.internalDate ? Number(json.internalDate) : undefined,
+        subject: firstHeader(json.payload?.headers, "Subject"),
+        threadId: json.threadId ?? messageRef.threadId,
+        to: firstHeader(json.payload?.headers, "To")?.split(",").map((entry) => entry.trim()).filter(Boolean)
+      };
+    }));
+    return {
+      messages,
+      nextCursor: listJson.nextPageToken
+    };
+  }
+});
+var createRAGGraphEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const fetchImpl = config.fetch ?? defaultFetch;
+    const url = new URL(`${config.baseUrl ?? "https://graph.microsoft.com/v1.0"}${buildGraphPath(config)}`);
+    url.searchParams.set("$select", "id,conversationId,subject,from,toRecipients,ccRecipients,sentDateTime,receivedDateTime,body,hasAttachments,internetMessageId");
+    url.searchParams.set("$top", String(input?.maxResults ?? config.top ?? 100));
+    if (config.filter) {
+      url.searchParams.set("$filter", config.filter);
+    }
+    if (config.search) {
+      url.searchParams.set("$search", config.search);
+    }
+    if (input?.cursor) {
+      return {
+        messages: [],
+        nextCursor: input.cursor
+      };
+    }
+    const listResponse = await fetchImpl(url, {
+      headers: {
+        Authorization: `Bearer ${config.accessToken}`,
+        Prefer: 'outlook.body-content-type="text"'
+      }
+    });
+    if (!listResponse.ok) {
+      throw new Error(`Graph message list failed: ${listResponse.status} ${listResponse.statusText}`);
+    }
+    const listJson = await listResponse.json();
+    const messages = await Promise.all((listJson.value ?? []).map(async (message) => {
+      const attachments = message.hasAttachments === true ? await (async () => {
+        const attachmentsResponse = await fetchImpl(`${config.baseUrl ?? "https://graph.microsoft.com/v1.0"}${buildGraphPath(config)}/${encodeURIComponent(String(message.id))}/attachments`, {
+          headers: {
+            Authorization: `Bearer ${config.accessToken}`
+          }
+        });
+        if (!attachmentsResponse.ok) {
+          return [];
+        }
+        const attachmentsJson = await attachmentsResponse.json();
+        return (attachmentsJson.value ?? []).map((attachment) => toGraphAttachment(attachment)).filter(Boolean);
+      })() : [];
+      return {
+        attachments,
+        bodyText: typeof message.body?.content === "string" ? String(message.body.content) : "",
+        cc: toGraphAddressList(message.ccRecipients),
+        from: message.from?.emailAddress?.address,
+        id: String(message.id),
+        metadata: {
+          internetMessageId: message.internetMessageId,
+          provider: "graph"
+        },
+        receivedAt: message.receivedDateTime,
+        sentAt: message.sentDateTime,
+        subject: message.subject,
+        threadId: message.conversationId,
+        to: toGraphAddressList(message.toRecipients)
+      };
+    }));
+    return {
+      messages,
+      nextCursor: listJson["@odata.nextLink"]
+    };
+  }
+});
+var createRAGIMAPEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const mailbox = config.mailbox ?? "INBOX";
+    const searchTerms = config.search?.length ? config.search : ["ALL"];
+    const maxResults = input?.maxResults ?? config.maxResults ?? 100;
+    const socket = connectTls({
+      host: config.host,
+      port: config.port ?? 993,
+      rejectUnauthorized: true
+    });
+    socket.setEncoding("utf8");
+    let buffer = "";
+    const readUntil = async (predicate) => await new Promise((resolve2, reject) => {
+      const onData = (chunk) => {
+        buffer += chunk;
+        if (predicate(buffer)) {
+          socket.off("data", onData);
+          resolve2(buffer);
+        }
+      };
+      const onError = (error) => {
+        socket.off("data", onData);
+        reject(error);
+      };
+      socket.on("data", onData);
+      socket.once("error", onError);
+    });
+    const send = async (tag, command) => {
+      socket.write(`${tag} ${command}\r
+`);
+      const response = await readUntil((chunk) => chunk.includes(`\r
+${tag} OK`) || chunk.startsWith(`${tag} OK`));
+      buffer = "";
+      return response;
+    };
+    await readUntil((chunk) => chunk.includes(`\r
+`) || chunk.startsWith("*"));
+    buffer = "";
+    await send("A001", `LOGIN "${config.username}" "${config.password}"`);
+    await send("A002", `SELECT "${mailbox}"`);
+    const searchResponse = await send("A003", `UID SEARCH ${searchTerms.join(" ")}`);
+    const uidLine = searchResponse.split(/\r?\n/).find((line) => line.startsWith("* SEARCH"));
+    const uids = (uidLine?.replace("* SEARCH", "").trim().split(/\s+/) ?? []).filter(Boolean).slice(-maxResults);
+    const messages = [];
+    for (const uid of uids) {
+      const response = await send(`F${uid}`, `UID FETCH ${uid} BODY.PEEK[]`);
+      const rawMatch = response.match(/\{(\d+)\}\r\n([\s\S]*?)\r\nF[^\r\n]+ OK/);
+      const raw = rawMatch?.[2]?.trim();
+      if (!raw) {
+        continue;
+      }
+      const parsed = parseRawEmail(raw);
+      messages.push({
+        ...parsed,
+        metadata: {
+          ...parsed.metadata ?? {},
+          imapMailbox: mailbox,
+          imapUid: uid,
+          provider: "imap"
+        }
+      });
+    }
+    socket.end(`ZZZZ LOGOUT\r
+`);
+    return { messages };
+  }
+});
 // src/ai/rag/presentation.ts
 var buildSourceGroupKey = (source) => source.source ?? source.title ?? source.chunkId;
 var buildSourceLabel = (source) => source.source ?? source.title ?? source.chunkId;
 var getContextNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
 var getContextString = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+var formatTimestampLabel = (value) => {
+  const timestamp = typeof value === "number" && Number.isFinite(value) ? value : typeof value === "string" ? Date.parse(value) : Number.NaN;
+  if (!Number.isFinite(timestamp)) {
+    return;
+  }
+  return new Date(timestamp).toLocaleString("en-US", {
+    dateStyle: "medium",
+    timeStyle: "short"
+  });
+};
+var formatMediaTimestamp = (value) => {
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+    return;
+  }
+  const totalSeconds = Math.floor(value / 1000);
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  const milliseconds = Math.floor(value % 1000);
+  return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
+};
+var getAttachmentName = (source, title) => {
+  const sourceAttachment = source?.split("/").at(-1);
+  if (sourceAttachment && sourceAttachment.includes(".")) {
+    return sourceAttachment;
+  }
+  const titleAttachment = title?.split(" \xB7 ").at(-1);
+  if (titleAttachment && titleAttachment.includes(".")) {
+    return titleAttachment;
+  }
+  return;
+};
 var buildContextLabel = (metadata) => {
   if (!metadata) {
     return;
   }
+  const emailKind = getContextString(metadata.emailKind);
+  if (emailKind === "attachment") {
+    return "Attachment evidence";
+  }
+  if (emailKind === "message") {
+    const from = getContextString(metadata.from);
+    return from ? `Message from ${from}` : "Message evidence";
+  }
   const page = getContextNumber(metadata.page) ?? getContextNumber(metadata.pageNumber) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
   if (page) {
     return `Page ${page}`;
@@ -6868,6 +7195,57 @@ var buildContextLabel = (metadata) => {
   }
   return;
 };
+var buildLocatorLabel = (metadata, source, title) => {
+  if (!metadata) {
+    return;
+  }
+  const page = getContextNumber(metadata.page) ?? getContextNumber(metadata.pageNumber) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
+  if (page) {
+    return `Page ${page}`;
+  }
+  const sheet = getContextString(metadata.sheetName) ?? (Array.isArray(metadata.sheetNames) ? getContextString(metadata.sheetNames[0]) : undefined);
+  if (sheet) {
+    return `Sheet ${sheet}`;
+  }
+  const slide = getContextNumber(metadata.slide) ?? getContextNumber(metadata.slideNumber) ?? (typeof metadata.slideIndex === "number" ? metadata.slideIndex + 1 : undefined);
+  if (slide) {
+    return `Slide ${slide}`;
+  }
+  const archiveEntry = getContextString(metadata.archiveEntryPath) ?? getContextString(metadata.entryPath);
+  if (archiveEntry) {
+    return `Archive entry ${archiveEntry}`;
+  }
+  const emailKind = getContextString(metadata.emailKind);
+  if (emailKind === "attachment") {
+    const attachmentName = getContextString(metadata.attachmentName) ?? getAttachmentName(source, title);
+    return attachmentName ? `Attachment ${attachmentName}` : "Attachment";
+  }
+  const mediaStart = formatMediaTimestamp(metadata.startMs);
+  const mediaEnd = formatMediaTimestamp(metadata.endMs);
+  if (mediaStart && mediaEnd) {
+    return `Timestamp ${mediaStart} - ${mediaEnd}`;
+  }
+  if (mediaStart) {
+    return `Timestamp ${mediaStart}`;
+  }
+  return;
+};
+var buildProvenanceLabel = (metadata) => {
+  if (!metadata) {
+    return;
+  }
+  const threadTopic = getContextString(metadata.threadTopic);
+  const from = getContextString(metadata.from);
+  const sentAt = formatTimestampLabel(metadata.sentAt) ?? formatTimestampLabel(metadata.receivedAt);
+  const speaker = getContextString(metadata.speaker);
+  const labels = [
+    threadTopic ? `Thread ${threadTopic}` : "",
+    speaker ? `Speaker ${speaker}` : "",
+    from ? `Sender ${from}` : "",
+    sentAt ? `Sent ${sentAt}` : ""
+  ].filter((value) => value.length > 0);
+  return labels.length > 0 ? labels.join(" \xB7 ") : undefined;
+};
 var buildRAGCitationReferenceMap = (citations) => Object.fromEntries(citations.map((citation, index) => [citation.chunkId, index + 1]));
 var buildRAGCitations = (sources) => {
   const unique = new Map;
@@ -6879,9 +7257,12 @@ var buildRAGCitations = (sources) => {
       continue;
     unique.set(key, {
       chunkId: source.chunkId,
+      contextLabel: buildContextLabel(source.metadata),
       key,
       label: buildSourceLabel(source),
+      locatorLabel: buildLocatorLabel(source.metadata, source.source, source.title),
       metadata: source.metadata,
+      provenanceLabel: buildProvenanceLabel(source.metadata),
       score: source.score,
       source: source.source,
       text: source.text,
@@ -6958,8 +7339,10 @@ var buildRAGGroundingReferences = (sources) => {
     contextLabel: buildContextLabel(citation.metadata),
     excerpt: buildExcerpt(citation.text),
     label: citation.label,
+    locatorLabel: citation.locatorLabel ?? buildLocatorLabel(citation.metadata, citation.source, citation.title),
     metadata: citation.metadata,
     number: citationReferenceMap[citation.chunkId] ?? 0,
+    provenanceLabel: citation.provenanceLabel ?? buildProvenanceLabel(citation.metadata),
     score: citation.score,
     source: citation.source,
     text: citation.text,
@@ -9756,8 +10139,11 @@ export {
   createRAGMediaTranscriber,
   createRAGMediaFileExtractor,
   createRAGImageOCRExtractor,
+  createRAGIMAPEmailSyncClient,
   createRAGHTMXWorkflowRenderConfig,
   createRAGHTMXConfig,
+  createRAGGraphEmailSyncClient,
+  createRAGGmailEmailSyncClient,
   createRAGFileSyncStateStore,
   createRAGFileExtractor,
   createRAGEvaluationSuite,
@@ -9805,5 +10191,5 @@ export {
   aiChat
 };
 
-//# debugId=CE610CD2937858F264756E2164756E21
+//# debugId=A6A9C1B546E5A8FD64756E2164756E21
 //# sourceMappingURL=index.js.map