@absolutejs/absolute 0.19.0-beta.501 → 0.19.0-beta.503

package/dist/ai/index.js

@@ -6833,6 +6833,294 @@ var openaiTranscriber = (config) => {
     }
   });
 };
+// src/ai/rag/emailProviders.ts
+import { connect as connectTls } from "tls";
+var defaultFetch = (...args) => fetch(...args);
+var stripHtml2 = (value) => value.replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/\s+/g, " ").trim();
+var decodeBase64Url = (value) => {
+  const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
+  const padding = normalized.length % 4 === 0 ? "" : "=".repeat(4 - normalized.length % 4);
+  return Buffer.from(normalized + padding, "base64");
+};
+var firstHeader = (headers, name) => headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value;
+var collectGmailPayloadParts = (payload) => {
+  if (!payload) {
+    return [];
+  }
+  return [
+    payload,
+    ...(payload.parts ?? []).flatMap((part) => collectGmailPayloadParts(part))
+  ];
+};
+var extractGmailBodyText = (payload) => {
+  const parts = collectGmailPayloadParts(payload);
+  const plain = parts.find((part) => part.mimeType?.toLowerCase() === "text/plain" && part.body?.data);
+  if (plain?.body?.data) {
+    return decodeBase64Url(plain.body.data).toString("utf8");
+  }
+  const html = parts.find((part) => part.mimeType?.toLowerCase() === "text/html" && part.body?.data);
+  if (html?.body?.data) {
+    return stripHtml2(decodeBase64Url(html.body.data).toString("utf8"));
+  }
+  if (payload?.body?.data) {
+    return decodeBase64Url(payload.body.data).toString("utf8");
+  }
+  return "";
+};
+var toGraphAddressList = (value) => value?.map((entry) => entry.emailAddress?.address).filter(Boolean);
+var toGraphAttachment = (attachment) => attachment.name && attachment.contentBytes ? {
+  content: attachment.contentBytes,
+  contentType: attachment.contentType,
+  encoding: "base64",
+  id: attachment.id,
+  name: attachment.name
+} : null;
+var buildGraphPath = (config) => {
+  const base = config.userId && config.userId !== "me" ? `/users/${encodeURIComponent(config.userId)}` : "/me";
+  return config.folderId ? `${base}/mailFolders/${encodeURIComponent(config.folderId)}/messages` : `${base}/messages`;
+};
+var parseRawEmail = (raw) => {
+  const [headerBlock, ...bodyBlocks] = raw.split(/\r?\n\r?\n/);
+  const body = bodyBlocks.join(`
+
+`).trim();
+  const lines = (headerBlock ?? "").split(/\r?\n/);
+  const headers = new Map;
+  let current = "";
+  for (const line of lines) {
+    if (/^\s/.test(line) && current) {
+      headers.set(current, `${headers.get(current) ?? ""} ${line.trim()}`.trim());
+      continue;
+    }
+    const separator = line.indexOf(":");
+    if (separator < 0) {
+      continue;
+    }
+    current = line.slice(0, separator).trim().toLowerCase();
+    headers.set(current, line.slice(separator + 1).trim());
+  }
+  const toList = (value) => value?.split(",").map((entry) => entry.trim()).filter(Boolean);
+  return {
+    bodyText: body,
+    cc: toList(headers.get("cc")),
+    from: headers.get("from"),
+    id: headers.get("message-id") ?? headers.get("subject") ?? crypto.randomUUID(),
+    receivedAt: headers.get("date"),
+    subject: headers.get("subject"),
+    threadId: headers.get("thread-topic") ?? headers.get("references") ?? headers.get("subject"),
+    to: toList(headers.get("to"))
+  };
+};
+var createRAGGmailEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const fetchImpl = config.fetch ?? defaultFetch;
+    const userId = config.userId ?? "me";
+    const listUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages`);
+    listUrl.searchParams.set("maxResults", String(input?.maxResults ?? config.maxResults ?? 100));
+    if (input?.cursor) {
+      listUrl.searchParams.set("pageToken", input.cursor);
+    }
+    if (config.query) {
+      listUrl.searchParams.set("q", config.query);
+    }
+    for (const labelId of config.labelIds ?? []) {
+      listUrl.searchParams.append("labelIds", labelId);
+    }
+    if (config.includeSpamTrash) {
+      listUrl.searchParams.set("includeSpamTrash", "true");
+    }
+    const listResponse = await fetchImpl(listUrl, {
+      headers: { Authorization: `Bearer ${config.accessToken}` }
+    });
+    if (!listResponse.ok) {
+      throw new Error(`Gmail list failed: ${listResponse.status} ${listResponse.statusText}`);
+    }
+    const listJson = await listResponse.json();
+    const messages = await Promise.all((listJson.messages ?? []).map(async (messageRef) => {
+      const getUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages/${encodeURIComponent(messageRef.id)}`);
+      getUrl.searchParams.set("format", "full");
+      const response = await fetchImpl(getUrl, {
+        headers: { Authorization: `Bearer ${config.accessToken}` }
+      });
+      if (!response.ok) {
+        throw new Error(`Gmail get failed for ${messageRef.id}: ${response.status} ${response.statusText}`);
+      }
+      const json = await response.json();
+      const attachments = await Promise.all(collectGmailPayloadParts(json.payload).filter((part) => !!part.filename && !!part.body?.attachmentId).map(async (part) => {
+        const attachmentUrl = new URL(`https://gmail.googleapis.com/gmail/v1/users/${encodeURIComponent(userId)}/messages/${encodeURIComponent(json.id)}/attachments/${encodeURIComponent(part.body?.attachmentId ?? "")}`);
+        const attachmentResponse = await fetchImpl(attachmentUrl, {
+          headers: {
+            Authorization: `Bearer ${config.accessToken}`
+          }
+        });
+        if (!attachmentResponse.ok) {
+          return null;
+        }
+        const attachmentJson = await attachmentResponse.json();
+        if (!attachmentJson.data || !part.filename) {
+          return null;
+        }
+        return {
+          content: decodeBase64Url(attachmentJson.data).toString("base64"),
+          contentType: part.mimeType,
+          encoding: "base64",
+          id: part.body?.attachmentId,
+          name: part.filename
+        };
+      }));
+      return {
+        attachments: attachments.filter(Boolean),
+        bodyText: extractGmailBodyText(json.payload) || json.snippet || "",
+        cc: firstHeader(json.payload?.headers, "Cc")?.split(",").map((entry) => entry.trim()).filter(Boolean),
+        from: firstHeader(json.payload?.headers, "From"),
+        id: json.id,
+        metadata: {
+          gmailLabelIds: json.labelIds,
+          provider: "gmail"
+        },
+        receivedAt: json.internalDate ? Number(json.internalDate) : undefined,
+        subject: firstHeader(json.payload?.headers, "Subject"),
+        threadId: json.threadId ?? messageRef.threadId,
+        to: firstHeader(json.payload?.headers, "To")?.split(",").map((entry) => entry.trim()).filter(Boolean)
+      };
+    }));
+    return {
+      messages,
+      nextCursor: listJson.nextPageToken
+    };
+  }
+});
+var createRAGGraphEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const fetchImpl = config.fetch ?? defaultFetch;
+    const url = new URL(`${config.baseUrl ?? "https://graph.microsoft.com/v1.0"}${buildGraphPath(config)}`);
+    url.searchParams.set("$select", "id,conversationId,subject,from,toRecipients,ccRecipients,sentDateTime,receivedDateTime,body,hasAttachments,internetMessageId");
+    url.searchParams.set("$top", String(input?.maxResults ?? config.top ?? 100));
+    if (config.filter) {
+      url.searchParams.set("$filter", config.filter);
+    }
+    if (config.search) {
+      url.searchParams.set("$search", config.search);
+    }
+    if (input?.cursor) {
+      return {
+        messages: [],
+        nextCursor: input.cursor
+      };
+    }
+    const listResponse = await fetchImpl(url, {
+      headers: {
+        Authorization: `Bearer ${config.accessToken}`,
+        Prefer: 'outlook.body-content-type="text"'
+      }
+    });
+    if (!listResponse.ok) {
+      throw new Error(`Graph message list failed: ${listResponse.status} ${listResponse.statusText}`);
+    }
+    const listJson = await listResponse.json();
+    const messages = await Promise.all((listJson.value ?? []).map(async (message) => {
+      const attachments = message.hasAttachments === true ? await (async () => {
+        const attachmentsResponse = await fetchImpl(`${config.baseUrl ?? "https://graph.microsoft.com/v1.0"}${buildGraphPath(config)}/${encodeURIComponent(String(message.id))}/attachments`, {
+          headers: {
+            Authorization: `Bearer ${config.accessToken}`
+          }
+        });
+        if (!attachmentsResponse.ok) {
+          return [];
+        }
+        const attachmentsJson = await attachmentsResponse.json();
+        return (attachmentsJson.value ?? []).map((attachment) => toGraphAttachment(attachment)).filter(Boolean);
+      })() : [];
+      return {
+        attachments,
+        bodyText: typeof message.body?.content === "string" ? String(message.body.content) : "",
+        cc: toGraphAddressList(message.ccRecipients),
+        from: message.from?.emailAddress?.address,
+        id: String(message.id),
+        metadata: {
+          internetMessageId: message.internetMessageId,
+          provider: "graph"
+        },
+        receivedAt: message.receivedDateTime,
+        sentAt: message.sentDateTime,
+        subject: message.subject,
+        threadId: message.conversationId,
+        to: toGraphAddressList(message.toRecipients)
+      };
+    }));
+    return {
+      messages,
+      nextCursor: listJson["@odata.nextLink"]
+    };
+  }
+});
+var createRAGIMAPEmailSyncClient = (config) => ({
+  listMessages: async (input) => {
+    const mailbox = config.mailbox ?? "INBOX";
+    const searchTerms = config.search?.length ? config.search : ["ALL"];
+    const maxResults = input?.maxResults ?? config.maxResults ?? 100;
+    const socket = connectTls({
+      host: config.host,
+      port: config.port ?? 993,
+      rejectUnauthorized: true
+    });
+    socket.setEncoding("utf8");
+    let buffer = "";
+    const readUntil = async (predicate) => await new Promise((resolve2, reject) => {
+      const onData = (chunk) => {
+        buffer += chunk;
+        if (predicate(buffer)) {
+          socket.off("data", onData);
+          resolve2(buffer);
+        }
+      };
+      const onError = (error) => {
+        socket.off("data", onData);
+        reject(error);
+      };
+      socket.on("data", onData);
+      socket.once("error", onError);
+    });
+    const send = async (tag, command) => {
+      socket.write(`${tag} ${command}\r
+`);
+      const response = await readUntil((chunk) => chunk.includes(`\r
+${tag} OK`) || chunk.startsWith(`${tag} OK`));
+      buffer = "";
+      return response;
+    };
+    await readUntil((chunk) => chunk.includes(`\r
+`) || chunk.startsWith("*"));
+    buffer = "";
+    await send("A001", `LOGIN "${config.username}" "${config.password}"`);
+    await send("A002", `SELECT "${mailbox}"`);
+    const searchResponse = await send("A003", `UID SEARCH ${searchTerms.join(" ")}`);
+    const uidLine = searchResponse.split(/\r?\n/).find((line) => line.startsWith("* SEARCH"));
+    const uids = (uidLine?.replace("* SEARCH", "").trim().split(/\s+/) ?? []).filter(Boolean).slice(-maxResults);
+    const messages = [];
+    for (const uid of uids) {
+      const response = await send(`F${uid}`, `UID FETCH ${uid} BODY.PEEK[]`);
+      const rawMatch = response.match(/\{(\d+)\}\r\n([\s\S]*?)\r\nF[^\r\n]+ OK/);
+      const raw = rawMatch?.[2]?.trim();
+      if (!raw) {
+        continue;
+      }
+      const parsed = parseRawEmail(raw);
+      messages.push({
+        ...parsed,
+        metadata: {
+          ...parsed.metadata ?? {},
+          imapMailbox: mailbox,
+          imapUid: uid,
+          provider: "imap"
+        }
+      });
+    }
+    socket.end(`ZZZZ LOGOUT\r
+`);
+    return { messages };
+  }
+});
 // src/ai/rag/presentation.ts
 var buildSourceGroupKey = (source) => source.source ?? source.title ?? source.chunkId;
 var buildSourceLabel = (source) => source.source ?? source.title ?? source.chunkId;
@@ -7217,6 +7505,23 @@ var toManagedSyncDocument = (sourceId, document, syncKey) => ({
     syncSourceId: sourceId
   }
 });
+var encodeAttachmentContent = (attachment) => typeof attachment.content === "string" ? {
+  content: attachment.content,
+  encoding: attachment.encoding ?? "utf8"
+} : {
+  content: Buffer.from(attachment.content).toString("base64"),
+  encoding: "base64"
+};
+var toTimestamp = (value) => {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string" || value instanceof Date) {
+    const parsed = new Date(value).getTime();
+    return Number.isFinite(parsed) ? parsed : undefined;
+  }
+  return;
+};
 var isManagedBySyncSource = (document, sourceId) => document.metadata?.syncSourceId === sourceId;
 var getDocumentSyncFingerprint = (document) => typeof document.metadata?.syncFingerprint === "string" ? document.metadata.syncFingerprint : undefined;
 var reconcileManagedDocuments = async (input) => {
@@ -7442,6 +7747,95 @@ var createRAGStorageSyncSource = (options) => ({
     };
   }
 });
+var createRAGStaticEmailSyncClient = (input) => ({
+  listMessages: (options) => ({
+    messages: typeof options?.maxResults === "number" ? input.messages.slice(0, options.maxResults) : input.messages
+  })
+});
+var createRAGEmailSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "email",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.label,
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const listed = await options.client.listMessages({
+      maxResults: options.maxResults
+    });
+    const messageDocuments = listed.messages.map((message) => ({
+      chunking: options.defaultChunking,
+      format: message.bodyHtml ? "html" : "text",
+      id: `email-${message.id}`,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...message.metadata ?? {},
+        emailKind: "message",
+        from: message.from,
+        hasAttachments: (message.attachments?.length ?? 0) > 0,
+        messageId: message.id,
+        receivedAt: toTimestamp(message.receivedAt),
+        sentAt: toTimestamp(message.sentAt),
+        threadId: message.threadId,
+        threadTopic: message.subject,
+        to: message.to,
+        cc: message.cc
+      },
+      source: `email/${message.threadId ?? message.id}`,
+      text: message.bodyText,
+      title: message.subject ?? message.id
+    }));
+    const attachmentUploads = listed.messages.flatMap((message) => (message.attachments ?? []).map((attachment, index) => ({
+      ...encodeAttachmentContent(attachment),
+      chunking: attachment.chunking ?? options.defaultChunking,
+      contentType: attachment.contentType,
+      format: attachment.format,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...attachment.metadata ?? {},
+        attachmentId: attachment.id ?? `${message.id}-attachment-${index + 1}`,
+        emailKind: "attachment",
+        from: message.from,
+        messageId: message.id,
+        sentAt: toTimestamp(message.sentAt),
+        threadId: message.threadId,
+        threadTopic: message.subject
+      },
+      name: attachment.name,
+      source: attachment.source ?? `email/${message.threadId ?? message.id}/attachments/${attachment.name}`,
+      title: attachment.title ?? `${message.subject ?? message.id} \xB7 ${attachment.name}`
+    })));
+    const loadedAttachments = attachmentUploads.length > 0 ? await loadRAGDocumentsFromUploads({
+      baseMetadata: options.baseMetadata,
+      defaultChunking: options.defaultChunking,
+      extractors: options.extractors,
+      uploads: attachmentUploads
+    }) : { documents: [] };
+    const managedDocuments = [
+      ...messageDocuments.map((document) => toManagedSyncDocument(options.id, document, `message:${document.metadata?.messageId}`)),
+      ...loadedAttachments.documents.map((document) => toManagedSyncDocument(options.id, document, `attachment:${String(document.metadata?.attachmentId ?? document.source ?? document.title ?? "")}`))
+    ];
+    const reconciled = await reconcileManagedDocuments({
+      collection,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        deletedCount: reconciled.deletedCount,
+        messageCount: listed.messages.length,
+        nextCursor: listed.nextCursor,
+        updatedCount: reconciled.updatedCount
+      }
+    };
+  }
+});
 var createRAGSyncManager = (options) => {
   const sourceMap = new Map(options.sources.map((source) => [source.id, source]));
   const state = new Map(options.sources.map((source) => [source.id, toSourceRecord(source)]));
@@ -9642,6 +10036,7 @@ export {
   createRAGSyncScheduler,
   createRAGSyncManager,
   createRAGStorageSyncSource,
+  createRAGStaticEmailSyncClient,
   createRAGReranker,
   createRAGQueryTransform,
   createRAGPDFOCRExtractor,
@@ -9649,12 +10044,16 @@ export {
   createRAGMediaTranscriber,
   createRAGMediaFileExtractor,
   createRAGImageOCRExtractor,
+  createRAGIMAPEmailSyncClient,
   createRAGHTMXWorkflowRenderConfig,
   createRAGHTMXConfig,
+  createRAGGraphEmailSyncClient,
+  createRAGGmailEmailSyncClient,
   createRAGFileSyncStateStore,
   createRAGFileExtractor,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
+  createRAGEmailSyncSource,
   createRAGDirectorySyncSource,
   createRAGCollection,
   createRAGClient,
@@ -9697,5 +10096,5 @@ export {
   aiChat
 };
 
-//# debugId=1890BBB7D959483A64756E2164756E21
+//# debugId=BD95265DF2B97B7764756E2164756E21
 //# sourceMappingURL=index.js.map