@absolutejs/absolute 0.19.0-beta.500 → 0.19.0-beta.502

package/dist/ai/index.js

@@ -7186,6 +7186,7 @@ var resolveRAGStreamStage = ({
   return "streaming";
 };
 // src/ai/rag/sync.ts
+var {S3Client } = globalThis.Bun;
 import { createHash } from "crypto";
 import { mkdir, readFile as readFile2, writeFile } from "fs/promises";
 import { dirname, resolve as resolve2 } from "path";
@@ -7216,6 +7217,23 @@ var toManagedSyncDocument = (sourceId, document, syncKey) => ({
     syncSourceId: sourceId
   }
 });
+var encodeAttachmentContent = (attachment) => typeof attachment.content === "string" ? {
+  content: attachment.content,
+  encoding: attachment.encoding ?? "utf8"
+} : {
+  content: Buffer.from(attachment.content).toString("base64"),
+  encoding: "base64"
+};
+var toTimestamp = (value) => {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string" || value instanceof Date) {
+    const parsed = new Date(value).getTime();
+    return Number.isFinite(parsed) ? parsed : undefined;
+  }
+  return;
+};
 var isManagedBySyncSource = (document, sourceId) => document.metadata?.syncSourceId === sourceId;
 var getDocumentSyncFingerprint = (document) => typeof document.metadata?.syncFingerprint === "string" ? document.metadata.syncFingerprint : undefined;
 var reconcileManagedDocuments = async (input) => {
@@ -7337,6 +7355,199 @@ var createRAGUrlSyncSource = (options) => ({
     };
   }
 });
+var createRAGBunS3SyncClient = (input) => {
+  const client = input instanceof S3Client ? input : new S3Client(input);
+  return {
+    file: (key) => client.file(key),
+    list: async (options) => {
+      const result = await client.list({
+        maxKeys: options?.maxKeys,
+        prefix: options?.prefix,
+        startAfter: options?.startAfter
+      });
+      return {
+        contents: (result.contents ?? []).map((entry) => ({
+          etag: entry.eTag,
+          key: entry.key,
+          lastModified: entry.lastModified,
+          size: entry.size
+        })),
+        isTruncated: result.isTruncated,
+        nextContinuationToken: result.nextContinuationToken
+      };
+    }
+  };
+};
+var createRAGStorageSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "storage",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.keys?.length ? `${options.keys.length} object${options.keys.length === 1 ? "" : "s"}` : options.prefix ?? "storage://",
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const keys = options.keys && options.keys.length > 0 ? options.keys : await (async () => {
+      const listed = [];
+      let startAfter;
+      let remaining = options.maxKeys;
+      for (;; ) {
+        const response = await options.client.list({
+          maxKeys: typeof remaining === "number" ? Math.max(1, remaining) : undefined,
+          prefix: options.prefix,
+          startAfter
+        });
+        for (const entry of response.contents) {
+          listed.push(entry.key);
+          startAfter = entry.key;
+          if (typeof remaining === "number" && listed.length >= remaining) {
+            return listed;
+          }
+        }
+        if (!response.isTruncated || response.contents.length === 0) {
+          return listed;
+        }
+        if (typeof remaining === "number") {
+          remaining -= response.contents.length;
+          if (remaining <= 0) {
+            return listed;
+          }
+        }
+      }
+    })();
+    const uploads = await Promise.all(keys.map(async (key) => {
+      const object = options.client.file(key);
+      const bytes = new Uint8Array(await object.arrayBuffer());
+      return {
+        chunking: options.defaultChunking,
+        content: Buffer.from(bytes).toString("base64"),
+        contentType: undefined,
+        encoding: "base64",
+        metadata: {
+          ...options.baseMetadata ?? {},
+          storageKey: key
+        },
+        name: key.split("/").at(-1) ?? key,
+        source: `storage/${key}`,
+        title: key.split("/").at(-1) ?? key
+      };
+    }));
+    const loaded = await loadRAGDocumentsFromUploads({
+      baseMetadata: options.baseMetadata,
+      defaultChunking: options.defaultChunking,
+      extractors: options.extractors,
+      uploads
+    });
+    const managedDocuments = loaded.documents.map((document) => toManagedSyncDocument(options.id, document, typeof document.metadata?.storageKey === "string" ? document.metadata.storageKey : document.source ?? document.title ?? ""));
+    const reconciled = await reconcileManagedDocuments({
+      collection,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        deletedCount: reconciled.deletedCount,
+        keyCount: keys.length,
+        prefix: options.prefix,
+        updatedCount: reconciled.updatedCount
+      }
+    };
+  }
+});
+var createRAGStaticEmailSyncClient = (input) => ({
+  listMessages: (options) => ({
+    messages: typeof options?.maxResults === "number" ? input.messages.slice(0, options.maxResults) : input.messages
+  })
+});
+var createRAGEmailSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "email",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.label,
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const listed = await options.client.listMessages({
+      maxResults: options.maxResults
+    });
+    const messageDocuments = listed.messages.map((message) => ({
+      chunking: options.defaultChunking,
+      format: message.bodyHtml ? "html" : "text",
+      id: `email-${message.id}`,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...message.metadata ?? {},
+        emailKind: "message",
+        from: message.from,
+        hasAttachments: (message.attachments?.length ?? 0) > 0,
+        messageId: message.id,
+        receivedAt: toTimestamp(message.receivedAt),
+        sentAt: toTimestamp(message.sentAt),
+        threadId: message.threadId,
+        threadTopic: message.subject,
+        to: message.to,
+        cc: message.cc
+      },
+      source: `email/${message.threadId ?? message.id}`,
+      text: message.bodyText,
+      title: message.subject ?? message.id
+    }));
+    const attachmentUploads = listed.messages.flatMap((message) => (message.attachments ?? []).map((attachment, index) => ({
+      ...encodeAttachmentContent(attachment),
+      chunking: attachment.chunking ?? options.defaultChunking,
+      contentType: attachment.contentType,
+      format: attachment.format,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...attachment.metadata ?? {},
+        attachmentId: attachment.id ?? `${message.id}-attachment-${index + 1}`,
+        emailKind: "attachment",
+        from: message.from,
+        messageId: message.id,
+        sentAt: toTimestamp(message.sentAt),
+        threadId: message.threadId,
+        threadTopic: message.subject
+      },
+      name: attachment.name,
+      source: attachment.source ?? `email/${message.threadId ?? message.id}/attachments/${attachment.name}`,
+      title: attachment.title ?? `${message.subject ?? message.id} \xB7 ${attachment.name}`
+    })));
+    const loadedAttachments = attachmentUploads.length > 0 ? await loadRAGDocumentsFromUploads({
+      baseMetadata: options.baseMetadata,
+      defaultChunking: options.defaultChunking,
+      extractors: options.extractors,
+      uploads: attachmentUploads
+    }) : { documents: [] };
+    const managedDocuments = [
+      ...messageDocuments.map((document) => toManagedSyncDocument(options.id, document, `message:${document.metadata?.messageId}`)),
+      ...loadedAttachments.documents.map((document) => toManagedSyncDocument(options.id, document, `attachment:${String(document.metadata?.attachmentId ?? document.source ?? document.title ?? "")}`))
+    ];
+    const reconciled = await reconcileManagedDocuments({
+      collection,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        deletedCount: reconciled.deletedCount,
+        messageCount: listed.messages.length,
+        nextCursor: listed.nextCursor,
+        updatedCount: reconciled.updatedCount
+      }
+    };
+  }
+});
 var createRAGSyncManager = (options) => {
   const sourceMap = new Map(options.sources.map((source) => [source.id, source]));
   const state = new Map(options.sources.map((source) => [source.id, toSourceRecord(source)]));
@@ -9536,6 +9747,8 @@ export {
   createAIStream as createRAGTransport,
   createRAGSyncScheduler,
   createRAGSyncManager,
+  createRAGStorageSyncSource,
+  createRAGStaticEmailSyncClient,
   createRAGReranker,
   createRAGQueryTransform,
   createRAGPDFOCRExtractor,
@@ -9549,9 +9762,11 @@ export {
   createRAGFileExtractor,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
+  createRAGEmailSyncSource,
   createRAGDirectorySyncSource,
   createRAGCollection,
   createRAGClient,
+  createRAGBunS3SyncClient,
   createRAGArchiveFileExtractor,
   createRAGArchiveExpander,
   createRAGAnswerWorkflow,
@@ -9590,5 +9805,5 @@ export {
   aiChat
 };
 
-//# debugId=61173B84573291CC64756E2164756E21
+//# debugId=CE610CD2937858F264756E2164756E21
 //# sourceMappingURL=index.js.map