@absolutejs/rag 0.0.5 → 0.0.6

package/dist/ai/rag/index.js

@@ -31820,6 +31820,175 @@ ${tag} OK`) || chunk.startsWith(`${tag} OK`));
     return { messages };
   }
 });
+// src/ai/rag/socialProviders.ts
+var defaultFetch2 = Object.assign((...args) => fetch(...args), { preconnect: fetch.preconnect });
+var META_GRAPH_BASE_URL = "https://graph.facebook.com/v22.0";
+var DEFAULT_META_PAGE_SIZE = 25;
+var FACEBOOK_PAGE_READ_SCOPES = [
+  "pages_show_list",
+  "pages_read_engagement"
+];
+var INSTAGRAM_BUSINESS_READ_SCOPES = [
+  "pages_show_list",
+  "instagram_basic"
+];
+var toErrorMessage3 = async (response, label) => {
+  let detailMessage;
+  try {
+    const body = await response.clone().json();
+    detailMessage = body.error?.message;
+  } catch {
+    const text = await response.clone().text();
+    detailMessage = text.trim().length > 0 ? text.trim() : undefined;
+  }
+  return new Error(`${label}: ${response.status} ${response.statusText}${detailMessage ? ` (${detailMessage})` : ""}`);
+};
+var fetchGraphList = async (input) => {
+  const fetchImpl = input.fetch ?? defaultFetch2;
+  const url = new URL(`${input.baseUrl ?? META_GRAPH_BASE_URL}${input.path}`);
+  url.searchParams.set("access_token", input.accessToken);
+  url.searchParams.set("fields", input.fields.join(","));
+  url.searchParams.set("limit", String(input.limit ?? DEFAULT_META_PAGE_SIZE));
+  if (typeof input.after === "string" && input.after.length > 0) {
+    url.searchParams.set("after", input.after);
+  }
+  const response = await fetchImpl(url);
+  if (!response.ok) {
+    throw await toErrorMessage3(response, input.label);
+  }
+  return await response.json();
+};
+var getCheckpointAfter = (checkpoint) => {
+  const after = checkpoint?.after;
+  return typeof after === "string" && after.trim().length > 0 ? after.trim() : undefined;
+};
+var getCredentialMetadataString = (metadata, key) => {
+  const value = metadata?.[key];
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+};
+var toFacebookPostItem = (credential, post) => {
+  if (typeof post.id !== "string" || post.id.trim().length === 0) {
+    return null;
+  }
+  const pageName = getCredentialMetadataString(credential.metadata, "label") ?? getCredentialMetadataString(credential.metadata, "pageName") ?? getCredentialMetadataString(credential.metadata, "parentPageName");
+  const message = post.message?.trim() ?? "";
+  return {
+    createdAt: post.created_time,
+    html: undefined,
+    id: post.id,
+    kind: "facebook_post",
+    metadata: {
+      facebookPageId: credential.externalAccountId,
+      facebookPageName: pageName,
+      facebookPictureUrl: post.full_picture,
+      facebookStatusType: post.status_type,
+      provider: "facebook",
+      providerAuthorId: post.from?.id,
+      providerAuthorName: post.from?.name
+    },
+    text: message.length > 0 ? message : post.permalink_url,
+    title: message.length > 0 ? message.slice(0, 120) : `Facebook post ${post.id}`,
+    updatedAt: post.updated_time,
+    url: post.permalink_url
+  };
+};
+var toInstagramMediaItem = (credential, media) => {
+  if (typeof media.id !== "string" || media.id.trim().length === 0) {
+    return null;
+  }
+  const caption = media.caption?.trim() ?? "";
+  const username = getCredentialMetadataString(credential.metadata, "instagramUsername");
+  return {
+    createdAt: media.timestamp,
+    id: media.id,
+    kind: "instagram_media",
+    metadata: {
+      instagramMediaType: media.media_type,
+      instagramThumbnailUrl: media.thumbnail_url,
+      instagramUsername: media.username ?? username,
+      provider: "instagram",
+      providerAccountId: credential.externalAccountId,
+      providerMediaUrl: media.media_url
+    },
+    text: caption.length > 0 ? caption : media.permalink,
+    title: caption.length > 0 ? caption.slice(0, 120) : `Instagram media ${media.id}`,
+    url: media.permalink
+  };
+};
+var createRAGFacebookPageConnector = (input) => ({
+  provider: "facebook",
+  requiredScopes: () => FACEBOOK_PAGE_READ_SCOPES,
+  sync: async ({ checkpoint, credential, resolver }) => {
+    const lease = await resolver.getAccessToken(credential, {
+      requiredScopes: FACEBOOK_PAGE_READ_SCOPES
+    });
+    const pageAccessToken = getCredentialMetadataString(credential.metadata, "pageAccessToken") ?? lease.accessToken;
+    const response = await fetchGraphList({
+      accessToken: pageAccessToken,
+      after: getCheckpointAfter(checkpoint),
+      baseUrl: input?.baseUrl,
+      fetch: input?.fetch,
+      fields: [
+        "id",
+        "message",
+        "created_time",
+        "updated_time",
+        "permalink_url",
+        "full_picture",
+        "status_type",
+        "from{id,name}"
+      ],
+      label: `Facebook Page sync failed for ${credential.externalAccountId}`,
+      limit: input?.limit,
+      path: `/${encodeURIComponent(credential.externalAccountId)}/posts`
+    });
+    return {
+      items: (response.data ?? []).map((post) => toFacebookPostItem(credential, post)).filter(Boolean),
+      nextCheckpoint: response.paging?.cursors?.after ? { after: response.paging.cursors.after } : undefined,
+      diagnostics: {
+        listedCount: response.data?.length ?? 0,
+        nextAfter: response.paging?.cursors?.after
+      }
+    };
+  }
+});
+var createRAGInstagramBusinessConnector = (input) => ({
+  provider: "instagram",
+  requiredScopes: () => INSTAGRAM_BUSINESS_READ_SCOPES,
+  sync: async ({ checkpoint, credential, resolver }) => {
+    const lease = await resolver.getAccessToken(credential, {
+      requiredScopes: INSTAGRAM_BUSINESS_READ_SCOPES
+    });
+    const accessToken = getCredentialMetadataString(credential.metadata, "parentPageAccessToken") ?? lease.accessToken;
+    const response = await fetchGraphList({
+      accessToken,
+      after: getCheckpointAfter(checkpoint),
+      baseUrl: input?.baseUrl,
+      fetch: input?.fetch,
+      fields: [
+        "id",
+        "caption",
+        "media_type",
+        "media_url",
+        "permalink",
+        "thumbnail_url",
+        "timestamp",
+        "username"
+      ],
+      label: `Instagram business sync failed for ${credential.externalAccountId}`,
+      limit: input?.limit,
+      path: `/${encodeURIComponent(credential.externalAccountId)}/media`
+    });
+    return {
+      items: (response.data ?? []).map((media) => toInstagramMediaItem(credential, media)).filter(Boolean),
+      nextCheckpoint: response.paging?.cursors?.after ? { after: response.paging.cursors.after } : undefined,
+      diagnostics: {
+        listedCount: response.data?.length ?? 0,
+        nextAfter: response.paging?.cursors?.after
+      }
+    };
+  }
+});
 // src/ai/rag/sync.ts
 var {S3Client } = globalThis.Bun;
 import { createHash } from "crypto";
@@ -31834,6 +32003,10 @@ var wait = async (delayMs) => {
 };
 var getSyncMetadataString = (metadata, key) => typeof metadata?.[key] === "string" ? metadata[key] : undefined;
 var getSyncMetadataBoolean = (metadata, key) => metadata?.[key] === true;
+var getSyncMetadataRecord = (metadata, key) => {
+  const value = metadata?.[key];
+  return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+};
 var DEFAULT_DIRECTORY_EXTENSIONS2 = [
   ".txt",
   ".md",
@@ -32172,12 +32345,30 @@ var toManagedSyncDocument = (sourceId, document, syncKey) => ({
   }
 });
 var encodeAttachmentContent = (attachment) => typeof attachment.content === "string" ? {
-  content: attachment.content,
+  content: sanitizeSyncString(attachment.content),
   encoding: attachment.encoding ?? "utf8"
 } : {
   content: Buffer.from(attachment.content).toString("base64"),
   encoding: "base64"
 };
+var sanitizeSyncString = (value) => value.replace(/\u0000/g, "");
+var sanitizeOptionalSyncString = (value) => typeof value === "string" ? sanitizeSyncString(value) : value;
+var sanitizeSyncStringList = (value) => value?.map((entry) => sanitizeSyncString(entry)).filter((entry) => typeof entry === "string");
+var sanitizeSyncMetadataValue = (value) => {
+  if (typeof value === "string") {
+    return sanitizeSyncString(value);
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) => sanitizeSyncMetadataValue(entry));
+  }
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([key, entry]) => [
+      key,
+      sanitizeSyncMetadataValue(entry)
+    ]));
+  }
+  return value;
+};
 var toTimestamp = (value) => {
   if (typeof value === "number" && Number.isFinite(value)) {
     return value;
@@ -33893,21 +34084,21 @@ var createRAGEmailSyncSource = (options) => ({
       id: `email-${message.id}`,
       metadata: {
         ...options.baseMetadata ?? {},
-        ...message.metadata ?? {},
+        ...sanitizeSyncMetadataValue(message.metadata ?? {}) ?? {},
         emailKind: "message",
-        from: message.from,
+        from: sanitizeOptionalSyncString(message.from),
         hasAttachments: (message.attachments?.length ?? 0) > 0,
         messageId: message.id,
         receivedAt: toTimestamp(message.receivedAt),
         sentAt: toTimestamp(message.sentAt),
-        threadId: message.threadId,
-        threadTopic: message.subject,
-        to: message.to,
-        cc: message.cc
+        threadId: sanitizeOptionalSyncString(message.threadId),
+        threadTopic: sanitizeOptionalSyncString(message.subject),
+        to: sanitizeSyncStringList(message.to),
+        cc: sanitizeSyncStringList(message.cc)
       },
-      source: `email/${message.threadId ?? message.id}`,
-      text: message.bodyText,
-      title: message.subject ?? message.id
+      source: `email/${sanitizeOptionalSyncString(message.threadId) ?? message.id}`,
+      text: sanitizeOptionalSyncString(message.bodyText) ?? "",
+      title: sanitizeOptionalSyncString(message.subject) ?? message.id
     }));
     const attachmentUploads = listed.messages.flatMap((message) => (message.attachments ?? []).map((attachment, index) => ({
       ...encodeAttachmentContent(attachment),
@@ -33916,18 +34107,18 @@ var createRAGEmailSyncSource = (options) => ({
       format: attachment.format,
       metadata: {
         ...options.baseMetadata ?? {},
-        ...attachment.metadata ?? {},
+        ...sanitizeSyncMetadataValue(attachment.metadata ?? {}) ?? {},
         attachmentId: attachment.id ?? `${message.id}-attachment-${index + 1}`,
         emailKind: "attachment",
-        from: message.from,
+        from: sanitizeOptionalSyncString(message.from),
         messageId: message.id,
         sentAt: toTimestamp(message.sentAt),
-        threadId: message.threadId,
-        threadTopic: message.subject
+        threadId: sanitizeOptionalSyncString(message.threadId),
+        threadTopic: sanitizeOptionalSyncString(message.subject)
       },
-      name: attachment.name,
-      source: attachment.source ?? `email/${message.threadId ?? message.id}/attachments/${attachment.name}`,
-      title: attachment.title ?? `${message.subject ?? message.id} \xB7 ${attachment.name}`
+      name: sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`,
+      source: sanitizeOptionalSyncString(attachment.source) ?? `email/${sanitizeOptionalSyncString(message.threadId) ?? message.id}/attachments/${sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`}`,
+      title: sanitizeOptionalSyncString(attachment.title) ?? `${sanitizeOptionalSyncString(message.subject) ?? message.id} \xB7 ${sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`}`
     })));
     const extractionFailures = [];
     const loadedAttachments = attachmentUploads.length > 0 ? (await Promise.all(attachmentUploads.map(async (upload) => {
@@ -33987,6 +34178,81 @@ var createRAGEmailSyncSource = (options) => ({
     };
   }
 });
+var createRAGLinkedConnectorSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "custom",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.externalAccountId ?? options.bindingId ?? options.label,
+  sync: async ({ collection, deleteDocument, listDocuments, sourceRecord }) => {
+    const requiredScopes = options.requiredScopes ?? options.runtime.requiredScopes({ mode: "read" });
+    const credential = await options.resolver.resolveCredential({
+      bindingId: options.bindingId,
+      connectorProvider: options.runtime.provider,
+      externalAccountId: options.externalAccountId,
+      ownerRef: options.ownerRef,
+      purpose: options.purpose ?? "background_sync",
+      requiredScopes
+    });
+    if (!credential) {
+      throw new Error(`No linked ${options.runtime.provider} credential could be resolved`);
+    }
+    const checkpoint = getSyncMetadataRecord(sourceRecord?.metadata, "connectorCheckpoint");
+    const result = await options.runtime.sync({
+      checkpoint,
+      credential,
+      resolver: options.resolver
+    });
+    const managedDocuments = result.items.map((item, index) => toManagedSyncDocument(options.id, {
+      chunking: options.defaultChunking,
+      format: item.html ? "html" : "text",
+      id: `${options.runtime.provider}-${item.id}`,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...sanitizeSyncMetadataValue(item.metadata ?? {}) ?? {},
+        connectorBindingId: credential.bindingId,
+        connectorExternalAccountId: credential.externalAccountId,
+        connectorKind: item.kind,
+        connectorProvider: options.runtime.provider,
+        createdAt: toTimestamp(item.createdAt),
+        itemId: item.id,
+        threadId: sanitizeOptionalSyncString(item.threadId),
+        updatedAt: toTimestamp(item.updatedAt),
+        url: sanitizeOptionalSyncString(item.url)
+      },
+      source: sanitizeOptionalSyncString(item.url) ?? `connector/${options.runtime.provider}/${sanitizeOptionalSyncString(item.threadId) ?? item.id}`,
+      text: sanitizeOptionalSyncString(item.text) ?? sanitizeOptionalSyncString(item.html) ?? "",
+      title: sanitizeOptionalSyncString(item.title) ?? item.id
+    }, `${item.kind}:${item.id}:${index + 1}`));
+    const reconciled = await reconcileManagedDocuments({
+      chunkingRegistry: options.chunkingRegistry,
+      collection,
+      defaultChunking: options.defaultChunking,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id,
+      allowDeletions: result.nextCheckpoint === undefined
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        connectorCheckpoint: result.nextCheckpoint,
+        connectorDiagnostics: result.diagnostics,
+        connectorItemCount: result.items.length,
+        connectorProvider: options.runtime.provider,
+        deletedCount: reconciled.deletedCount,
+        resumePending: result.nextCheckpoint !== undefined,
+        updatedCount: reconciled.updatedCount
+      },
+      reconciliation: reconciled.reconciliation
+    };
+  }
+});
 var createRAGLinkedGmailEmailSyncSource = (options) => createRAGEmailSyncSource({
   ...options,
   client: createRAGLinkedGmailEmailSyncClient(options)
@@ -36922,6 +37188,8 @@ export {
   createRAGMediaFileExtractor,
   createRAGLinkedGmailEmailSyncSource,
   createRAGLinkedGmailEmailSyncClient,
+  createRAGLinkedConnectorSyncSource,
+  createRAGInstagramBusinessConnector,
   createRAGImageOCRExtractor,
   createRAGIMAPEmailSyncClient,
   createRAGHTMXWorkflowRenderConfig,
@@ -36953,6 +37221,7 @@ export {
   createRAGFileAnswerGroundingEvaluationHistoryStore,
   createRAGFileAnswerGroundingCaseDifficultyHistoryStore,
   createRAGFeedSyncSource,
+  createRAGFacebookPageConnector,
   createRAGEvaluationSuiteSnapshot,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
@@ -37059,5 +37328,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=46D96811C8BB335F64756E2164756E21
+//# debugId=B8D6F85D648EA8ED64756E2164756E21
 //# sourceMappingURL=index.js.map