@absolutejs/rag 0.0.5 → 0.0.7

package/dist/ai/rag/index.js

@@ -31820,6 +31820,308 @@ ${tag} OK`) || chunk.startsWith(`${tag} OK`));
     return { messages };
   }
 });
+// src/ai/rag/contactProviders.ts
+var defaultFetch2 = Object.assign((...args) => fetch(...args), { preconnect: fetch.preconnect });
+var GOOGLE_PEOPLE_BASE_URL = "https://people.googleapis.com/v1";
+var DEFAULT_GOOGLE_CONTACTS_PAGE_SIZE = 200;
+var GOOGLE_CONTACTS_READ_SCOPES = [
+  "https://www.googleapis.com/auth/contacts.readonly"
+];
+var toErrorMessage3 = async (response, label) => {
+  let detailMessage;
+  try {
+    const body = await response.clone().json();
+    detailMessage = body.error?.message;
+  } catch {
+    const text = await response.clone().text();
+    detailMessage = text.trim().length > 0 ? text.trim() : undefined;
+  }
+  return new Error(`${label}: ${response.status} ${response.statusText}${detailMessage ? ` (${detailMessage})` : ""}`);
+};
+var normalizeString = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+var toUniqueStringArray = (values) => [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))];
+var toContactTitle = (person, fallbackId) => {
+  const primaryName = person.names?.find((name) => normalizeString(name.displayName) !== undefined);
+  const displayName = normalizeString(primaryName?.displayName);
+  if (displayName) {
+    return displayName;
+  }
+  const primaryEmail = normalizeString(person.emailAddresses?.[0]?.value);
+  if (primaryEmail) {
+    return primaryEmail;
+  }
+  return `Google contact ${fallbackId}`;
+};
+var toContactText = (person) => {
+  const name = normalizeString(person.names?.[0]?.displayName);
+  const givenName = normalizeString(person.names?.[0]?.givenName);
+  const familyName = normalizeString(person.names?.[0]?.familyName);
+  const emails = toUniqueStringArray((person.emailAddresses ?? []).map((entry) => normalizeString(entry.value)));
+  const phones = toUniqueStringArray((person.phoneNumbers ?? []).map((entry) => normalizeString(entry.value) ?? normalizeString(entry.canonicalForm)));
+  const organizations = toUniqueStringArray((person.organizations ?? []).flatMap((entry) => [
+    normalizeString(entry.name),
+    normalizeString(entry.title)
+  ]));
+  const biography = normalizeString(person.biographies?.[0]?.value);
+  const urls = toUniqueStringArray((person.urls ?? []).map((entry) => normalizeString(entry.value)));
+  return [
+    name,
+    givenName && familyName ? `${givenName} ${familyName}` : givenName ?? familyName,
+    emails.length > 0 ? `Emails: ${emails.join(", ")}` : undefined,
+    phones.length > 0 ? `Phones: ${phones.join(", ")}` : undefined,
+    organizations.length > 0 ? `Organizations: ${organizations.join(", ")}` : undefined,
+    biography,
+    urls.length > 0 ? `Links: ${urls.join(", ")}` : undefined
+  ].filter((value) => typeof value === "string" && value.length > 0).join(`
+`);
+};
+var toContactItem = (person) => {
+  const resourceName = normalizeString(person.resourceName);
+  if (!resourceName) {
+    return null;
+  }
+  const contactId = resourceName.replace(/^people\//, "");
+  const title = toContactTitle(person, contactId);
+  const emails = toUniqueStringArray((person.emailAddresses ?? []).map((entry) => normalizeString(entry.value)));
+  const phones = toUniqueStringArray((person.phoneNumbers ?? []).map((entry) => normalizeString(entry.value) ?? normalizeString(entry.canonicalForm)));
+  const organizations = (person.organizations ?? []).map((entry) => ({
+    name: normalizeString(entry.name),
+    title: normalizeString(entry.title)
+  })).filter((entry) => entry.name || entry.title);
+  const urls = toUniqueStringArray((person.urls ?? []).map((entry) => normalizeString(entry.value)));
+  const photoUrl = normalizeString(person.photos?.find((photo) => normalizeString(photo.url) !== undefined)?.url);
+  const text = toContactText(person);
+  return {
+    id: resourceName,
+    kind: "google_contact",
+    metadata: {
+      emails,
+      etag: normalizeString(person.etag),
+      organizations,
+      phones,
+      photoUrl,
+      provider: "google_contacts",
+      resourceName,
+      urls
+    },
+    text: text.length > 0 ? text : title,
+    title,
+    url: urls[0]
+  };
+};
+var createRAGGoogleContactsConnector = (input) => ({
+  provider: "google_contacts",
+  requiredScopes: () => GOOGLE_CONTACTS_READ_SCOPES,
+  sync: async ({ checkpoint, credential, resolver }) => {
+    const lease = await resolver.getAccessToken(credential, {
+      requiredScopes: GOOGLE_CONTACTS_READ_SCOPES
+    });
+    const fetchImpl = input?.fetch ?? defaultFetch2;
+    const url = new URL(`${input?.baseUrl ?? GOOGLE_PEOPLE_BASE_URL}/people/me/connections`);
+    url.searchParams.set("personFields", [
+      "names",
+      "emailAddresses",
+      "phoneNumbers",
+      "organizations",
+      "biographies",
+      "urls",
+      "photos"
+    ].join(","));
+    url.searchParams.set("pageSize", String(input?.pageSize ?? DEFAULT_GOOGLE_CONTACTS_PAGE_SIZE));
+    const pageToken = normalizeString(checkpoint?.pageToken);
+    if (pageToken) {
+      url.searchParams.set("pageToken", pageToken);
+    }
+    const response = await fetchImpl(url, {
+      headers: {
+        Authorization: `Bearer ${lease.accessToken}`
+      }
+    });
+    if (!response.ok) {
+      throw await toErrorMessage3(response, `Google Contacts sync failed for ${credential.externalAccountId}`);
+    }
+    const json = await response.json();
+    return {
+      items: (json.connections ?? []).map((person) => toContactItem(person)).filter(Boolean),
+      nextCheckpoint: normalizeString(json.nextPageToken) ? { pageToken: normalizeString(json.nextPageToken) } : undefined,
+      diagnostics: {
+        listedCount: json.connections?.length ?? 0,
+        nextPageToken: normalizeString(json.nextPageToken),
+        totalItems: json.totalItems,
+        totalPeople: json.totalPeople
+      }
+    };
+  }
+});
+// src/ai/rag/socialProviders.ts
+var defaultFetch3 = Object.assign((...args) => fetch(...args), { preconnect: fetch.preconnect });
+var META_GRAPH_BASE_URL = "https://graph.facebook.com/v22.0";
+var DEFAULT_META_PAGE_SIZE = 25;
+var FACEBOOK_PAGE_READ_SCOPES = [
+  "pages_show_list",
+  "pages_read_engagement"
+];
+var INSTAGRAM_BUSINESS_READ_SCOPES = [
+  "pages_show_list",
+  "instagram_basic"
+];
+var toErrorMessage4 = async (response, label) => {
+  let detailMessage;
+  try {
+    const body = await response.clone().json();
+    detailMessage = body.error?.message;
+  } catch {
+    const text = await response.clone().text();
+    detailMessage = text.trim().length > 0 ? text.trim() : undefined;
+  }
+  return new Error(`${label}: ${response.status} ${response.statusText}${detailMessage ? ` (${detailMessage})` : ""}`);
+};
+var fetchGraphList = async (input) => {
+  const fetchImpl = input.fetch ?? defaultFetch3;
+  const url = new URL(`${input.baseUrl ?? META_GRAPH_BASE_URL}${input.path}`);
+  url.searchParams.set("access_token", input.accessToken);
+  url.searchParams.set("fields", input.fields.join(","));
+  url.searchParams.set("limit", String(input.limit ?? DEFAULT_META_PAGE_SIZE));
+  if (typeof input.after === "string" && input.after.length > 0) {
+    url.searchParams.set("after", input.after);
+  }
+  const response = await fetchImpl(url);
+  if (!response.ok) {
+    throw await toErrorMessage4(response, input.label);
+  }
+  return await response.json();
+};
+var getCheckpointAfter = (checkpoint) => {
+  const after = checkpoint?.after;
+  return typeof after === "string" && after.trim().length > 0 ? after.trim() : undefined;
+};
+var getCredentialMetadataString = (metadata, key) => {
+  const value = metadata?.[key];
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+};
+var toFacebookPostItem = (credential, post) => {
+  if (typeof post.id !== "string" || post.id.trim().length === 0) {
+    return null;
+  }
+  const pageName = getCredentialMetadataString(credential.metadata, "label") ?? getCredentialMetadataString(credential.metadata, "pageName") ?? getCredentialMetadataString(credential.metadata, "parentPageName");
+  const message = post.message?.trim() ?? "";
+  return {
+    createdAt: post.created_time,
+    html: undefined,
+    id: post.id,
+    kind: "facebook_post",
+    metadata: {
+      facebookPageId: credential.externalAccountId,
+      facebookPageName: pageName,
+      facebookPictureUrl: post.full_picture,
+      facebookStatusType: post.status_type,
+      provider: "facebook",
+      providerAuthorId: post.from?.id,
+      providerAuthorName: post.from?.name
+    },
+    text: message.length > 0 ? message : post.permalink_url,
+    title: message.length > 0 ? message.slice(0, 120) : `Facebook post ${post.id}`,
+    updatedAt: post.updated_time,
+    url: post.permalink_url
+  };
+};
+var toInstagramMediaItem = (credential, media) => {
+  if (typeof media.id !== "string" || media.id.trim().length === 0) {
+    return null;
+  }
+  const caption = media.caption?.trim() ?? "";
+  const username = getCredentialMetadataString(credential.metadata, "instagramUsername");
+  return {
+    createdAt: media.timestamp,
+    id: media.id,
+    kind: "instagram_media",
+    metadata: {
+      instagramMediaType: media.media_type,
+      instagramThumbnailUrl: media.thumbnail_url,
+      instagramUsername: media.username ?? username,
+      provider: "instagram",
+      providerAccountId: credential.externalAccountId,
+      providerMediaUrl: media.media_url
+    },
+    text: caption.length > 0 ? caption : media.permalink,
+    title: caption.length > 0 ? caption.slice(0, 120) : `Instagram media ${media.id}`,
+    url: media.permalink
+  };
+};
+var createRAGFacebookPageConnector = (input) => ({
+  provider: "facebook",
+  requiredScopes: () => FACEBOOK_PAGE_READ_SCOPES,
+  sync: async ({ checkpoint, credential, resolver }) => {
+    const lease = await resolver.getAccessToken(credential, {
+      requiredScopes: FACEBOOK_PAGE_READ_SCOPES
+    });
+    const pageAccessToken = getCredentialMetadataString(credential.metadata, "pageAccessToken") ?? lease.accessToken;
+    const response = await fetchGraphList({
+      accessToken: pageAccessToken,
+      after: getCheckpointAfter(checkpoint),
+      baseUrl: input?.baseUrl,
+      fetch: input?.fetch,
+      fields: [
+        "id",
+        "message",
+        "created_time",
+        "updated_time",
+        "permalink_url",
+        "full_picture",
+        "status_type",
+        "from{id,name}"
+      ],
+      label: `Facebook Page sync failed for ${credential.externalAccountId}`,
+      limit: input?.limit,
+      path: `/${encodeURIComponent(credential.externalAccountId)}/posts`
+    });
+    return {
+      items: (response.data ?? []).map((post) => toFacebookPostItem(credential, post)).filter(Boolean),
+      nextCheckpoint: response.paging?.cursors?.after ? { after: response.paging.cursors.after } : undefined,
+      diagnostics: {
+        listedCount: response.data?.length ?? 0,
+        nextAfter: response.paging?.cursors?.after
+      }
+    };
+  }
+});
+var createRAGInstagramBusinessConnector = (input) => ({
+  provider: "instagram",
+  requiredScopes: () => INSTAGRAM_BUSINESS_READ_SCOPES,
+  sync: async ({ checkpoint, credential, resolver }) => {
+    const lease = await resolver.getAccessToken(credential, {
+      requiredScopes: INSTAGRAM_BUSINESS_READ_SCOPES
+    });
+    const accessToken = getCredentialMetadataString(credential.metadata, "parentPageAccessToken") ?? lease.accessToken;
+    const response = await fetchGraphList({
+      accessToken,
+      after: getCheckpointAfter(checkpoint),
+      baseUrl: input?.baseUrl,
+      fetch: input?.fetch,
+      fields: [
+        "id",
+        "caption",
+        "media_type",
+        "media_url",
+        "permalink",
+        "thumbnail_url",
+        "timestamp",
+        "username"
+      ],
+      label: `Instagram business sync failed for ${credential.externalAccountId}`,
+      limit: input?.limit,
+      path: `/${encodeURIComponent(credential.externalAccountId)}/media`
+    });
+    return {
+      items: (response.data ?? []).map((media) => toInstagramMediaItem(credential, media)).filter(Boolean),
+      nextCheckpoint: response.paging?.cursors?.after ? { after: response.paging.cursors.after } : undefined,
+      diagnostics: {
+        listedCount: response.data?.length ?? 0,
+        nextAfter: response.paging?.cursors?.after
+      }
+    };
+  }
+});
 // src/ai/rag/sync.ts
 var {S3Client } = globalThis.Bun;
 import { createHash } from "crypto";
@@ -31834,6 +32136,10 @@ var wait = async (delayMs) => {
 };
 var getSyncMetadataString = (metadata, key) => typeof metadata?.[key] === "string" ? metadata[key] : undefined;
 var getSyncMetadataBoolean = (metadata, key) => metadata?.[key] === true;
+var getSyncMetadataRecord = (metadata, key) => {
+  const value = metadata?.[key];
+  return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+};
 var DEFAULT_DIRECTORY_EXTENSIONS2 = [
   ".txt",
   ".md",
@@ -32172,12 +32478,30 @@ var toManagedSyncDocument = (sourceId, document, syncKey) => ({
   }
 });
 var encodeAttachmentContent = (attachment) => typeof attachment.content === "string" ? {
-  content: attachment.content,
+  content: sanitizeSyncString(attachment.content),
   encoding: attachment.encoding ?? "utf8"
 } : {
   content: Buffer.from(attachment.content).toString("base64"),
   encoding: "base64"
 };
+var sanitizeSyncString = (value) => value.replace(/\u0000/g, "");
+var sanitizeOptionalSyncString = (value) => typeof value === "string" ? sanitizeSyncString(value) : value;
+var sanitizeSyncStringList = (value) => value?.map((entry) => sanitizeSyncString(entry)).filter((entry) => typeof entry === "string");
+var sanitizeSyncMetadataValue = (value) => {
+  if (typeof value === "string") {
+    return sanitizeSyncString(value);
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) => sanitizeSyncMetadataValue(entry));
+  }
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([key, entry]) => [
+      key,
+      sanitizeSyncMetadataValue(entry)
+    ]));
+  }
+  return value;
+};
 var toTimestamp = (value) => {
   if (typeof value === "number" && Number.isFinite(value)) {
     return value;
@@ -33893,21 +34217,21 @@ var createRAGEmailSyncSource = (options) => ({
       id: `email-${message.id}`,
       metadata: {
         ...options.baseMetadata ?? {},
-        ...message.metadata ?? {},
+        ...sanitizeSyncMetadataValue(message.metadata ?? {}) ?? {},
         emailKind: "message",
-        from: message.from,
+        from: sanitizeOptionalSyncString(message.from),
         hasAttachments: (message.attachments?.length ?? 0) > 0,
         messageId: message.id,
         receivedAt: toTimestamp(message.receivedAt),
         sentAt: toTimestamp(message.sentAt),
-        threadId: message.threadId,
-        threadTopic: message.subject,
-        to: message.to,
-        cc: message.cc
+        threadId: sanitizeOptionalSyncString(message.threadId),
+        threadTopic: sanitizeOptionalSyncString(message.subject),
+        to: sanitizeSyncStringList(message.to),
+        cc: sanitizeSyncStringList(message.cc)
       },
-      source: `email/${message.threadId ?? message.id}`,
-      text: message.bodyText,
-      title: message.subject ?? message.id
+      source: `email/${sanitizeOptionalSyncString(message.threadId) ?? message.id}`,
+      text: sanitizeOptionalSyncString(message.bodyText) ?? "",
+      title: sanitizeOptionalSyncString(message.subject) ?? message.id
     }));
     const attachmentUploads = listed.messages.flatMap((message) => (message.attachments ?? []).map((attachment, index) => ({
       ...encodeAttachmentContent(attachment),
@@ -33916,18 +34240,18 @@ var createRAGEmailSyncSource = (options) => ({
       format: attachment.format,
       metadata: {
         ...options.baseMetadata ?? {},
-        ...attachment.metadata ?? {},
+        ...sanitizeSyncMetadataValue(attachment.metadata ?? {}) ?? {},
         attachmentId: attachment.id ?? `${message.id}-attachment-${index + 1}`,
         emailKind: "attachment",
-        from: message.from,
+        from: sanitizeOptionalSyncString(message.from),
         messageId: message.id,
         sentAt: toTimestamp(message.sentAt),
-        threadId: message.threadId,
-        threadTopic: message.subject
+        threadId: sanitizeOptionalSyncString(message.threadId),
+        threadTopic: sanitizeOptionalSyncString(message.subject)
       },
-      name: attachment.name,
-      source: attachment.source ?? `email/${message.threadId ?? message.id}/attachments/${attachment.name}`,
-      title: attachment.title ?? `${message.subject ?? message.id} \xB7 ${attachment.name}`
+      name: sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`,
+      source: sanitizeOptionalSyncString(attachment.source) ?? `email/${sanitizeOptionalSyncString(message.threadId) ?? message.id}/attachments/${sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`}`,
+      title: sanitizeOptionalSyncString(attachment.title) ?? `${sanitizeOptionalSyncString(message.subject) ?? message.id} \xB7 ${sanitizeOptionalSyncString(attachment.name) ?? `attachment-${index + 1}`}`
     })));
     const extractionFailures = [];
     const loadedAttachments = attachmentUploads.length > 0 ? (await Promise.all(attachmentUploads.map(async (upload) => {
@@ -33987,6 +34311,81 @@ var createRAGEmailSyncSource = (options) => ({
     };
   }
 });
+var createRAGLinkedConnectorSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "custom",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.externalAccountId ?? options.bindingId ?? options.label,
+  sync: async ({ collection, deleteDocument, listDocuments, sourceRecord }) => {
+    const requiredScopes = options.requiredScopes ?? options.runtime.requiredScopes({ mode: "read" });
+    const credential = await options.resolver.resolveCredential({
+      bindingId: options.bindingId,
+      connectorProvider: options.runtime.provider,
+      externalAccountId: options.externalAccountId,
+      ownerRef: options.ownerRef,
+      purpose: options.purpose ?? "background_sync",
+      requiredScopes
+    });
+    if (!credential) {
+      throw new Error(`No linked ${options.runtime.provider} credential could be resolved`);
+    }
+    const checkpoint = getSyncMetadataRecord(sourceRecord?.metadata, "connectorCheckpoint");
+    const result = await options.runtime.sync({
+      checkpoint,
+      credential,
+      resolver: options.resolver
+    });
+    const managedDocuments = result.items.map((item, index) => toManagedSyncDocument(options.id, {
+      chunking: options.defaultChunking,
+      format: item.html ? "html" : "text",
+      id: `${options.runtime.provider}-${item.id}`,
+      metadata: {
+        ...options.baseMetadata ?? {},
+        ...sanitizeSyncMetadataValue(item.metadata ?? {}) ?? {},
+        connectorBindingId: credential.bindingId,
+        connectorExternalAccountId: credential.externalAccountId,
+        connectorKind: item.kind,
+        connectorProvider: options.runtime.provider,
+        createdAt: toTimestamp(item.createdAt),
+        itemId: item.id,
+        threadId: sanitizeOptionalSyncString(item.threadId),
+        updatedAt: toTimestamp(item.updatedAt),
+        url: sanitizeOptionalSyncString(item.url)
+      },
+      source: sanitizeOptionalSyncString(item.url) ?? `connector/${options.runtime.provider}/${sanitizeOptionalSyncString(item.threadId) ?? item.id}`,
+      text: sanitizeOptionalSyncString(item.text) ?? sanitizeOptionalSyncString(item.html) ?? "",
+      title: sanitizeOptionalSyncString(item.title) ?? item.id
+    }, `${item.kind}:${item.id}:${index + 1}`));
+    const reconciled = await reconcileManagedDocuments({
+      chunkingRegistry: options.chunkingRegistry,
+      collection,
+      defaultChunking: options.defaultChunking,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id,
+      allowDeletions: result.nextCheckpoint === undefined
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        connectorCheckpoint: result.nextCheckpoint,
+        connectorDiagnostics: result.diagnostics,
+        connectorItemCount: result.items.length,
+        connectorProvider: options.runtime.provider,
+        deletedCount: reconciled.deletedCount,
+        resumePending: result.nextCheckpoint !== undefined,
+        updatedCount: reconciled.updatedCount
+      },
+      reconciliation: reconciled.reconciliation
+    };
+  }
+});
 var createRAGLinkedGmailEmailSyncSource = (options) => createRAGEmailSyncSource({
   ...options,
   client: createRAGLinkedGmailEmailSyncClient(options)
@@ -36922,11 +37321,14 @@ export {
   createRAGMediaFileExtractor,
   createRAGLinkedGmailEmailSyncSource,
   createRAGLinkedGmailEmailSyncClient,
+  createRAGLinkedConnectorSyncSource,
+  createRAGInstagramBusinessConnector,
   createRAGImageOCRExtractor,
   createRAGIMAPEmailSyncClient,
   createRAGHTMXWorkflowRenderConfig,
   createRAGHTMXConfig,
   createRAGGraphEmailSyncClient,
+  createRAGGoogleContactsConnector,
   createRAGGmailEmailSyncClient,
   createRAGGitHubSyncSource,
   createRAGFileSyncStateStore,
@@ -36953,6 +37355,7 @@ export {
   createRAGFileAnswerGroundingEvaluationHistoryStore,
   createRAGFileAnswerGroundingCaseDifficultyHistoryStore,
   createRAGFeedSyncSource,
+  createRAGFacebookPageConnector,
   createRAGEvaluationSuiteSnapshot,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
@@ -37059,5 +37462,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=46D96811C8BB335F64756E2164756E21
+//# debugId=8A6927B24E3DD10164756E2164756E21
 //# sourceMappingURL=index.js.map