openvolo 0.1.2

package/src/lib/platforms/sync-content.ts

@@ -0,0 +1,225 @@
+import { nanoid } from "nanoid";
+import { db } from "@/lib/db/client";
+import { contentItems, contentPosts, engagementMetrics } from "@/lib/db/schema";
+import { eq, and } from "drizzle-orm";
+import { getContentPostByPlatformId } from "@/lib/db/queries/content";
+import { getSyncCursor, updateSyncCursor } from "@/lib/db/queries/sync";
+import { updatePlatformAccount } from "@/lib/db/queries/platform-accounts";
+import { getAuthenticatedUser, getUserTweets, getUserMentions } from "@/lib/platforms/x/client";
+import { mapXTweetToContentItem, mapXTweetToContentPost, extractTweetMetrics } from "@/lib/platforms/x/mappers";
+import type { XTweet } from "@/lib/platforms/x/client";
+import type { SyncResult } from "@/lib/platforms/adapter";
+
+interface ContentSyncResult extends SyncResult {
+  /** Total items processed (including skipped). */
+  total: number;
+}
+
+/**
+ * Sync tweets from the authenticated user's timeline.
+ * Deduplicates by platformPostId on the content_posts table.
+ * Uses sync_cursors for pagination state.
+ */
+export async function syncTweetsFromX(
+  accountId: string,
+  opts?: { maxPages?: number }
+): Promise<ContentSyncResult> {
+  const result: ContentSyncResult = { added: 0, updated: 0, skipped: 0, errors: [], total: 0 };
+  const maxPages = opts?.maxPages ?? 5;
+
+  const cursor = getSyncCursor(accountId, "tweets");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  try {
+    const me = await getAuthenticatedUser(accountId);
+    let paginationToken: string | undefined = cursor.cursor ?? undefined;
+    let page = 0;
+
+    while (page < maxPages) {
+      const res = await getUserTweets(accountId, me.id, {
+        maxResults: 10,
+        paginationToken,
+      });
+
+      const tweets = Array.isArray(res.data) ? res.data : [];
+      if (tweets.length === 0) break;
+
+      for (const tweet of tweets) {
+        try {
+          const wasAdded = processTweet(tweet, accountId, "authored");
+          if (wasAdded) {
+            result.added++;
+          } else {
+            result.skipped++;
+          }
+          result.total++;
+        } catch (err) {
+          result.errors.push(
+            `Failed to process tweet ${tweet.id}: ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+
+      // Update cursor with pagination token
+      paginationToken = res.meta?.next_token ?? undefined;
+      updateSyncCursor(cursor.id, {
+        cursor: paginationToken ?? null,
+        totalItemsSynced: cursor.totalItemsSynced + result.added,
+        syncProgress: JSON.stringify({
+          current: result.total,
+          message: `Imported ${result.added} tweets`,
+        }),
+      });
+
+      if (!paginationToken) break;
+      page++;
+    }
+
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      totalItemsSynced: cursor.totalItemsSynced + result.added,
+    });
+  } catch (err) {
+    const errorMessage = err instanceof Error ? err.message : String(err);
+    result.errors.push(`Tweet sync failed: ${errorMessage}`);
+    updateSyncCursor(cursor.id, {
+      syncStatus: result.added > 0 ? "completed" : "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      totalItemsSynced: cursor.totalItemsSynced + result.added,
+      lastError: errorMessage,
+    });
+  }
+
+  // Update account last synced time
+  updatePlatformAccount(accountId, {
+    lastSyncedAt: Math.floor(Date.now() / 1000),
+  });
+
+  return result;
+}
+
+/**
+ * Sync mentions of the authenticated user.
+ * Mentions are treated as "received" content.
+ */
+export async function syncMentionsFromX(
+  accountId: string,
+  opts?: { maxPages?: number }
+): Promise<ContentSyncResult> {
+  const result: ContentSyncResult = { added: 0, updated: 0, skipped: 0, errors: [], total: 0 };
+  const maxPages = opts?.maxPages ?? 5;
+
+  const cursor = getSyncCursor(accountId, "mentions");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  try {
+    const me = await getAuthenticatedUser(accountId);
+    let paginationToken: string | undefined = cursor.cursor ?? undefined;
+    let page = 0;
+
+    while (page < maxPages) {
+      const res = await getUserMentions(accountId, me.id, {
+        maxResults: 10,
+        paginationToken,
+      });
+
+      const mentions = Array.isArray(res.data) ? res.data : [];
+      if (mentions.length === 0) break;
+
+      for (const mention of mentions) {
+        try {
+          const wasAdded = processTweet(mention, accountId, "received");
+          if (wasAdded) {
+            result.added++;
+          } else {
+            result.skipped++;
+          }
+          result.total++;
+        } catch (err) {
+          result.errors.push(
+            `Failed to process mention ${mention.id}: ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+
+      paginationToken = res.meta?.next_token ?? undefined;
+      updateSyncCursor(cursor.id, {
+        cursor: paginationToken ?? null,
+        totalItemsSynced: cursor.totalItemsSynced + result.added,
+        syncProgress: JSON.stringify({
+          current: result.total,
+          message: `Imported ${result.added} mentions`,
+        }),
+      });
+
+      if (!paginationToken) break;
+      page++;
+    }
+
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      totalItemsSynced: cursor.totalItemsSynced + result.added,
+    });
+  } catch (err) {
+    const errorMessage = err instanceof Error ? err.message : String(err);
+    result.errors.push(`Mention sync failed: ${errorMessage}`);
+    updateSyncCursor(cursor.id, {
+      syncStatus: result.added > 0 ? "completed" : "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      totalItemsSynced: cursor.totalItemsSynced + result.added,
+      lastError: errorMessage,
+    });
+  }
+
+  updatePlatformAccount(accountId, {
+    lastSyncedAt: Math.floor(Date.now() / 1000),
+  });
+
+  return result;
+}
+
+/**
+ * Process a single tweet/mention — create content_item + content_post + metrics.
+ * Returns true if added, false if skipped (already exists).
+ */
+function processTweet(
+  tweet: XTweet,
+  accountId: string,
+  origin: "authored" | "received"
+): boolean {
+  // Dedup check: does this tweet already exist?
+  const existingPost = getContentPostByPlatformId(tweet.id, accountId);
+  if (existingPost) return false;
+
+  // Create content item
+  const itemData = mapXTweetToContentItem(tweet, accountId, origin);
+  const itemId = nanoid();
+  db.insert(contentItems).values({ ...itemData, id: itemId }).run();
+
+  // Create content post (published instance)
+  const postData = mapXTweetToContentPost(tweet, accountId);
+  const postId = nanoid();
+  db.insert(contentPosts)
+    .values({ ...postData, id: postId, contentItemId: itemId })
+    .run();
+
+  // Store engagement metrics snapshot
+  const metrics = extractTweetMetrics(tweet);
+  db.insert(engagementMetrics)
+    .values({
+      id: nanoid(),
+      contentPostId: postId,
+      ...metrics,
+    })
+    .run();
+
+  return true;
+}

package/src/lib/platforms/sync-gmail-contacts.ts

@@ -0,0 +1,186 @@
+import { eq, and } from "drizzle-orm";
+import { db } from "@/lib/db/client";
+import { contacts, contactIdentities } from "@/lib/db/schema";
+import { createContact, updateContact, recalcEnrichment } from "@/lib/db/queries/contacts";
+import { createIdentity } from "@/lib/db/queries/identities";
+import { updatePlatformAccount } from "@/lib/db/queries/platform-accounts";
+import { getSyncCursor, updateSyncCursor } from "@/lib/db/queries/sync";
+import {
+  mapGooglePersonToContact,
+  mapGooglePersonToIdentity,
+} from "@/lib/platforms/gmail/mappers";
+import { getGoogleContacts } from "@/lib/platforms/gmail/client";
+import type { GooglePerson } from "@/lib/platforms/gmail/client";
+import type { SyncResult } from "@/lib/platforms/adapter";
+
+/**
+ * Sync contacts from Google People API into OpenVolo.
+ * Uses token-based pagination (pageToken, not offset-based like LinkedIn).
+ * 2-tier dedup: gmail identity match → email match → create new.
+ */
+export async function syncContactsFromGmail(
+  accountId: string,
+  opts?: { maxPages?: number }
+): Promise<SyncResult> {
+  const result: SyncResult = { added: 0, updated: 0, skipped: 0, errors: [] };
+  const maxPages = opts?.maxPages ?? 10; // Safety limit: 10 pages * 100 = 1000 contacts max
+
+  // Get or create sync cursor for tracking
+  const cursor = getSyncCursor(accountId, "google_contacts");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  try {
+    let pageToken: string | undefined = undefined;
+    let page = 0;
+
+    while (page < maxPages) {
+      const res = await getGoogleContacts(accountId, { pageToken, pageSize: 100 });
+      const people = res.connections ?? [];
+      if (people.length === 0) break;
+
+      // Process each person
+      for (const person of people) {
+        try {
+          processGooglePerson(person, result);
+        } catch (err) {
+          const name = person.names?.[0]?.displayName ?? person.resourceName;
+          result.errors.push(
+            `Failed to process ${name}: ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+
+      // Update cursor progress
+      updateSyncCursor(cursor.id, {
+        totalItemsSynced: (cursor.totalItemsSynced ?? 0) + people.length,
+        cursor: res.nextPageToken ?? null,
+      });
+
+      // Check if there are more pages
+      if (!res.nextPageToken) break;
+      pageToken = res.nextPageToken;
+      page++;
+    }
+
+    // Update sync timestamps — always set lastSyncCompletedAt (partial sync pattern)
+    const now = Math.floor(Date.now() / 1000);
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      lastSyncCompletedAt: now,
+    });
+
+    updatePlatformAccount(accountId, {
+      lastSyncedAt: now,
+    });
+  } catch (err) {
+    result.errors.push(
+      `Sync failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+
+    // Always set lastSyncCompletedAt even on error (partial sync pattern)
+    updateSyncCursor(cursor.id, {
+      syncStatus: "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      lastError: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  return result;
+}
+
+/** Process a single Google person — create or update contact + identity with cross-platform dedup. */
+function processGooglePerson(person: GooglePerson, result: SyncResult): void {
+  const resourceId = person.resourceName.replace(/^people\//, "");
+
+  // 1. Check for existing identity by (platform="gmail", platformUserId)
+  const existingIdentity = db
+    .select()
+    .from(contactIdentities)
+    .where(
+      and(
+        eq(contactIdentities.platform, "gmail"),
+        eq(contactIdentities.platformUserId, resourceId)
+      )
+    )
+    .get();
+
+  if (existingIdentity) {
+    // Update existing contact with latest Google data
+    const contactData = mapGooglePersonToContact(person);
+    updateContact(existingIdentity.contactId, {
+      firstName: contactData.firstName,
+      lastName: contactData.lastName,
+      email: contactData.email,
+      phone: contactData.phone,
+      company: contactData.company,
+      title: contactData.title,
+      location: contactData.location,
+      bio: contactData.bio,
+      photoUrl: contactData.photoUrl,
+      avatarUrl: contactData.avatarUrl,
+      website: contactData.website,
+    });
+
+    // Update identity with latest platform data
+    const identityData = mapGooglePersonToIdentity(person, existingIdentity.contactId);
+    db.update(contactIdentities)
+      .set({
+        platformHandle: identityData.platformHandle,
+        platformData: identityData.platformData,
+        lastSyncedAt: Math.floor(Date.now() / 1000),
+        updatedAt: Math.floor(Date.now() / 1000),
+      })
+      .where(eq(contactIdentities.id, existingIdentity.id))
+      .run();
+
+    recalcEnrichment(existingIdentity.contactId);
+    result.updated++;
+    return;
+  }
+
+  // 2. Cross-platform dedup: check contacts.email matching any of the person's emails
+  const contactData = mapGooglePersonToContact(person);
+  const personEmails = (person.emailAddresses ?? [])
+    .map((e) => e.value)
+    .filter((v): v is string => !!v);
+
+  for (const email of personEmails) {
+    const emailMatch = db
+      .select()
+      .from(contacts)
+      .where(eq(contacts.email, email))
+      .get();
+
+    if (emailMatch) {
+      // Found existing contact by email — add Gmail identity to it
+      updateContact(emailMatch.id, {
+        firstName: contactData.firstName ?? emailMatch.firstName,
+        lastName: contactData.lastName ?? emailMatch.lastName,
+        phone: contactData.phone ?? emailMatch.phone,
+        company: contactData.company ?? emailMatch.company,
+        title: contactData.title ?? emailMatch.title,
+        location: contactData.location ?? emailMatch.location,
+        bio: contactData.bio ?? emailMatch.bio,
+        photoUrl: contactData.photoUrl ?? emailMatch.photoUrl,
+        avatarUrl: contactData.avatarUrl ?? emailMatch.avatarUrl,
+        website: contactData.website ?? emailMatch.website,
+      });
+
+      const identityData = mapGooglePersonToIdentity(person, emailMatch.id);
+      createIdentity(identityData);
+      recalcEnrichment(emailMatch.id);
+      result.updated++;
+      return;
+    }
+  }
+
+  // 3. Create new contact
+  const contact = createContact(contactData);
+  const identityData = mapGooglePersonToIdentity(person, contact.id);
+  createIdentity(identityData);
+  recalcEnrichment(contact.id);
+  result.added++;
+}

package/src/lib/platforms/sync-gmail-metadata.ts

@@ -0,0 +1,158 @@
+import { eq, and, isNotNull } from "drizzle-orm";
+import { db } from "@/lib/db/client";
+import { contacts, contactIdentities } from "@/lib/db/schema";
+import { updateContact } from "@/lib/db/queries/contacts";
+import { updatePlatformAccount } from "@/lib/db/queries/platform-accounts";
+import { getSyncCursor, updateSyncCursor } from "@/lib/db/queries/sync";
+import {
+  getGmailMessagesByContact,
+  getGmailMessageMetadata,
+} from "@/lib/platforms/gmail/client";
+import type { SyncResult } from "@/lib/platforms/adapter";
+
+/**
+ * Sync Gmail metadata for contacts that have email addresses.
+ * Queries Gmail for per-contact message frequency and last interaction date.
+ * Stores in contacts.metadata JSON (merged, not replaced).
+ */
+export async function syncGmailMetadata(
+  accountId: string,
+  opts?: { maxContacts?: number }
+): Promise<SyncResult> {
+  const result: SyncResult = { added: 0, updated: 0, skipped: 0, errors: [] };
+  const maxContacts = opts?.maxContacts ?? 50; // Rate limit safety
+
+  // Get or create sync cursor for tracking
+  const cursor = getSyncCursor(accountId, "gmail_metadata");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  try {
+    // Find contacts with email addresses
+    const contactsWithEmail = db
+      .select()
+      .from(contacts)
+      .where(isNotNull(contacts.email))
+      .limit(maxContacts)
+      .all();
+
+    if (contactsWithEmail.length === 0) {
+      updateSyncCursor(cursor.id, {
+        syncStatus: "completed",
+        lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      });
+      return result;
+    }
+
+    // Calculate 30-day window for frequency counting
+    const now = Math.floor(Date.now() / 1000);
+    const thirtyDaysAgo = new Date(now * 1000);
+    thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+    const afterDate = `${thirtyDaysAgo.getFullYear()}/${String(thirtyDaysAgo.getMonth() + 1).padStart(2, "0")}/${String(thirtyDaysAgo.getDate()).padStart(2, "0")}`;
+
+    let processed = 0;
+
+    for (const contact of contactsWithEmail) {
+      if (!contact.email) continue;
+
+      try {
+        // Get most recent messages for last interaction date
+        const recentMessages = await getGmailMessagesByContact(
+          accountId,
+          contact.email,
+          { maxResults: 1 }
+        );
+
+        let lastMessageAt: number | null = null;
+
+        if (recentMessages.messages && recentMessages.messages.length > 0) {
+          // Get metadata of the most recent message
+          const latestMsg = await getGmailMessageMetadata(
+            accountId,
+            recentMessages.messages[0].id
+          );
+          if (latestMsg.internalDate) {
+            lastMessageAt = Math.floor(parseInt(latestMsg.internalDate) / 1000);
+          }
+        }
+
+        // Get sent messages in the last 30 days
+        const sentMessages = await getGmailMessagesByContact(
+          accountId,
+          contact.email,
+          {
+            maxResults: 100,
+            query: `to:${contact.email} after:${afterDate}`,
+          }
+        );
+        const sent30d = sentMessages.messages?.length ?? 0;
+
+        // Get received messages in the last 30 days
+        const receivedMessages = await getGmailMessagesByContact(
+          accountId,
+          contact.email,
+          {
+            maxResults: 100,
+            query: `from:${contact.email} after:${afterDate}`,
+          }
+        );
+        const received30d = receivedMessages.messages?.length ?? 0;
+
+        // Merge metadata (preserve existing, add messageFrequency)
+        const existingMetadata = contact.metadata ? JSON.parse(contact.metadata) : {};
+        const updatedMetadata = {
+          ...existingMetadata,
+          messageFrequency: {
+            sent30d,
+            received30d,
+            lastMessageAt,
+          },
+        };
+
+        // Update contact
+        const updates: Record<string, unknown> = {
+          metadata: JSON.stringify(updatedMetadata),
+        };
+
+        // Update lastInteractionAt if we found a more recent message
+        if (lastMessageAt && (!contact.lastInteractionAt || lastMessageAt > contact.lastInteractionAt)) {
+          updates.lastInteractionAt = lastMessageAt;
+        }
+
+        updateContact(contact.id, updates);
+        result.updated++;
+        processed++;
+      } catch (err) {
+        result.errors.push(
+          `Failed to sync metadata for ${contact.name} (${contact.email}): ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+
+    // Update sync timestamps
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      totalItemsSynced: (cursor.totalItemsSynced ?? 0) + processed,
+      lastSyncCompletedAt: now,
+    });
+
+    updatePlatformAccount(accountId, {
+      lastSyncedAt: now,
+    });
+  } catch (err) {
+    result.errors.push(
+      `Metadata sync failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+
+    // Always set lastSyncCompletedAt even on error (partial sync pattern)
+    updateSyncCursor(cursor.id, {
+      syncStatus: "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      lastError: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  return result;
+}