openvolo 0.1.2

package/src/lib/platforms/sync-linkedin-contacts.ts

@@ -0,0 +1,148 @@
+import { eq, and } from "drizzle-orm";
+import { db } from "@/lib/db/client";
+import { contacts, contactIdentities } from "@/lib/db/schema";
+import { createContact, updateContact, recalcEnrichment } from "@/lib/db/queries/contacts";
+import { createIdentity } from "@/lib/db/queries/identities";
+import { updatePlatformAccount } from "@/lib/db/queries/platform-accounts";
+import { getSyncCursor, updateSyncCursor } from "@/lib/db/queries/sync";
+import {
+  mapLinkedInConnectionToContact,
+  mapLinkedInConnectionToIdentity,
+} from "@/lib/platforms/linkedin/mappers";
+import { getConnections } from "@/lib/platforms/linkedin/client";
+import type { LinkedInConnection } from "@/lib/platforms/linkedin/client";
+import type { SyncResult } from "@/lib/platforms/adapter";
+
+/**
+ * Sync contacts from LinkedIn connections into OpenVolo.
+ * Uses offset-based pagination (not cursor-based like X).
+ * Cross-platform dedup: matches by LinkedIn ID, then by email.
+ */
+export async function syncContactsFromLinkedIn(
+  accountId: string,
+  opts?: { maxPages?: number }
+): Promise<SyncResult> {
+  const result: SyncResult = { added: 0, updated: 0, skipped: 0, errors: [] };
+  const maxPages = opts?.maxPages ?? 10; // Safety limit: 10 pages * 50 = 500 contacts max
+
+  // Get or create sync cursor for tracking
+  const cursor = getSyncCursor(accountId, "connections");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  try {
+    let start = 0;
+    let page = 0;
+
+    while (page < maxPages) {
+      const res = await getConnections(accountId, { start, count: 50 });
+      const connections = res.elements ?? [];
+      if (connections.length === 0) break;
+
+      // Process each connection
+      for (const connection of connections) {
+        try {
+          processLinkedInConnection(connection, result);
+        } catch (err) {
+          const name = `${connection.localizedFirstName ?? ""} ${connection.localizedLastName ?? ""}`.trim();
+          result.errors.push(
+            `Failed to process ${name || connection.id}: ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+
+      // Update cursor progress
+      updateSyncCursor(cursor.id, {
+        totalItemsSynced: (cursor.totalItemsSynced ?? 0) + connections.length,
+        cursor: String(start + connections.length),
+      });
+
+      // Check if there are more pages
+      const total = res.paging?.total ?? 0;
+      start += res.paging?.count ?? 50;
+      if (start >= total) break;
+      page++;
+    }
+
+    // Update sync timestamps — always set lastSyncCompletedAt (partial sync pattern)
+    const now = Math.floor(Date.now() / 1000);
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      lastSyncCompletedAt: now,
+    });
+
+    updatePlatformAccount(accountId, {
+      lastSyncedAt: now,
+    });
+  } catch (err) {
+    result.errors.push(
+      `Sync failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+
+    // Always set lastSyncCompletedAt even on error (partial sync pattern)
+    updateSyncCursor(cursor.id, {
+      syncStatus: "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      lastError: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  return result;
+}
+
+/** Process a single LinkedIn connection — create or update contact + identity with cross-platform dedup. */
+function processLinkedInConnection(connection: LinkedInConnection, result: SyncResult): void {
+  // 1. Check for existing identity by (platform="linkedin", platformUserId)
+  const existingIdentity = db
+    .select()
+    .from(contactIdentities)
+    .where(
+      and(
+        eq(contactIdentities.platform, "linkedin"),
+        eq(contactIdentities.platformUserId, connection.id)
+      )
+    )
+    .get();
+
+  if (existingIdentity) {
+    // Update existing contact with latest LinkedIn data
+    const contactData = mapLinkedInConnectionToContact(connection);
+    updateContact(existingIdentity.contactId, {
+      headline: contactData.headline,
+      photoUrl: contactData.photoUrl,
+      avatarUrl: contactData.avatarUrl,
+    });
+
+    // Update identity with latest platform data
+    db.update(contactIdentities)
+      .set({
+        platformHandle: mapLinkedInConnectionToIdentity(connection, existingIdentity.contactId).platformHandle,
+        platformData: mapLinkedInConnectionToIdentity(connection, existingIdentity.contactId).platformData,
+        lastSyncedAt: Math.floor(Date.now() / 1000),
+        updatedAt: Math.floor(Date.now() / 1000),
+      })
+      .where(eq(contactIdentities.id, existingIdentity.id))
+      .run();
+
+    recalcEnrichment(existingIdentity.contactId);
+    result.updated++;
+    return;
+  }
+
+  // 2. Cross-platform dedup: check if we can match by name (first+last)
+  //    Note: email is not available at the connection level without additional API scopes
+  const contactData = mapLinkedInConnectionToContact(connection);
+
+  // Create new contact
+  const contact = createContact(contactData);
+
+  // Create identity linking this contact to the LinkedIn connection
+  const identityData = mapLinkedInConnectionToIdentity(connection, contact.id);
+  createIdentity(identityData);
+
+  // Recompute enrichment with the new identity
+  recalcEnrichment(contact.id);
+  result.added++;
+}

package/src/lib/platforms/sync-x-profiles.ts

@@ -0,0 +1,280 @@
+import { eq, and, asc, isNotNull, inArray } from "drizzle-orm";
+import { db } from "@/lib/db/client";
+import { contacts, contactIdentities } from "@/lib/db/schema";
+import { updateContact } from "@/lib/db/queries/contacts";
+import { updateIdentity } from "@/lib/db/queries/identities";
+import { updatePlatformAccount } from "@/lib/db/queries/platform-accounts";
+import { getSyncCursor, updateSyncCursor } from "@/lib/db/queries/sync";
+import { XScraper } from "@/lib/browser/platforms/x-scraper";
+import { parseProfile } from "@/lib/browser/extractors/profile-parser";
+import {
+  mergeProfileData,
+  buildPlatformDataUpdate,
+} from "@/lib/browser/extractors/profile-merger";
+import type { SyncResult } from "@/lib/platforms/adapter";
+
+/** Contact with its X identity info, ready for enrichment. */
+interface EnrichableContact {
+  id: string;
+  name: string;
+  company: string | null;
+  title: string | null;
+  headline: string | null;
+  email: string | null;
+  phone: string | null;
+  metadata: string | null;
+  enrichmentScore: number;
+  identityId: string;
+  platformHandle: string;
+  platformData: string | null;
+}
+
+/** Minimum days between re-scraping the same contact. */
+const COOLDOWN_DAYS = 7;
+
+/**
+ * Enrich X contacts by scraping their profile pages and extracting
+ * structured data via LLM. Follows the sync-gmail-metadata.ts pattern:
+ * sync cursor lifecycle, per-contact error isolation, partial failure.
+ */
+export async function syncXProfiles(
+  accountId: string,
+  opts?: {
+    maxProfiles?: number;
+    contactIds?: string[];
+  }
+): Promise<SyncResult> {
+  const maxProfiles = opts?.maxProfiles ?? 15;
+  const result: SyncResult = { added: 0, updated: 0, skipped: 0, errors: [] };
+
+  // 1. Get or create sync cursor
+  const cursor = getSyncCursor(accountId, "x_profiles");
+  updateSyncCursor(cursor.id, {
+    syncStatus: "syncing",
+    lastSyncStartedAt: Math.floor(Date.now() / 1000),
+  });
+
+  let scraper: XScraper | null = null;
+
+  try {
+    // 2. Select contacts to enrich
+    const enrichable = selectContactsForEnrichment({
+      maxProfiles,
+      contactIds: opts?.contactIds,
+    });
+
+    if (enrichable.length === 0) {
+      updateSyncCursor(cursor.id, {
+        syncStatus: "completed",
+        lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      });
+      return result;
+    }
+
+    // 3. Initialize scraper and validate session
+    scraper = new XScraper();
+    await scraper.init();
+
+    if (!(await scraper.validateSession())) {
+      throw new Error(
+        "X browser session is invalid or expired. Please re-authenticate in Settings."
+      );
+    }
+
+    let processed = 0;
+
+    // 4. Per-contact enrichment loop
+    for (const contact of enrichable) {
+      if (scraper.batchLimitReached) {
+        result.errors.push(
+          `Batch limit reached after ${processed} profiles. Remaining contacts will be enriched in the next batch.`
+        );
+        break;
+      }
+
+      try {
+        // 4a. Scrape profile page
+        const raw = await scraper.scrapeProfile(contact.platformHandle);
+        if (!raw) {
+          result.skipped++;
+          continue;
+        }
+
+        // 4b. LLM extraction
+        const parsed = await parseProfile(raw);
+
+        // 4c. Merge into contact (fill gaps, don't overwrite)
+        // Build a minimal Contact-like object for the merger
+        const contactForMerge = {
+          id: contact.id,
+          name: contact.name,
+          company: contact.company,
+          title: contact.title,
+          headline: contact.headline,
+          email: contact.email,
+          phone: contact.phone,
+          metadata: contact.metadata,
+        };
+
+        const updates = mergeProfileData(
+          contactForMerge as Parameters<typeof mergeProfileData>[0],
+          parsed,
+          raw
+        );
+
+        if (Object.keys(updates).length > 0) {
+          // updateContact auto-calls recalcEnrichment(contactId)
+          updateContact(contact.id, updates);
+          result.updated++;
+        } else {
+          result.skipped++;
+        }
+
+        // 4d. Update identity platformData for rich data scoring (+10 pts)
+        const updatedPlatformData = buildPlatformDataUpdate(
+          contact.platformData,
+          parsed
+        );
+        updateIdentity(contact.identityId, {
+          platformData: updatedPlatformData,
+        });
+
+        processed++;
+      } catch (err) {
+        const message =
+          err instanceof Error ? err.message : String(err);
+        result.errors.push(
+          `Failed to enrich @${contact.platformHandle}: ${message}`
+        );
+
+        // Stop batch on challenge/CAPTCHA detection
+        if (message.includes("Challenge/CAPTCHA")) {
+          result.errors.push("Stopping batch due to challenge detection.");
+          break;
+        }
+      }
+    }
+
+    // 5. Mark sync completed
+    const now = Math.floor(Date.now() / 1000);
+    updateSyncCursor(cursor.id, {
+      syncStatus: "completed",
+      totalItemsSynced: (cursor.totalItemsSynced ?? 0) + processed,
+      lastSyncCompletedAt: now,
+    });
+    updatePlatformAccount(accountId, { lastSyncedAt: now });
+  } catch (err) {
+    // Partial sync pattern: ALWAYS set lastSyncCompletedAt even on error
+    result.errors.push(
+      `Profile enrichment failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+    updateSyncCursor(cursor.id, {
+      syncStatus: "failed",
+      lastSyncCompletedAt: Math.floor(Date.now() / 1000),
+      lastError: err instanceof Error ? err.message : String(err),
+    });
+  } finally {
+    await scraper?.close();
+  }
+
+  return result;
+}
+
+/**
+ * Select contacts for browser enrichment.
+ * Prioritizes low enrichment scores, skips recently scraped (7-day cooldown).
+ */
+function selectContactsForEnrichment(opts: {
+  maxProfiles: number;
+  contactIds?: string[];
+}): EnrichableContact[] {
+  const cooldownThreshold =
+    Math.floor(Date.now() / 1000) - COOLDOWN_DAYS * 86400;
+
+  // Base query: contacts with an X identity that has a handle
+  let rows;
+  if (opts.contactIds?.length) {
+    // Specific contacts requested
+    rows = db
+      .select({
+        id: contacts.id,
+        name: contacts.name,
+        company: contacts.company,
+        title: contacts.title,
+        headline: contacts.headline,
+        email: contacts.email,
+        phone: contacts.phone,
+        metadata: contacts.metadata,
+        enrichmentScore: contacts.enrichmentScore,
+        identityId: contactIdentities.id,
+        platformHandle: contactIdentities.platformHandle,
+        platformData: contactIdentities.platformData,
+      })
+      .from(contacts)
+      .innerJoin(
+        contactIdentities,
+        eq(contacts.id, contactIdentities.contactId)
+      )
+      .where(
+        and(
+          inArray(contacts.id, opts.contactIds),
+          eq(contactIdentities.platform, "x"),
+          isNotNull(contactIdentities.platformHandle)
+        )
+      )
+      .all();
+  } else {
+    // Auto-select: lowest enrichment scores first
+    // Fetch more than needed to account for cooldown filtering
+    rows = db
+      .select({
+        id: contacts.id,
+        name: contacts.name,
+        company: contacts.company,
+        title: contacts.title,
+        headline: contacts.headline,
+        email: contacts.email,
+        phone: contacts.phone,
+        metadata: contacts.metadata,
+        enrichmentScore: contacts.enrichmentScore,
+        identityId: contactIdentities.id,
+        platformHandle: contactIdentities.platformHandle,
+        platformData: contactIdentities.platformData,
+      })
+      .from(contacts)
+      .innerJoin(
+        contactIdentities,
+        eq(contacts.id, contactIdentities.contactId)
+      )
+      .where(
+        and(
+          eq(contactIdentities.platform, "x"),
+          isNotNull(contactIdentities.platformHandle)
+        )
+      )
+      .orderBy(asc(contacts.enrichmentScore))
+      .limit(opts.maxProfiles * 2) // over-fetch for cooldown filtering
+      .all();
+  }
+
+  // Filter: skip recently scraped contacts (7-day cooldown)
+  const filtered = rows.filter((row) => {
+    if (!row.metadata) return true;
+    try {
+      const meta = JSON.parse(row.metadata);
+      const scrapedAt = meta?.browserEnrichment?.scrapedAt;
+      if (typeof scrapedAt === "number" && scrapedAt > cooldownThreshold) {
+        return false; // too recent
+      }
+    } catch {
+      // invalid metadata JSON — allow enrichment
+    }
+    return true;
+  });
+
+  // Cast platformHandle (Drizzle returns string | null from isNotNull)
+  return filtered.slice(0, opts.maxProfiles).map((row) => ({
+    ...row,
+    platformHandle: row.platformHandle!,
+  }));
+}

package/src/lib/platforms/x/adapter.ts

@@ -0,0 +1,129 @@
+import type {
+  PlatformAdapter,
+  PlatformCredentials,
+  PlatformUserProfile,
+  PaginatedResult,
+  RateLimitState,
+} from "@/lib/platforms/adapter";
+import { getXClientCredentials, refreshXTokenAsync, disconnectXAccount } from "@/lib/platforms/x/auth";
+import { savePkceState } from "@/lib/platforms/x/pkce-store";
+import {
+  getAuthenticatedUser,
+  getUserById as xGetUserById,
+  getFollowing,
+} from "@/lib/platforms/x/client";
+import { getRateLimitState } from "@/lib/platforms/rate-limiter";
+import { getPlatformAccountByPlatform } from "@/lib/db/queries/platform-accounts";
+import { randomBytes, createHash } from "crypto";
+import type { XUser } from "@/lib/platforms/x/client";
+
+/** Map an XUser to the normalized PlatformUserProfile. */
+function toProfile(xUser: XUser): PlatformUserProfile {
+  return {
+    platformUserId: xUser.id,
+    platformHandle: `@${xUser.username}`,
+    displayName: xUser.name,
+    bio: xUser.description ?? null,
+    location: xUser.location ?? null,
+    website: xUser.url ?? null,
+    photoUrl: xUser.profile_image_url?.replace("_normal", "_400x400") ?? null,
+    followersCount: xUser.public_metrics?.followers_count ?? 0,
+    followingCount: xUser.public_metrics?.following_count ?? 0,
+    rawData: xUser as unknown as Record<string, unknown>,
+  };
+}
+
+export class XPlatformAdapter implements PlatformAdapter {
+  readonly platform = "x" as const;
+
+  async getAuthorizationUrl(redirectUri: string, extended = false): Promise<{ authUrl: string; state: string }> {
+    const { clientId } = getXClientCredentials();
+
+    const codeVerifier = randomBytes(32).toString("base64url");
+    const codeChallenge = createHash("sha256").update(codeVerifier).digest("base64url");
+    const state = randomBytes(16).toString("hex");
+
+    savePkceState(state, codeVerifier, extended);
+
+    // Free tier: CRM engagement + messaging; Extended (Basic+): adds contact management
+    const FREE_SCOPES = "tweet.read tweet.write tweet.moderate.write users.read like.read like.write bookmark.read bookmark.write dm.read dm.write offline.access";
+    const EXTENDED_SCOPES = "tweet.read tweet.write tweet.moderate.write users.read like.read like.write bookmark.read bookmark.write dm.read dm.write follows.read follows.write list.read list.write mute.read mute.write block.read block.write space.read offline.access";
+
+    const params = new URLSearchParams({
+      response_type: "code",
+      client_id: clientId,
+      redirect_uri: redirectUri,
+      scope: extended ? EXTENDED_SCOPES : FREE_SCOPES,
+      state,
+      code_challenge: codeChallenge,
+      code_challenge_method: "S256",
+    });
+
+    return {
+      authUrl: `https://x.com/i/oauth2/authorize?${params.toString()}`,
+      state,
+    };
+  }
+
+  async exchangeCode(
+    code: string,
+    _state: string,
+    redirectUri: string
+  ): Promise<PlatformCredentials> {
+    // Code exchange is handled by the callback API route directly
+    // This is here to satisfy the interface
+    throw new Error("Use /api/platforms/x/callback for code exchange");
+  }
+
+  async refreshToken(accountId: string): Promise<PlatformCredentials> {
+    return refreshXTokenAsync(accountId);
+  }
+
+  async revokeToken(accountId: string): Promise<void> {
+    return disconnectXAccount(accountId);
+  }
+
+  async getProfile(accountId: string): Promise<PlatformUserProfile> {
+    const xUser = await getAuthenticatedUser(accountId);
+    return toProfile(xUser);
+  }
+
+  async getContacts(
+    accountId: string,
+    cursor?: string
+  ): Promise<PaginatedResult<PlatformUserProfile>> {
+    // Get the authenticated user's ID to fetch their "following" list
+    const account = getPlatformAccountByPlatform("x");
+    if (!account) throw new Error("No X account connected");
+
+    const me = await getAuthenticatedUser(accountId);
+    const res = await getFollowing(accountId, me.id, {
+      maxResults: 100,
+      paginationToken: cursor || undefined,
+    });
+
+    const items = Array.isArray(res.data) ? res.data.map(toProfile) : [];
+
+    return {
+      items,
+      nextCursor: res.meta?.next_token ?? null,
+      hasMore: !!res.meta?.next_token,
+    };
+  }
+
+  async getUserById(
+    accountId: string,
+    userId: string
+  ): Promise<PlatformUserProfile | null> {
+    try {
+      const xUser = await xGetUserById(accountId, userId);
+      return toProfile(xUser);
+    } catch {
+      return null;
+    }
+  }
+
+  getRateLimitState(accountId: string): RateLimitState {
+    return getRateLimitState(accountId);
+  }
+}