@lobehub/lobehub 2.0.0-next.339 → 2.0.0-next.340

package/scripts/clerk-to-betterauth/export-clerk-users-with-api.ts

@@ -0,0 +1,211 @@
+/* eslint-disable unicorn/prefer-top-level-await, unicorn/no-process-exit */
+import { type User, createClerkClient } from '@clerk/backend';
+import { writeFile } from 'node:fs/promises';
+
+import { getClerkSecret, getMigrationMode, resolveDataPaths } from './_internal/config';
+import './_internal/env';
+import { ClerkUser } from './_internal/types';
+
+/**
+ * Fetch all Clerk users via REST API and persist them into a local JSON file.
+ *
+ * Usage:
+ *   tsx scripts/clerk-to-betterauth/export-clerk-users.ts [outputPath]
+ *
+ * Env vars required (set by CLERK_TO_BETTERAUTH_MODE=test|prod):
+ *   - TEST_CLERK_TO_BETTERAUTH_CLERK_SECRET_KEY (test)
+ *   - PROD_CLERK_TO_BETTERAUTH_CLERK_SECRET_KEY (prod)
+ */
+const PAGE_SIZE = 500;
+const CONCURRENCY = Number(process.env.CLERK_EXPORT_CONCURRENCY ?? 10);
+const MAX_RETRIES = Number(process.env.CLERK_EXPORT_RETRIES ?? 10);
+const RETRY_DELAY_MS = 1000;
+const ORDER_BY = '+created_at';
+const DEFAULT_OUTPUT_PATH = resolveDataPaths().clerkUsersPath;
+const formatDuration = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
+
+const sleep = (ms: number) =>
+  new Promise<void>((resolve) => {
+    setTimeout(resolve, ms);
+  });
+
+function getClerkClient(secretKey: string) {
+  return createClerkClient({
+    secretKey,
+  });
+}
+
+function mapClerkUser(user: User): ClerkUser {
+  const raw = user.raw!;
+
+  const primaryEmail = raw.email_addresses?.find(
+    (email) => email.id === raw.primary_email_address_id,
+  )?.email_address;
+
+  return {
+    banned: raw.banned,
+    created_at: raw.created_at,
+    external_accounts: (raw.external_accounts ?? []).map((acc) => ({
+      approved_scopes: acc.approved_scopes,
+      created_at: (acc as any).created_at,
+      id: acc.id,
+      provider: acc.provider,
+      provider_user_id: acc.provider_user_id,
+      updated_at: (acc as any).updated_at,
+      verificationStatus: acc.verification?.status === 'verified',
+    })),
+    id: raw.id,
+    image_url: raw.image_url,
+    lockout_expires_in_seconds: raw.lockout_expires_in_seconds,
+    password_enabled: raw.password_enabled,
+    password_last_updated_at: raw.password_last_updated_at,
+    primaryEmail,
+    two_factor_enabled: raw.two_factor_enabled,
+    updated_at: raw.updated_at,
+  } satisfies ClerkUser;
+}
+
+async function fetchClerkUserPage(
+  offset: number,
+  clerkClient: ReturnType<typeof getClerkClient>,
+  pageIndex: number,
+): Promise<ClerkUser[]> {
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt += 1) {
+    try {
+      console.log(
+        `🚚 [clerk-export] Fetching page #${pageIndex + 1} offset=${offset} limit=${PAGE_SIZE} (attempt ${attempt}/${MAX_RETRIES})`,
+      );
+
+      const { data } = await clerkClient.users.getUserList({
+        limit: PAGE_SIZE,
+        offset,
+        orderBy: ORDER_BY,
+      });
+
+      console.log(
+        `📥 [clerk-export] Received page #${pageIndex + 1} (${data.length} users) offset=${offset}`,
+      );
+
+      return data.map(mapClerkUser);
+    } catch (error) {
+      const isLastAttempt = attempt === MAX_RETRIES;
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(
+        `⚠️ [clerk-export] Page #${pageIndex + 1} offset=${offset} failed (attempt ${attempt}/${MAX_RETRIES}): ${message}`,
+      );
+
+      if (isLastAttempt) {
+        throw error;
+      }
+
+      await sleep(RETRY_DELAY_MS);
+    }
+  }
+
+  // Unreachable, but satisfies TypeScript return.
+  return [];
+}
+
+async function runWithConcurrency<T>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<void>,
+): Promise<void> {
+  const queue = [...items];
+  const inFlight: Promise<void>[] = [];
+  let index = 0;
+
+  const launchNext = () => {
+    if (!queue.length) return;
+    const currentItem = queue.shift() as T;
+    const currentIndex = index;
+    index += 1;
+    const task = worker(currentItem, currentIndex).finally(() => {
+      const pos = inFlight.indexOf(task);
+      if (pos !== -1) inFlight.splice(pos, 1);
+    });
+    inFlight.push(task);
+  };
+
+  for (let i = 0; i < concurrency && queue.length; i += 1) {
+    launchNext();
+  }
+
+  while (inFlight.length) {
+    await Promise.race(inFlight);
+    launchNext();
+  }
+}
+
+async function fetchAllClerkUsers(secretKey: string): Promise<ClerkUser[]> {
+  const clerkClient = getClerkClient(secretKey);
+  const userMap = new Map<string, ClerkUser>();
+
+  const firstPageResponse = await clerkClient.users.getUserList({
+    limit: PAGE_SIZE,
+    offset: 0,
+    orderBy: ORDER_BY,
+  });
+
+  const totalCount = firstPageResponse.totalCount ?? firstPageResponse.data.length;
+  const totalPages = Math.ceil(totalCount / PAGE_SIZE);
+  const offsets = Array.from({ length: totalPages }, (_, pageIndex) => pageIndex * PAGE_SIZE);
+
+  console.log(
+    `📊 [clerk-export] Total users: ${totalCount}. Pages: ${totalPages}. Concurrency: ${CONCURRENCY}.`,
+  );
+
+  await runWithConcurrency(offsets, CONCURRENCY, async (offset, index) => {
+    const page = await fetchClerkUserPage(offset, clerkClient, index);
+
+    for (const user of page) {
+      userMap.set(user.id, user);
+    }
+
+    if ((index + 1) % CONCURRENCY === 0 || index === offsets.length - 1) {
+      console.log(
+        `⏳ [clerk-export] Progress: ${userMap.size}/${totalCount} unique users collected.`,
+      );
+    }
+  });
+
+  const uniqueCount = userMap.size;
+  const extraUsers = Math.max(0, uniqueCount - totalCount);
+
+  console.log(
+    `🆕 [clerk-export] Snapshot total=${totalCount}, final unique=${uniqueCount}, new during export=${extraUsers}`,
+  );
+
+  return Array.from(userMap.values());
+}
+
+async function main() {
+  const startedAt = Date.now();
+  const mode = getMigrationMode();
+  const secretKey = getClerkSecret();
+  const outputPath = process.argv[2] ?? DEFAULT_OUTPUT_PATH;
+
+  console.log('');
+  console.log('╔════════════════════════════════════════════════════════════╗');
+  console.log('║           Clerk Users Export Script (via API)              ║');
+  console.log('╠════════════════════════════════════════════════════════════╣');
+  console.log(`║  Mode:     ${mode.padEnd(48)}║`);
+  console.log(`║  Output:   ${outputPath.padEnd(48)}║`);
+  console.log('╚════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  const clerkUsers = await fetchAllClerkUsers(secretKey);
+
+  await writeFile(outputPath, JSON.stringify(clerkUsers, null, 2), 'utf8');
+
+  console.log('');
+  console.log(
+    `✅ Export success! Saved ${clerkUsers.length} users to ${outputPath} (${formatDuration(Date.now() - startedAt)})`,
+  );
+}
+
+void main().catch((error) => {
+  console.log('');
+  console.error('❌ Export failed:', error);
+  process.exit(1);
+});

package/scripts/clerk-to-betterauth/index.ts

@@ -0,0 +1,314 @@
+/* eslint-disable unicorn/prefer-top-level-await */
+import { sql } from 'drizzle-orm';
+
+import { getMigrationMode } from './_internal/config';
+import { db, pool, schema } from './_internal/db';
+import { loadCSVData, loadClerkUsersFromFile } from './_internal/load-data-from-files';
+import { ClerkExternalAccount } from './_internal/types';
+import { generateBackupCodes, safeDateConversion } from './_internal/utils';
+
+const BATCH_SIZE = Number(process.env.CLERK_TO_BETTERAUTH_BATCH_SIZE) || 300;
+const PROGRESS_TABLE = sql.identifier('clerk_migration_progress');
+const IS_DRY_RUN =
+  process.argv.includes('--dry-run') || process.env.CLERK_TO_BETTERAUTH_DRY_RUN === '1';
+const formatDuration = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
+
+function chunk<T>(items: T[], size: number): T[][] {
+  if (!Number.isFinite(size) || size <= 0) return [items];
+  const result: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    result.push(items.slice(i, i + size));
+  }
+  return result;
+}
+
+function computeBanExpires(lockoutSeconds?: number | null): Date | undefined {
+  if (typeof lockoutSeconds !== 'number') return undefined;
+  return new Date(Date.now() + lockoutSeconds * 1000);
+}
+
+async function migrateFromClerk() {
+  const mode = getMigrationMode();
+  const csvUsers = await loadCSVData();
+  const clerkUsers = await loadClerkUsersFromFile();
+  const clerkUserMap = new Map(clerkUsers.map((u) => [u.id, u]));
+
+  if (!IS_DRY_RUN) {
+    await db.execute(sql`
+      CREATE TABLE IF NOT EXISTS ${PROGRESS_TABLE} (
+        user_id TEXT PRIMARY KEY,
+        processed_at TIMESTAMPTZ DEFAULT NOW()
+      );
+    `);
+  }
+
+  const processedUsers = new Set<string>();
+
+  if (!IS_DRY_RUN) {
+    try {
+      const processedResult = await db.execute<{ user_id: string }>(
+        sql`SELECT user_id FROM ${PROGRESS_TABLE};`,
+      );
+      const rows = (processedResult as { rows?: { user_id: string }[] }).rows ?? [];
+
+      for (const row of rows) {
+        const userId = row?.user_id;
+        if (typeof userId === 'string') {
+          processedUsers.add(userId);
+        }
+      }
+    } catch (error) {
+      console.warn('[clerk-to-betterauth] failed to read progress table, treating as empty', error);
+    }
+  }
+
+  console.log(`[clerk-to-betterauth] mode: ${mode} (dryRun=${IS_DRY_RUN})`);
+  console.log(`[clerk-to-betterauth] csv users: ${csvUsers.length}`);
+  console.log(`[clerk-to-betterauth] clerk api users: ${clerkUsers.length}`);
+  console.log(`[clerk-to-betterauth] already processed: ${processedUsers.size}`);
+
+  const unprocessedUsers = csvUsers.filter((user) => !processedUsers.has(user.id));
+  const batches = chunk(unprocessedUsers, BATCH_SIZE);
+  console.log(
+    `[clerk-to-betterauth] batches: ${batches.length} (batchSize=${BATCH_SIZE}, toProcess=${unprocessedUsers.length})`,
+  );
+
+  let processed = 0;
+  let accountAttempts = 0;
+  let twoFactorAttempts = 0;
+  let skipped = csvUsers.length - unprocessedUsers.length;
+  const startedAt = Date.now();
+  const accountCounts: Record<string, number> = {};
+  let missingScopeNonCredential = 0;
+  let passwordEnabledButNoDigest = 0;
+  const sampleMissingScope: string[] = [];
+  const sampleMissingDigest: string[] = [];
+
+  const bumpAccountCount = (providerId: string) => {
+    accountCounts[providerId] = (accountCounts[providerId] ?? 0) + 1;
+  };
+
+  for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+    const batch = batches[batchIndex];
+    const userRows: (typeof schema.users.$inferInsert)[] = [];
+    const accountRows: (typeof schema.account.$inferInsert)[] = [];
+    const twoFactorRows: (typeof schema.twoFactor.$inferInsert)[] = [];
+
+    for (const user of batch) {
+      const clerkUser = clerkUserMap.get(user.id);
+      const lockoutSeconds = clerkUser?.lockout_expires_in_seconds;
+      const externalAccounts = clerkUser?.external_accounts as ClerkExternalAccount[] | undefined;
+
+      const userRow: typeof schema.users.$inferInsert = {
+        avatar: clerkUser?.image_url,
+        banExpires: computeBanExpires(lockoutSeconds) ?? undefined,
+        banned: Boolean(clerkUser?.banned),
+        clerkCreatedAt: safeDateConversion(clerkUser?.created_at),
+        email: user.primary_email_address,
+        emailVerified: Boolean(user.verified_email_addresses?.length),
+        firstName: user.first_name || undefined,
+        id: user.id,
+        lastName: user.last_name || undefined,
+        phone: user.primary_phone_number || undefined,
+        phoneNumberVerified: Boolean(user.verified_phone_numbers?.length),
+        role: 'user',
+        twoFactorEnabled: Boolean(clerkUser?.two_factor_enabled),
+        username: user.username || undefined,
+      };
+      userRows.push(userRow);
+
+      if (externalAccounts) {
+        for (const externalAccount of externalAccounts) {
+          const provider = externalAccount.provider;
+          const providerUserId = externalAccount.provider_user_id;
+
+          /**
+           * Clerk external accounts never contain credential providers and always include provider_user_id.
+           * Enforce this assumption to avoid inserting invalid account rows.
+           */
+          if (provider === 'credential') {
+            throw new Error(
+              `[clerk-to-betterauth] unexpected credential external account: userId=${user.id}, externalAccountId=${externalAccount.id}`,
+            );
+          }
+          if (!providerUserId) {
+            throw new Error(
+              `[clerk-to-betterauth] missing provider_user_id: userId=${user.id}, externalAccountId=${externalAccount.id}, provider=${provider}`,
+            );
+          }
+
+          const providerId = provider.replace('oauth_', '');
+
+          if (!externalAccount.approved_scopes) {
+            missingScopeNonCredential += 1;
+            if (sampleMissingScope.length < 5) sampleMissingScope.push(user.id);
+          }
+
+          accountRows.push({
+            accountId: providerUserId,
+            createdAt: safeDateConversion(externalAccount.created_at),
+            id: externalAccount.id,
+            providerId,
+            scope: externalAccount.approved_scopes?.replace(/\s+/g, ',') || undefined,
+            updatedAt: safeDateConversion(externalAccount.updated_at),
+            userId: user.id,
+          });
+          accountAttempts += 1;
+          bumpAccountCount(providerId);
+        }
+      }
+
+      // Clerk API 不返回 credential external_account；若用户开启密码并且 CSV 提供散列，则补齐本地密码账号
+      const passwordEnabled = Boolean(clerkUser?.password_enabled);
+      if (passwordEnabled && user.password_digest) {
+        const passwordUpdatedAt = clerkUser?.password_last_updated_at;
+
+        accountRows.push({
+          accountId: user.id,
+          createdAt: safeDateConversion(clerkUser?.created_at),
+          id: 'cred_' + user.id,
+          password: user.password_digest,
+          providerId: 'credential',
+          updatedAt: safeDateConversion(
+            passwordUpdatedAt ?? clerkUser?.updated_at ?? clerkUser?.created_at,
+          ),
+          userId: user.id,
+        });
+        accountAttempts += 1;
+        bumpAccountCount('credential');
+      } else if (passwordEnabled && !user.password_digest) {
+        passwordEnabledButNoDigest += 1;
+        if (sampleMissingDigest.length < 5) sampleMissingDigest.push(user.id);
+      }
+
+      if (user.totp_secret) {
+        twoFactorRows.push({
+          backupCodes: await generateBackupCodes(user.totp_secret),
+          id: `tf_${user.id}`,
+          secret: user.totp_secret,
+          userId: user.id,
+        });
+        twoFactorAttempts += 1;
+      }
+    }
+
+    if (!IS_DRY_RUN) {
+      await db.transaction(async (tx) => {
+        await tx
+          .insert(schema.users)
+          .values(userRows)
+          .onConflictDoUpdate({
+            set: {
+              avatar: sql`excluded.avatar`,
+              banExpires: sql`excluded.ban_expires`,
+              banned: sql`excluded.banned`,
+              clerkCreatedAt: sql`excluded.clerk_created_at`,
+              email: sql`excluded.email`,
+              emailVerified: sql`excluded.email_verified`,
+              firstName: sql`excluded.first_name`,
+              lastName: sql`excluded.last_name`,
+              phone: sql`excluded.phone`,
+              phoneNumberVerified: sql`excluded.phone_number_verified`,
+              role: sql`excluded.role`,
+              twoFactorEnabled: sql`excluded.two_factor_enabled`,
+              username: sql`excluded.username`,
+            },
+            target: schema.users.id,
+          });
+
+        if (accountRows.length > 0) {
+          await tx.insert(schema.account).values(accountRows).onConflictDoNothing();
+        }
+
+        if (twoFactorRows.length > 0) {
+          await tx.insert(schema.twoFactor).values(twoFactorRows).onConflictDoNothing();
+        }
+
+        const userIdValues = userRows.map((row) => sql`(${row.id})`);
+        if (userIdValues.length > 0) {
+          await tx.execute(sql`
+            INSERT INTO ${PROGRESS_TABLE} (user_id)
+            VALUES ${sql.join(userIdValues, sql`, `)}
+            ON CONFLICT (user_id) DO NOTHING;
+          `);
+        }
+      });
+    }
+    processed += batch.length;
+    console.log(
+      `[clerk-to-betterauth] batch ${batchIndex + 1}/${batches.length} done, users ${processed}/${unprocessedUsers.length}, accounts+=${accountRows.length}, 2fa+=${twoFactorRows.length}, dryRun=${IS_DRY_RUN}`,
+    );
+  }
+
+  console.log(
+    `[clerk-to-betterauth] completed users=${processed}, skipped=${skipped}, accounts attempted=${accountAttempts}, 2fa attempted=${twoFactorAttempts}, dryRun=${IS_DRY_RUN}, elapsed=${formatDuration(Date.now() - startedAt)}`,
+  );
+
+  const accountCountsText = Object.entries(accountCounts)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([providerId, count]) => `${providerId}=${count}`)
+    .join(', ');
+
+  console.log(
+    `[clerk-to-betterauth] account provider counts: ${accountCountsText || 'none recorded'}`,
+  );
+
+  console.log(
+    [
+      '[clerk-to-betterauth] anomalies:',
+      `  - missing scope (non-credential): ${missingScopeNonCredential} sample=${sampleMissingScope.join(';') || 'n/a'}`,
+      `  - passwordEnabled without digest: ${passwordEnabledButNoDigest} sample=${sampleMissingDigest.join(';') || 'n/a'}`,
+    ].join('\n'),
+  );
+}
+async function main() {
+  const startedAt = Date.now();
+  const mode = getMigrationMode();
+
+  console.log('');
+  console.log('╔════════════════════════════════════════════════════════════╗');
+  console.log('║           Clerk to Better Auth Migration Script            ║');
+  console.log('╠════════════════════════════════════════════════════════════╣');
+  console.log(`║  Mode:     ${mode.padEnd(48)}║`);
+  console.log(`║  Dry Run:  ${(IS_DRY_RUN ? 'YES (no changes will be made)' : 'NO').padEnd(48)}║`);
+  console.log(`║  Batch:    ${String(BATCH_SIZE).padEnd(48)}║`);
+  console.log('╚════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  if (mode === 'prod' && !IS_DRY_RUN) {
+    console.log('⚠️  WARNING: Running in PRODUCTION mode. Data will be modified!');
+    console.log('   Type "yes" to continue or press Ctrl+C to abort.');
+    console.log('');
+
+    const readline = await import('node:readline');
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const answer = await new Promise<string>((resolve) => {
+      rl.question('   Confirm (yes/no): ', (ans) => {
+        resolve(ans);
+      });
+    });
+    rl.close();
+
+    if (answer.toLowerCase() !== 'yes') {
+      console.log('❌ Aborted by user.');
+      process.exitCode = 0;
+      await pool.end();
+      return;
+    }
+    console.log('');
+  }
+
+  try {
+    await migrateFromClerk();
+    console.log('');
+    console.log(`✅ Migration success! (${formatDuration(Date.now() - startedAt)})`);
+  } catch (error) {
+    console.log('');
+    console.error(`❌ Migration failed (${formatDuration(Date.now() - startedAt)}):`, error);
+    process.exitCode = 1;
+  } finally {
+    await pool.end();
+  }
+}
+
+void main();