@paymentsdb/sync-engine 0.0.5 → 0.0.7

package/dist/cli/lib.cjs

@@ -117,7 +117,7 @@ async function loadConfig(options) {
 // package.json
 var package_default = {
   name: "@paymentsdb/sync-engine",
-  version: "0.0.4",
+  version: "0.0.7",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",
@@ -45515,6 +45515,13 @@ var StripeSync = class {
         listFn: (p) => this.stripe.checkout.sessions.list(p),
         upsertFn: (items, id) => this.upsertCheckoutSessions(items, id),
         supportsCreatedFilter: true
+      },
+      _event_catchup: {
+        order: 99,
+        // Always runs last — catches missed webhook events
+        listFn: (_p) => Promise.resolve({ data: [], has_more: false }),
+        upsertFn: async () => [],
+        supportsCreatedFilter: true
       }
     };
     const maxOrder = Math.max(...Object.values(core).map((cfg) => cfg.order));
@@ -46281,7 +46288,8 @@ ${message}`;
       credit_note: "credit_notes",
       early_fraud_warning: "early_fraud_warnings",
       refund: "refunds",
-      checkout_sessions: "checkout_sessions"
+      checkout_sessions: "checkout_sessions",
+      _event_catchup: "_event_catchup"
     };
     return mapping[object] || object;
   }
@@ -46292,11 +46300,16 @@ ${message}`;
    */
   async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
     const limit = 100;
-    if (object === "payment_method" || object === "tax_id") {
-      this.config.logger?.warn(`processNext for ${object} requires customer context`);
+    if (object === "payment_method") {
+      return this.fetchOnePagePaymentMethods(accountId, resourceName, runStartedAt, cursor, pageCursor, params);
+    }
+    if (object === "tax_id") {
       await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
       return { processed: 0, hasMore: false, runStartedAt };
     }
+    if (object === "_event_catchup") {
+      return this.fetchOnePageEventCatchup(accountId, resourceName, runStartedAt, cursor, pageCursor);
+    }
     const config = this.resourceRegistry[object];
     if (!config) {
       throw new Error(`Unsupported object type for processNext: ${object}`);
@@ -46473,6 +46486,437 @@ ${message}`;
     }
     return { processed: entries.length, hasMore, runStartedAt };
   }
+  /**
+   * Fetch payment methods by iterating customers in batches.
+   *
+   * Most customers have 1-2 PMs, so processing one customer per processNext call
+   * would be extremely wasteful (10,000 customers = 10,000 worker round-trips).
+   * Instead, each call fetches a batch of customers (sized by maxConcurrentCustomers,
+   * default 10) and lists+upserts PMs for all of them concurrently.
+   *
+   * Cursor semantics:
+   * - cursor: max customer `created` timestamp — filters to only new customers on subsequent runs
+   * - pageCursor: last processed customer ID — tracks position in customer iteration
+   */
+  async fetchOnePagePaymentMethods(accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
+    const batchSize = this.config.maxConcurrentCustomers ?? 10;
+    try {
+      const customerBatch = await this.findNextCustomerBatchForPmSync(
+        accountId,
+        pageCursor,
+        cursor,
+        batchSize
+      );
+      if (customerBatch.length === 0) {
+        await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
+        return { processed: 0, hasMore: false, runStartedAt };
+      }
+      this.config.logger?.info(
+        `processNext: fetching payment_methods for ${customerBatch.length} customers`
+      );
+      let totalProcessed = 0;
+      await Promise.all(
+        customerBatch.map(async (customer) => {
+          const allPms = [];
+          let hasMore = true;
+          let startingAfter;
+          while (hasMore) {
+            const response = await this.stripe.paymentMethods.list({
+              customer: customer.id,
+              limit: 100,
+              ...startingAfter ? { starting_after: startingAfter } : {}
+            });
+            allPms.push(...response.data);
+            hasMore = response.has_more;
+            if (response.data.length > 0) {
+              startingAfter = response.data[response.data.length - 1].id;
+            }
+          }
+          if (allPms.length > 0) {
+            await this.upsertPaymentMethods(allPms, accountId, params?.backfillRelatedEntities);
+            totalProcessed += allPms.length;
+          }
+        })
+      );
+      if (totalProcessed > 0) {
+        await this.postgresClient.incrementObjectProgress(
+          accountId,
+          runStartedAt,
+          resourceName,
+          totalProcessed
+        );
+      }
+      const maxCreated = Math.max(...customerBatch.map((c) => c.created).filter((c) => c != null));
+      if (maxCreated > 0) {
+        await this.postgresClient.updateObjectCursor(
+          accountId,
+          runStartedAt,
+          resourceName,
+          String(maxCreated)
+        );
+      }
+      const lastCustomerId = customerBatch[customerBatch.length - 1].id;
+      const nextBatch = await this.findNextCustomerBatchForPmSync(
+        accountId,
+        lastCustomerId,
+        cursor,
+        1
+      );
+      if (nextBatch.length > 0) {
+        await this.postgresClient.updateObjectPageCursor(
+          accountId,
+          runStartedAt,
+          resourceName,
+          lastCustomerId
+        );
+        return { processed: totalProcessed, hasMore: true, runStartedAt };
+      }
+      await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
+      return { processed: totalProcessed, hasMore: false, runStartedAt };
+    } catch (error) {
+      await this.postgresClient.failObjectSync(
+        accountId,
+        runStartedAt,
+        resourceName,
+        error instanceof Error ? error.message : "Unknown error"
+      );
+      throw error;
+    }
+  }
+  /**
+   * Fetch the next batch of non-deleted customers for PM sync.
+   * Returns customer IDs and created timestamps, ordered by id ASC.
+   */
+  async findNextCustomerBatchForPmSync(accountId, afterCustomerId, cursor, limit) {
+    let query = `SELECT id, created FROM stripe.customers WHERE _account_id = $1 AND COALESCE(deleted, false) <> true`;
+    const params = [accountId];
+    if (afterCustomerId) {
+      params.push(afterCustomerId);
+      query += ` AND id > $${params.length}`;
+    }
+    if (cursor && /^\d+$/.test(cursor)) {
+      params.push(Number.parseInt(cursor, 10));
+      query += ` AND created >= $${params.length}`;
+    }
+    params.push(limit);
+    query += ` ORDER BY id ASC LIMIT $${params.length}`;
+    const result = await this.postgresClient.query(query, params);
+    return result.rows;
+  }
+  /**
+   * Fetch one page of events from the Stripe Events API and reconcile affected entities.
+   *
+   * Instead of replaying events (which can resurrect deleted objects due to newest-first ordering),
+   * we deduplicate by entity and re-fetch current state from Stripe for each affected entity.
+   *
+   * Cursor: event `created` timestamp. On first run, starts from the sync run's startedAt.
+   * On subsequent runs, picks up where the last completed run left off.
+   */
+  async fetchOnePageEventCatchup(accountId, resourceName, runStartedAt, cursor, pageCursor) {
+    try {
+      let createdGte;
+      if (cursor && /^\d+$/.test(cursor)) {
+        createdGte = Number.parseInt(cursor, 10);
+      } else {
+        createdGte = Math.floor(runStartedAt.getTime() / 1e3);
+      }
+      const thirtyDaysAgo = Math.floor(Date.now() / 1e3) - 30 * 24 * 60 * 60;
+      if (createdGte < thirtyDaysAgo) {
+        this.config.logger?.warn(
+          `_event_catchup: cursor ${createdGte} is older than 30 days, clamping to ${thirtyDaysAgo}`
+        );
+        createdGte = thirtyDaysAgo;
+      }
+      const listParams = {
+        limit: 100,
+        created: { gte: createdGte }
+      };
+      if (pageCursor) {
+        listParams.starting_after = pageCursor;
+      }
+      const response = await this.stripe.events.list(listParams);
+      if (response.data.length === 0) {
+        await this.postgresClient.updateObjectCursor(
+          accountId,
+          runStartedAt,
+          resourceName,
+          String(createdGte)
+        );
+        await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
+        return { processed: 0, hasMore: false, runStartedAt };
+      }
+      const entityMap = /* @__PURE__ */ new Map();
+      for (const event of response.data) {
+        const obj = event.data.object;
+        if (!obj?.id || !obj?.object) continue;
+        const key = `${obj.object}:${obj.id}`;
+        const existing = entityMap.get(key);
+        if (!existing || event.created > existing.created) {
+          entityMap.set(key, event);
+        }
+      }
+      const hardDeleteEventTypes = /* @__PURE__ */ new Set([
+        "product.deleted",
+        "price.deleted",
+        "plan.deleted",
+        "customer.deleted",
+        "customer.tax_id.deleted"
+      ]);
+      let processed = 0;
+      let skipped = 0;
+      const skipObjectTypes = /* @__PURE__ */ new Set(["tax_id"]);
+      for (const [, event] of entityMap) {
+        const obj = event.data.object;
+        const eventType = event.type;
+        if (skipObjectTypes.has(obj.object)) continue;
+        try {
+          if (!hardDeleteEventTypes.has(eventType)) {
+            const tableName = eventObjectTypeToTable(obj.object);
+            if (tableName) {
+              const localRecord = await this.postgresClient.query(
+                `SELECT "_last_synced_at" FROM stripe."${tableName}"
+                 WHERE id = $1 AND "_account_id" = $2 LIMIT 1`,
+                [obj.id, accountId]
+              );
+              if (localRecord.rows.length > 0 && localRecord.rows[0]._last_synced_at != null) {
+                const syncedAt = new Date(localRecord.rows[0]._last_synced_at).getTime() / 1e3;
+                if (syncedAt >= event.created) {
+                  skipped++;
+                  continue;
+                }
+              }
+            }
+          }
+          if (hardDeleteEventTypes.has(eventType)) {
+            await this.handleEventCatchupDelete(obj.object, obj.id, accountId);
+          } else {
+            await this.handleEventCatchupUpsert(obj.object, obj.id, accountId);
+          }
+          processed++;
+        } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          this.config.logger?.warn(
+            `_event_catchup: failed to process ${obj.object}:${obj.id} (event ${event.id}): ${errMsg}`
+          );
+        }
+      }
+      if (skipped > 0) {
+        this.config.logger?.info(
+          `_event_catchup: skipped ${skipped} entities already up-to-date, processed ${processed}`
+        );
+      }
+      if (processed > 0) {
+        await this.postgresClient.incrementObjectProgress(
+          accountId,
+          runStartedAt,
+          resourceName,
+          processed
+        );
+      }
+      const maxCreated = Math.max(...response.data.map((e) => e.created));
+      await this.postgresClient.updateObjectCursor(
+        accountId,
+        runStartedAt,
+        resourceName,
+        String(maxCreated)
+      );
+      const lastEventId = response.data[response.data.length - 1].id;
+      if (response.has_more) {
+        await this.postgresClient.updateObjectPageCursor(
+          accountId,
+          runStartedAt,
+          resourceName,
+          lastEventId
+        );
+      }
+      if (!response.has_more) {
+        await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
+      }
+      return { processed, hasMore: response.has_more, runStartedAt };
+    } catch (error) {
+      await this.postgresClient.failObjectSync(
+        accountId,
+        runStartedAt,
+        resourceName,
+        error instanceof Error ? error.message : "Unknown error"
+      );
+      throw error;
+    }
+  }
+  /**
+   * Handle a delete for an entity discovered via event catch-up.
+   * Maps Stripe object types to the appropriate delete method.
+   */
+  async handleEventCatchupDelete(objectType, entityId, accountId) {
+    switch (objectType) {
+      case "product":
+        await this.deleteProduct(entityId);
+        break;
+      case "price":
+        await this.deletePrice(entityId);
+        break;
+      case "plan":
+        await this.deletePlan(entityId);
+        break;
+      case "customer": {
+        const deletedCustomer = {
+          id: entityId,
+          object: "customer",
+          deleted: true
+        };
+        await this.upsertCustomers([deletedCustomer], accountId);
+        break;
+      }
+      case "tax_id":
+        await this.deleteTaxId(entityId);
+        break;
+      default:
+        this.config.logger?.warn(
+          `_event_catchup: no delete handler for object type "${objectType}", skipping ${entityId}`
+        );
+    }
+  }
+  /**
+   * Handle an upsert for an entity discovered via event catch-up.
+   * Re-fetches the current state from Stripe and upserts it.
+   * If the entity has been deleted (404), falls back to the delete handler.
+   */
+  async handleEventCatchupUpsert(objectType, entityId, accountId) {
+    try {
+      switch (objectType) {
+        case "product": {
+          const product = await this.stripe.products.retrieve(entityId);
+          await this.upsertProducts([product], accountId);
+          break;
+        }
+        case "price": {
+          const price = await this.stripe.prices.retrieve(entityId);
+          await this.upsertPrices([price], accountId);
+          break;
+        }
+        case "plan": {
+          const plan = await this.stripe.plans.retrieve(entityId);
+          await this.upsertPlans([plan], accountId);
+          break;
+        }
+        case "customer": {
+          const customer = await this.stripe.customers.retrieve(entityId);
+          await this.upsertCustomers([customer], accountId);
+          break;
+        }
+        case "subscription": {
+          const sub = await this.stripe.subscriptions.retrieve(entityId);
+          await this.upsertSubscriptions([sub], accountId);
+          break;
+        }
+        case "subscription_schedule": {
+          const schedule = await this.stripe.subscriptionSchedules.retrieve(entityId);
+          await this.upsertSubscriptionSchedules([schedule], accountId);
+          break;
+        }
+        case "invoice": {
+          const invoice = await this.stripe.invoices.retrieve(entityId);
+          await this.upsertInvoices([invoice], accountId);
+          break;
+        }
+        case "charge": {
+          const charge = await this.stripe.charges.retrieve(entityId, {
+            expand: ["balance_transaction"]
+          });
+          if (charge.balance_transaction && typeof charge.balance_transaction === "object") {
+            await this.upsertBalanceTransactions(
+              [charge.balance_transaction],
+              accountId
+            );
+          }
+          await this.upsertCharges([charge], accountId);
+          break;
+        }
+        case "payment_intent": {
+          const pi = await this.stripe.paymentIntents.retrieve(entityId);
+          await this.upsertPaymentIntents([pi], accountId);
+          break;
+        }
+        case "payment_method": {
+          const pm = await this.stripe.paymentMethods.retrieve(entityId);
+          await this.upsertPaymentMethods([pm], accountId);
+          break;
+        }
+        case "setup_intent": {
+          const si = await this.stripe.setupIntents.retrieve(entityId);
+          await this.upsertSetupIntents([si], accountId);
+          break;
+        }
+        case "dispute": {
+          const dispute = await this.stripe.disputes.retrieve(entityId, {
+            expand: ["balance_transactions"]
+          });
+          if (dispute.balance_transactions && Array.isArray(dispute.balance_transactions)) {
+            const expandedBts = dispute.balance_transactions.filter(
+              (bt) => typeof bt === "object"
+            );
+            if (expandedBts.length > 0) {
+              await this.upsertBalanceTransactions(expandedBts, accountId);
+            }
+          }
+          await this.upsertDisputes([dispute], accountId);
+          break;
+        }
+        case "credit_note": {
+          const cn = await this.stripe.creditNotes.retrieve(entityId);
+          await this.upsertCreditNotes([cn], accountId);
+          break;
+        }
+        case "refund": {
+          const refund = await this.stripe.refunds.retrieve(entityId, {
+            expand: ["balance_transaction"]
+          });
+          if (refund.balance_transaction && typeof refund.balance_transaction === "object") {
+            await this.upsertBalanceTransactions(
+              [refund.balance_transaction],
+              accountId
+            );
+          }
+          await this.upsertRefunds([refund], accountId);
+          break;
+        }
+        case "tax_id": {
+          const taxId = await this.stripe.taxIds.retrieve(entityId);
+          await this.upsertTaxIds([taxId], accountId);
+          break;
+        }
+        case "balance_transaction": {
+          const bt = await this.stripe.balanceTransactions.retrieve(entityId);
+          await this.upsertBalanceTransactions([bt], accountId);
+          break;
+        }
+        case "checkout.session": {
+          const session = await this.stripe.checkout.sessions.retrieve(entityId);
+          await this.upsertCheckoutSessions([session], accountId);
+          break;
+        }
+        case "radar.early_fraud_warning": {
+          const efw = await this.stripe.radar.earlyFraudWarnings.retrieve(entityId);
+          await this.upsertEarlyFraudWarning([efw], accountId);
+          break;
+        }
+        default:
+          this.config.logger?.warn(
+            `_event_catchup: no upsert handler for object type "${objectType}", skipping ${entityId}`
+          );
+      }
+    } catch (err) {
+      const isNotFound = (err instanceof import_stripe3.default.errors.StripeInvalidRequestError || err instanceof import_stripe3.default.errors.StripeAPIError) && (err.code === "resource_missing" || err.statusCode === 404);
+      if (isNotFound) {
+        this.config.logger?.info(
+          `_event_catchup: ${objectType}:${entityId} not found on Stripe, treating as deleted`
+        );
+        await this.handleEventCatchupDelete(objectType, entityId, accountId);
+        return;
+      }
+      throw err;
+    }
+  }
   /**
    * Process all pages for all (or specified) object types until complete.
    *
@@ -46546,6 +46990,7 @@ ${message}`;
     };
   }
   applySyncBackfillResult(results, object, result) {
+    if (object === "_event_catchup") return;
     if (this.isSigmaResource(object)) {
       results.sigma = results.sigma ?? {};
       results.sigma[object] = result;
@@ -46581,6 +47026,9 @@ ${message}`;
       case "setup_intent":
         results.setupIntents = result;
         break;
+      case "payment_method":
+        results.paymentMethods = result;
+        break;
       case "payment_intent":
         results.paymentIntents = result;
         break;
@@ -48088,6 +48536,29 @@ function chunkArray(array, chunkSize) {
   }
   return result;
 }
+function eventObjectTypeToTable(objectType) {
+  const mapping = {
+    product: "products",
+    price: "prices",
+    plan: "plans",
+    customer: "customers",
+    subscription: "subscriptions",
+    subscription_schedule: "subscription_schedules",
+    invoice: "invoices",
+    charge: "charges",
+    balance_transaction: "balance_transactions",
+    payment_intent: "payment_intents",
+    payment_method: "payment_methods",
+    setup_intent: "setup_intents",
+    dispute: "disputes",
+    credit_note: "credit_notes",
+    refund: "refunds",
+    tax_id: "tax_ids",
+    "checkout.session": "checkout_sessions",
+    "radar.early_fraud_warning": "early_fraud_warnings"
+  };
+  return mapping[objectType] ?? null;
+}
 
 // src/database/migrate.ts
 var import_pg2 = require("pg");

package/dist/cli/lib.js

@@ -6,10 +6,10 @@ import {
   migrateCommand,
   syncCommand,
   uninstallCommand
-} from "../chunk-DZMKGCU5.js";
-import "../chunk-LB4HG4Q6.js";
-import "../chunk-Q3AGYUHN.js";
-import "../chunk-HXSDJSKR.js";
+} from "../chunk-LN6KHV6O.js";
+import "../chunk-IQ64IUIL.js";
+import "../chunk-OLHHINPH.js";
+import "../chunk-7DYBM7H3.js";
 export {
   backfillCommand,
   createTunnel,