@paymentsdb/sync-engine 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-2KB2ISYF.js +48878 -0
- package/dist/{chunk-FII5OTPO.js → chunk-E6BGC7CB.js} +4 -2
- package/dist/{chunk-WQOTGHLT.js → chunk-GXIMCH5Y.js} +79 -8
- package/dist/{chunk-3X3TM6V4.js → chunk-WHNAH5B7.js} +95 -28
- package/dist/cli/index.cjs +45191 -377
- package/dist/cli/index.js +17 -11
- package/dist/cli/lib.cjs +46076 -1268
- package/dist/cli/lib.d.cts +1 -0
- package/dist/cli/lib.d.ts +1 -0
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +45074 -402
- package/dist/index.d.cts +113 -5
- package/dist/index.d.ts +113 -5
- package/dist/index.js +2 -2
- package/dist/migrations/0062_sigma_query_runs.sql +8 -0
- package/dist/migrations/0064_add_account_id_indexes.sql +120 -0
- package/dist/supabase/index.cjs +83 -9
- package/dist/supabase/index.d.cts +9 -2
- package/dist/supabase/index.d.ts +9 -2
- package/dist/supabase/index.js +4 -2
- package/package.json +13 -10
- package/dist/chunk-UD6RQUDV.js +0 -4207
- /package/dist/migrations/{0062_balance_transactions.sql → 0063_balance_transactions.sql} +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -33,7 +33,7 @@ declare class PostgresClient {
|
|
|
33
33
|
}>(entries: T[], table: string): Promise<T[]>;
|
|
34
34
|
upsertManyWithTimestampProtection<T extends {
|
|
35
35
|
[Key: string]: any;
|
|
36
|
-
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
|
|
36
|
+
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
|
|
37
37
|
private cleanseArrayField;
|
|
38
38
|
findMissingEntries(table: string, ids: string[]): Promise<string[]>;
|
|
39
39
|
upsertAccount(accountData: {
|
|
@@ -102,9 +102,10 @@ declare class PostgresClient {
|
|
|
102
102
|
cancelStaleRuns(accountId: string): Promise<void>;
|
|
103
103
|
/**
|
|
104
104
|
* Get or create a sync run for this account.
|
|
105
|
-
* Returns existing run if one is active, otherwise creates new one.
|
|
105
|
+
* Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
|
|
106
106
|
* Auto-cancels stale runs before checking.
|
|
107
107
|
*
|
|
108
|
+
* @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
|
|
108
109
|
* @returns RunKey with isNew flag, or null if constraint violation (race condition)
|
|
109
110
|
*/
|
|
110
111
|
getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
@@ -114,8 +115,9 @@ declare class PostgresClient {
|
|
|
114
115
|
} | null>;
|
|
115
116
|
/**
|
|
116
117
|
* Get the active sync run for an account (if any).
|
|
118
|
+
* @param triggeredBy - If provided, only returns run matching this triggeredBy value
|
|
117
119
|
*/
|
|
118
|
-
getActiveSyncRun(accountId: string): Promise<{
|
|
120
|
+
getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
119
121
|
accountId: string;
|
|
120
122
|
runStartedAt: Date;
|
|
121
123
|
} | null>;
|
|
@@ -129,6 +131,10 @@ declare class PostgresClient {
|
|
|
129
131
|
maxConcurrent: number;
|
|
130
132
|
closedAt: Date | null;
|
|
131
133
|
} | null>;
|
|
134
|
+
/**
|
|
135
|
+
* Ensure a sync run has at least the requested max_concurrent value.
|
|
136
|
+
*/
|
|
137
|
+
ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
|
|
132
138
|
/**
|
|
133
139
|
* Close a sync run (mark as done).
|
|
134
140
|
* Status (complete/error) is derived from object run states.
|
|
@@ -179,6 +185,11 @@ declare class PostgresClient {
|
|
|
179
185
|
* For non-numeric cursors, just sets the value directly.
|
|
180
186
|
*/
|
|
181
187
|
updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
188
|
+
setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
189
|
+
/**
|
|
190
|
+
* List object names for a run by status, optionally filtered to a subset.
|
|
191
|
+
*/
|
|
192
|
+
listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
|
|
182
193
|
/**
|
|
183
194
|
* Get the highest cursor from previous syncs for an object type.
|
|
184
195
|
* Uses only completed object runs.
|
|
@@ -195,6 +206,11 @@ declare class PostgresClient {
|
|
|
195
206
|
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
196
207
|
*/
|
|
197
208
|
getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
209
|
+
/**
|
|
210
|
+
* Get the most recent cursor for an object run before the given run.
|
|
211
|
+
* This returns the raw cursor value without interpretation.
|
|
212
|
+
*/
|
|
213
|
+
getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
198
214
|
/**
|
|
199
215
|
* Delete all sync runs and object runs for an account.
|
|
200
216
|
* Useful for testing or resetting sync state.
|
|
@@ -243,6 +259,12 @@ type SigmaCursorSpec = {
|
|
|
243
259
|
version: 1;
|
|
244
260
|
columns: SigmaCursorColumnSpec[];
|
|
245
261
|
};
|
|
262
|
+
type SigmaColumnSpec = {
|
|
263
|
+
name: string;
|
|
264
|
+
sigmaType: string;
|
|
265
|
+
pgType: string;
|
|
266
|
+
primaryKey: boolean;
|
|
267
|
+
};
|
|
246
268
|
type SigmaIngestionConfig = {
|
|
247
269
|
/**
|
|
248
270
|
* The Sigma table name to query (no quoting, no schema).
|
|
@@ -258,6 +280,11 @@ type SigmaIngestionConfig = {
|
|
|
258
280
|
* Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
|
|
259
281
|
*/
|
|
260
282
|
cursor: SigmaCursorSpec;
|
|
283
|
+
/**
|
|
284
|
+
* Column metadata for the destination table.
|
|
285
|
+
* If provided, used to dynamically create/reconcile the table.
|
|
286
|
+
*/
|
|
287
|
+
columns?: SigmaColumnSpec[];
|
|
261
288
|
/** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
|
|
262
289
|
additionalWhere?: string;
|
|
263
290
|
/** Columns to SELECT (defaults to `*`). */
|
|
@@ -291,6 +318,16 @@ type StripeSyncConfig = {
|
|
|
291
318
|
* Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
|
|
292
319
|
*/
|
|
293
320
|
enableSigma?: boolean;
|
|
321
|
+
/**
|
|
322
|
+
* Optional override for Sigma page size (per query).
|
|
323
|
+
* Useful for edge function CPU limits; lower values reduce per-invocation work.
|
|
324
|
+
*/
|
|
325
|
+
sigmaPageSizeOverride?: number;
|
|
326
|
+
/**
|
|
327
|
+
* Postgres schema name for Sigma tables.
|
|
328
|
+
* Default: "sigma"
|
|
329
|
+
*/
|
|
330
|
+
sigmaSchemaName?: string;
|
|
294
331
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
295
332
|
stripeAccountId?: string;
|
|
296
333
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
@@ -345,7 +382,7 @@ type StripeSyncConfig = {
|
|
|
345
382
|
*/
|
|
346
383
|
maxConcurrentCustomers?: number;
|
|
347
384
|
};
|
|
348
|
-
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | '
|
|
385
|
+
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | '_event_catchup';
|
|
349
386
|
interface Sync {
|
|
350
387
|
synced: number;
|
|
351
388
|
}
|
|
@@ -370,6 +407,8 @@ interface SyncBackfill {
|
|
|
370
407
|
checkoutSessions?: Sync;
|
|
371
408
|
subscriptionItemChangeEventsV2Beta?: Sync;
|
|
372
409
|
exchangeRatesFromUsd?: Sync;
|
|
410
|
+
/** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
|
|
411
|
+
sigma?: Record<string, Sync>;
|
|
373
412
|
}
|
|
374
413
|
interface SyncParams {
|
|
375
414
|
created?: {
|
|
@@ -412,6 +451,8 @@ interface ProcessNextResult {
|
|
|
412
451
|
hasMore: boolean;
|
|
413
452
|
/** The sync run this processing belongs to */
|
|
414
453
|
runStartedAt: Date;
|
|
454
|
+
/** Sigma-only: whether this step started a new Sigma query run */
|
|
455
|
+
startedQuery?: boolean;
|
|
415
456
|
}
|
|
416
457
|
/**
|
|
417
458
|
* Parameters for processNext() including optional run context
|
|
@@ -502,7 +543,12 @@ declare class StripeSync {
|
|
|
502
543
|
private config;
|
|
503
544
|
stripe: Stripe;
|
|
504
545
|
postgresClient: PostgresClient;
|
|
546
|
+
private readonly resourceRegistry;
|
|
547
|
+
private get sigmaSchemaName();
|
|
505
548
|
constructor(config: StripeSyncConfig);
|
|
549
|
+
private buildResourceRegistry;
|
|
550
|
+
private isSigmaResource;
|
|
551
|
+
private sigmaResultKey;
|
|
506
552
|
/**
|
|
507
553
|
* Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
|
|
508
554
|
*/
|
|
@@ -545,7 +591,6 @@ declare class StripeSync {
|
|
|
545
591
|
}>;
|
|
546
592
|
processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
|
|
547
593
|
private readonly eventHandlers;
|
|
548
|
-
private readonly resourceRegistry;
|
|
549
594
|
processEvent(event: Stripe.Event): Promise<void>;
|
|
550
595
|
/**
|
|
551
596
|
* Returns an array of all webhook event types that this sync engine can handle.
|
|
@@ -558,6 +603,13 @@ declare class StripeSync {
|
|
|
558
603
|
* Order is determined by the `order` field in resourceRegistry.
|
|
559
604
|
*/
|
|
560
605
|
getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
606
|
+
/**
|
|
607
|
+
* Get the list of Sigma-backed object types that can be synced.
|
|
608
|
+
* Only returns sigma objects when enableSigma is true.
|
|
609
|
+
*
|
|
610
|
+
* @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
|
|
611
|
+
*/
|
|
612
|
+
getSupportedSigmaObjects(): string[];
|
|
561
613
|
private handleChargeEvent;
|
|
562
614
|
private handleCustomerDeletedEvent;
|
|
563
615
|
private handleCustomerEvent;
|
|
@@ -620,6 +672,45 @@ declare class StripeSync {
|
|
|
620
672
|
private fetchOnePage;
|
|
621
673
|
private getSigmaFallbackCursorFromDestination;
|
|
622
674
|
private fetchOneSigmaPage;
|
|
675
|
+
/**
|
|
676
|
+
* Fetch payment methods by iterating customers in batches.
|
|
677
|
+
*
|
|
678
|
+
* Most customers have 1-2 PMs, so processing one customer per processNext call
|
|
679
|
+
* would be extremely wasteful (10,000 customers = 10,000 worker round-trips).
|
|
680
|
+
* Instead, each call fetches a batch of customers (sized by maxConcurrentCustomers,
|
|
681
|
+
* default 10) and lists+upserts PMs for all of them concurrently.
|
|
682
|
+
*
|
|
683
|
+
* Cursor semantics:
|
|
684
|
+
* - cursor: max customer `created` timestamp — filters to only new customers on subsequent runs
|
|
685
|
+
* - pageCursor: last processed customer ID — tracks position in customer iteration
|
|
686
|
+
*/
|
|
687
|
+
private fetchOnePagePaymentMethods;
|
|
688
|
+
/**
|
|
689
|
+
* Fetch the next batch of non-deleted customers for PM sync.
|
|
690
|
+
* Returns customer IDs and created timestamps, ordered by id ASC.
|
|
691
|
+
*/
|
|
692
|
+
private findNextCustomerBatchForPmSync;
|
|
693
|
+
/**
|
|
694
|
+
* Fetch one page of events from the Stripe Events API and reconcile affected entities.
|
|
695
|
+
*
|
|
696
|
+
* Instead of replaying events (which can resurrect deleted objects due to newest-first ordering),
|
|
697
|
+
* we deduplicate by entity and re-fetch current state from Stripe for each affected entity.
|
|
698
|
+
*
|
|
699
|
+
* Cursor: event `created` timestamp. On first run, starts from the sync run's startedAt.
|
|
700
|
+
* On subsequent runs, picks up where the last completed run left off.
|
|
701
|
+
*/
|
|
702
|
+
private fetchOnePageEventCatchup;
|
|
703
|
+
/**
|
|
704
|
+
* Handle a delete for an entity discovered via event catch-up.
|
|
705
|
+
* Maps Stripe object types to the appropriate delete method.
|
|
706
|
+
*/
|
|
707
|
+
private handleEventCatchupDelete;
|
|
708
|
+
/**
|
|
709
|
+
* Handle an upsert for an entity discovered via event catch-up.
|
|
710
|
+
* Re-fetches the current state from Stripe and upserts it.
|
|
711
|
+
* If the entity has been deleted (404), falls back to the delete handler.
|
|
712
|
+
*/
|
|
713
|
+
private handleEventCatchupUpsert;
|
|
623
714
|
/**
|
|
624
715
|
* Process all pages for all (or specified) object types until complete.
|
|
625
716
|
*
|
|
@@ -651,6 +742,22 @@ declare class StripeSync {
|
|
|
651
742
|
runKey: RunKey;
|
|
652
743
|
objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
653
744
|
}>;
|
|
745
|
+
private applySyncBackfillResult;
|
|
746
|
+
processUntilDoneParallel(params?: SyncParams & {
|
|
747
|
+
maxParallel?: number;
|
|
748
|
+
triggeredBy?: string;
|
|
749
|
+
continueOnError?: boolean;
|
|
750
|
+
skipInaccessibleSigmaTables?: boolean;
|
|
751
|
+
}): Promise<{
|
|
752
|
+
results: SyncBackfill;
|
|
753
|
+
totals: Record<string, number>;
|
|
754
|
+
totalSynced: number;
|
|
755
|
+
skipped: string[];
|
|
756
|
+
errors: Array<{
|
|
757
|
+
object: string;
|
|
758
|
+
message: string;
|
|
759
|
+
}>;
|
|
760
|
+
}>;
|
|
654
761
|
processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
|
|
655
762
|
/**
|
|
656
763
|
* Internal implementation of processUntilDone with an existing run.
|
|
@@ -771,6 +878,7 @@ type MigrationConfig = {
|
|
|
771
878
|
databaseUrl: string;
|
|
772
879
|
ssl?: ConnectionOptions;
|
|
773
880
|
logger?: Logger;
|
|
881
|
+
enableSigma?: boolean;
|
|
774
882
|
};
|
|
775
883
|
declare function runMigrations(config: MigrationConfig): Promise<void>;
|
|
776
884
|
|
package/dist/index.d.ts
CHANGED
|
@@ -33,7 +33,7 @@ declare class PostgresClient {
|
|
|
33
33
|
}>(entries: T[], table: string): Promise<T[]>;
|
|
34
34
|
upsertManyWithTimestampProtection<T extends {
|
|
35
35
|
[Key: string]: any;
|
|
36
|
-
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
|
|
36
|
+
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
|
|
37
37
|
private cleanseArrayField;
|
|
38
38
|
findMissingEntries(table: string, ids: string[]): Promise<string[]>;
|
|
39
39
|
upsertAccount(accountData: {
|
|
@@ -102,9 +102,10 @@ declare class PostgresClient {
|
|
|
102
102
|
cancelStaleRuns(accountId: string): Promise<void>;
|
|
103
103
|
/**
|
|
104
104
|
* Get or create a sync run for this account.
|
|
105
|
-
* Returns existing run if one is active, otherwise creates new one.
|
|
105
|
+
* Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
|
|
106
106
|
* Auto-cancels stale runs before checking.
|
|
107
107
|
*
|
|
108
|
+
* @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
|
|
108
109
|
* @returns RunKey with isNew flag, or null if constraint violation (race condition)
|
|
109
110
|
*/
|
|
110
111
|
getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
@@ -114,8 +115,9 @@ declare class PostgresClient {
|
|
|
114
115
|
} | null>;
|
|
115
116
|
/**
|
|
116
117
|
* Get the active sync run for an account (if any).
|
|
118
|
+
* @param triggeredBy - If provided, only returns run matching this triggeredBy value
|
|
117
119
|
*/
|
|
118
|
-
getActiveSyncRun(accountId: string): Promise<{
|
|
120
|
+
getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
119
121
|
accountId: string;
|
|
120
122
|
runStartedAt: Date;
|
|
121
123
|
} | null>;
|
|
@@ -129,6 +131,10 @@ declare class PostgresClient {
|
|
|
129
131
|
maxConcurrent: number;
|
|
130
132
|
closedAt: Date | null;
|
|
131
133
|
} | null>;
|
|
134
|
+
/**
|
|
135
|
+
* Ensure a sync run has at least the requested max_concurrent value.
|
|
136
|
+
*/
|
|
137
|
+
ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
|
|
132
138
|
/**
|
|
133
139
|
* Close a sync run (mark as done).
|
|
134
140
|
* Status (complete/error) is derived from object run states.
|
|
@@ -179,6 +185,11 @@ declare class PostgresClient {
|
|
|
179
185
|
* For non-numeric cursors, just sets the value directly.
|
|
180
186
|
*/
|
|
181
187
|
updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
188
|
+
setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
189
|
+
/**
|
|
190
|
+
* List object names for a run by status, optionally filtered to a subset.
|
|
191
|
+
*/
|
|
192
|
+
listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
|
|
182
193
|
/**
|
|
183
194
|
* Get the highest cursor from previous syncs for an object type.
|
|
184
195
|
* Uses only completed object runs.
|
|
@@ -195,6 +206,11 @@ declare class PostgresClient {
|
|
|
195
206
|
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
196
207
|
*/
|
|
197
208
|
getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
209
|
+
/**
|
|
210
|
+
* Get the most recent cursor for an object run before the given run.
|
|
211
|
+
* This returns the raw cursor value without interpretation.
|
|
212
|
+
*/
|
|
213
|
+
getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
198
214
|
/**
|
|
199
215
|
* Delete all sync runs and object runs for an account.
|
|
200
216
|
* Useful for testing or resetting sync state.
|
|
@@ -243,6 +259,12 @@ type SigmaCursorSpec = {
|
|
|
243
259
|
version: 1;
|
|
244
260
|
columns: SigmaCursorColumnSpec[];
|
|
245
261
|
};
|
|
262
|
+
type SigmaColumnSpec = {
|
|
263
|
+
name: string;
|
|
264
|
+
sigmaType: string;
|
|
265
|
+
pgType: string;
|
|
266
|
+
primaryKey: boolean;
|
|
267
|
+
};
|
|
246
268
|
type SigmaIngestionConfig = {
|
|
247
269
|
/**
|
|
248
270
|
* The Sigma table name to query (no quoting, no schema).
|
|
@@ -258,6 +280,11 @@ type SigmaIngestionConfig = {
|
|
|
258
280
|
* Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
|
|
259
281
|
*/
|
|
260
282
|
cursor: SigmaCursorSpec;
|
|
283
|
+
/**
|
|
284
|
+
* Column metadata for the destination table.
|
|
285
|
+
* If provided, used to dynamically create/reconcile the table.
|
|
286
|
+
*/
|
|
287
|
+
columns?: SigmaColumnSpec[];
|
|
261
288
|
/** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
|
|
262
289
|
additionalWhere?: string;
|
|
263
290
|
/** Columns to SELECT (defaults to `*`). */
|
|
@@ -291,6 +318,16 @@ type StripeSyncConfig = {
|
|
|
291
318
|
* Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
|
|
292
319
|
*/
|
|
293
320
|
enableSigma?: boolean;
|
|
321
|
+
/**
|
|
322
|
+
* Optional override for Sigma page size (per query).
|
|
323
|
+
* Useful for edge function CPU limits; lower values reduce per-invocation work.
|
|
324
|
+
*/
|
|
325
|
+
sigmaPageSizeOverride?: number;
|
|
326
|
+
/**
|
|
327
|
+
* Postgres schema name for Sigma tables.
|
|
328
|
+
* Default: "sigma"
|
|
329
|
+
*/
|
|
330
|
+
sigmaSchemaName?: string;
|
|
294
331
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
295
332
|
stripeAccountId?: string;
|
|
296
333
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
@@ -345,7 +382,7 @@ type StripeSyncConfig = {
|
|
|
345
382
|
*/
|
|
346
383
|
maxConcurrentCustomers?: number;
|
|
347
384
|
};
|
|
348
|
-
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | '
|
|
385
|
+
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | '_event_catchup';
|
|
349
386
|
interface Sync {
|
|
350
387
|
synced: number;
|
|
351
388
|
}
|
|
@@ -370,6 +407,8 @@ interface SyncBackfill {
|
|
|
370
407
|
checkoutSessions?: Sync;
|
|
371
408
|
subscriptionItemChangeEventsV2Beta?: Sync;
|
|
372
409
|
exchangeRatesFromUsd?: Sync;
|
|
410
|
+
/** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
|
|
411
|
+
sigma?: Record<string, Sync>;
|
|
373
412
|
}
|
|
374
413
|
interface SyncParams {
|
|
375
414
|
created?: {
|
|
@@ -412,6 +451,8 @@ interface ProcessNextResult {
|
|
|
412
451
|
hasMore: boolean;
|
|
413
452
|
/** The sync run this processing belongs to */
|
|
414
453
|
runStartedAt: Date;
|
|
454
|
+
/** Sigma-only: whether this step started a new Sigma query run */
|
|
455
|
+
startedQuery?: boolean;
|
|
415
456
|
}
|
|
416
457
|
/**
|
|
417
458
|
* Parameters for processNext() including optional run context
|
|
@@ -502,7 +543,12 @@ declare class StripeSync {
|
|
|
502
543
|
private config;
|
|
503
544
|
stripe: Stripe;
|
|
504
545
|
postgresClient: PostgresClient;
|
|
546
|
+
private readonly resourceRegistry;
|
|
547
|
+
private get sigmaSchemaName();
|
|
505
548
|
constructor(config: StripeSyncConfig);
|
|
549
|
+
private buildResourceRegistry;
|
|
550
|
+
private isSigmaResource;
|
|
551
|
+
private sigmaResultKey;
|
|
506
552
|
/**
|
|
507
553
|
* Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
|
|
508
554
|
*/
|
|
@@ -545,7 +591,6 @@ declare class StripeSync {
|
|
|
545
591
|
}>;
|
|
546
592
|
processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
|
|
547
593
|
private readonly eventHandlers;
|
|
548
|
-
private readonly resourceRegistry;
|
|
549
594
|
processEvent(event: Stripe.Event): Promise<void>;
|
|
550
595
|
/**
|
|
551
596
|
* Returns an array of all webhook event types that this sync engine can handle.
|
|
@@ -558,6 +603,13 @@ declare class StripeSync {
|
|
|
558
603
|
* Order is determined by the `order` field in resourceRegistry.
|
|
559
604
|
*/
|
|
560
605
|
getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
606
|
+
/**
|
|
607
|
+
* Get the list of Sigma-backed object types that can be synced.
|
|
608
|
+
* Only returns sigma objects when enableSigma is true.
|
|
609
|
+
*
|
|
610
|
+
* @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
|
|
611
|
+
*/
|
|
612
|
+
getSupportedSigmaObjects(): string[];
|
|
561
613
|
private handleChargeEvent;
|
|
562
614
|
private handleCustomerDeletedEvent;
|
|
563
615
|
private handleCustomerEvent;
|
|
@@ -620,6 +672,45 @@ declare class StripeSync {
|
|
|
620
672
|
private fetchOnePage;
|
|
621
673
|
private getSigmaFallbackCursorFromDestination;
|
|
622
674
|
private fetchOneSigmaPage;
|
|
675
|
+
/**
|
|
676
|
+
* Fetch payment methods by iterating customers in batches.
|
|
677
|
+
*
|
|
678
|
+
* Most customers have 1-2 PMs, so processing one customer per processNext call
|
|
679
|
+
* would be extremely wasteful (10,000 customers = 10,000 worker round-trips).
|
|
680
|
+
* Instead, each call fetches a batch of customers (sized by maxConcurrentCustomers,
|
|
681
|
+
* default 10) and lists+upserts PMs for all of them concurrently.
|
|
682
|
+
*
|
|
683
|
+
* Cursor semantics:
|
|
684
|
+
* - cursor: max customer `created` timestamp — filters to only new customers on subsequent runs
|
|
685
|
+
* - pageCursor: last processed customer ID — tracks position in customer iteration
|
|
686
|
+
*/
|
|
687
|
+
private fetchOnePagePaymentMethods;
|
|
688
|
+
/**
|
|
689
|
+
* Fetch the next batch of non-deleted customers for PM sync.
|
|
690
|
+
* Returns customer IDs and created timestamps, ordered by id ASC.
|
|
691
|
+
*/
|
|
692
|
+
private findNextCustomerBatchForPmSync;
|
|
693
|
+
/**
|
|
694
|
+
* Fetch one page of events from the Stripe Events API and reconcile affected entities.
|
|
695
|
+
*
|
|
696
|
+
* Instead of replaying events (which can resurrect deleted objects due to newest-first ordering),
|
|
697
|
+
* we deduplicate by entity and re-fetch current state from Stripe for each affected entity.
|
|
698
|
+
*
|
|
699
|
+
* Cursor: event `created` timestamp. On first run, starts from the sync run's startedAt.
|
|
700
|
+
* On subsequent runs, picks up where the last completed run left off.
|
|
701
|
+
*/
|
|
702
|
+
private fetchOnePageEventCatchup;
|
|
703
|
+
/**
|
|
704
|
+
* Handle a delete for an entity discovered via event catch-up.
|
|
705
|
+
* Maps Stripe object types to the appropriate delete method.
|
|
706
|
+
*/
|
|
707
|
+
private handleEventCatchupDelete;
|
|
708
|
+
/**
|
|
709
|
+
* Handle an upsert for an entity discovered via event catch-up.
|
|
710
|
+
* Re-fetches the current state from Stripe and upserts it.
|
|
711
|
+
* If the entity has been deleted (404), falls back to the delete handler.
|
|
712
|
+
*/
|
|
713
|
+
private handleEventCatchupUpsert;
|
|
623
714
|
/**
|
|
624
715
|
* Process all pages for all (or specified) object types until complete.
|
|
625
716
|
*
|
|
@@ -651,6 +742,22 @@ declare class StripeSync {
|
|
|
651
742
|
runKey: RunKey;
|
|
652
743
|
objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
653
744
|
}>;
|
|
745
|
+
private applySyncBackfillResult;
|
|
746
|
+
processUntilDoneParallel(params?: SyncParams & {
|
|
747
|
+
maxParallel?: number;
|
|
748
|
+
triggeredBy?: string;
|
|
749
|
+
continueOnError?: boolean;
|
|
750
|
+
skipInaccessibleSigmaTables?: boolean;
|
|
751
|
+
}): Promise<{
|
|
752
|
+
results: SyncBackfill;
|
|
753
|
+
totals: Record<string, number>;
|
|
754
|
+
totalSynced: number;
|
|
755
|
+
skipped: string[];
|
|
756
|
+
errors: Array<{
|
|
757
|
+
object: string;
|
|
758
|
+
message: string;
|
|
759
|
+
}>;
|
|
760
|
+
}>;
|
|
654
761
|
processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
|
|
655
762
|
/**
|
|
656
763
|
* Internal implementation of processUntilDone with an existing run.
|
|
@@ -771,6 +878,7 @@ type MigrationConfig = {
|
|
|
771
878
|
databaseUrl: string;
|
|
772
879
|
ssl?: ConnectionOptions;
|
|
773
880
|
logger?: Logger;
|
|
881
|
+
enableSigma?: boolean;
|
|
774
882
|
};
|
|
775
883
|
declare function runMigrations(config: MigrationConfig): Promise<void>;
|
|
776
884
|
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Allow parallel sync runs per triggered_by (sigma-worker vs stripe-worker)
|
|
2
|
+
ALTER TABLE "stripe"."_sync_runs" DROP CONSTRAINT IF EXISTS one_active_run_per_account;
|
|
3
|
+
ALTER TABLE "stripe"."_sync_runs"
|
|
4
|
+
ADD CONSTRAINT one_active_run_per_account_triggered_by
|
|
5
|
+
EXCLUDE (
|
|
6
|
+
"_account_id" WITH =,
|
|
7
|
+
COALESCE(triggered_by, 'default') WITH =
|
|
8
|
+
) WHERE (closed_at IS NULL);
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
-- Add indexes on _account_id for all entity tables
|
|
2
|
+
-- This dramatically improves query performance for multi-tenant queries
|
|
3
|
+
-- like COUNT(*) and MAX(_last_synced_at) filtered by account
|
|
4
|
+
--
|
|
5
|
+
-- CRITICAL: Foreign keys do NOT automatically create indexes in PostgreSQL.
|
|
6
|
+
-- Without these indexes, queries filtering by _account_id do full table scans.
|
|
7
|
+
--
|
|
8
|
+
-- NOTE: Using regular CREATE INDEX (not CONCURRENTLY) because:
|
|
9
|
+
-- 1. pg-node-migrations runs migrations in transactions
|
|
10
|
+
-- 2. CREATE INDEX CONCURRENTLY cannot run inside a transaction
|
|
11
|
+
-- 3. For new deployments, tables are empty so this is instant
|
|
12
|
+
-- 4. For existing deployments, brief lock is acceptable during maintenance window
|
|
13
|
+
|
|
14
|
+
-- Products (catalog)
|
|
15
|
+
CREATE INDEX IF NOT EXISTS idx_products_account_id
|
|
16
|
+
ON stripe.products (_account_id);
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_products_account_last_synced
|
|
18
|
+
ON stripe.products (_account_id, _last_synced_at DESC);
|
|
19
|
+
|
|
20
|
+
-- Prices (catalog)
|
|
21
|
+
CREATE INDEX IF NOT EXISTS idx_prices_account_id
|
|
22
|
+
ON stripe.prices (_account_id);
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_prices_account_last_synced
|
|
24
|
+
ON stripe.prices (_account_id, _last_synced_at DESC);
|
|
25
|
+
|
|
26
|
+
-- Plans (catalog)
|
|
27
|
+
CREATE INDEX IF NOT EXISTS idx_plans_account_id
|
|
28
|
+
ON stripe.plans (_account_id);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_plans_account_last_synced
|
|
30
|
+
ON stripe.plans (_account_id, _last_synced_at DESC);
|
|
31
|
+
|
|
32
|
+
-- Customers
|
|
33
|
+
CREATE INDEX IF NOT EXISTS idx_customers_account_id
|
|
34
|
+
ON stripe.customers (_account_id);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS idx_customers_account_last_synced
|
|
36
|
+
ON stripe.customers (_account_id, _last_synced_at DESC);
|
|
37
|
+
|
|
38
|
+
-- Subscriptions
|
|
39
|
+
CREATE INDEX IF NOT EXISTS idx_subscriptions_account_id
|
|
40
|
+
ON stripe.subscriptions (_account_id);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_subscriptions_account_last_synced
|
|
42
|
+
ON stripe.subscriptions (_account_id, _last_synced_at DESC);
|
|
43
|
+
|
|
44
|
+
-- Subscription Schedules
|
|
45
|
+
CREATE INDEX IF NOT EXISTS idx_subscription_schedules_account_id
|
|
46
|
+
ON stripe.subscription_schedules (_account_id);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_subscription_schedules_account_last_synced
|
|
48
|
+
ON stripe.subscription_schedules (_account_id, _last_synced_at DESC);
|
|
49
|
+
|
|
50
|
+
-- Invoices
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_invoices_account_id
|
|
52
|
+
ON stripe.invoices (_account_id);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_invoices_account_last_synced
|
|
54
|
+
ON stripe.invoices (_account_id, _last_synced_at DESC);
|
|
55
|
+
|
|
56
|
+
-- Credit Notes
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_credit_notes_account_id
|
|
58
|
+
ON stripe.credit_notes (_account_id);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_credit_notes_account_last_synced
|
|
60
|
+
ON stripe.credit_notes (_account_id, _last_synced_at DESC);
|
|
61
|
+
|
|
62
|
+
-- Balance Transactions
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_balance_transactions_account_id
|
|
64
|
+
ON stripe.balance_transactions (_account_id);
|
|
65
|
+
CREATE INDEX IF NOT EXISTS idx_balance_transactions_account_last_synced
|
|
66
|
+
ON stripe.balance_transactions (_account_id, _last_synced_at DESC);
|
|
67
|
+
|
|
68
|
+
-- Charges
|
|
69
|
+
CREATE INDEX IF NOT EXISTS idx_charges_account_id
|
|
70
|
+
ON stripe.charges (_account_id);
|
|
71
|
+
CREATE INDEX IF NOT EXISTS idx_charges_account_last_synced
|
|
72
|
+
ON stripe.charges (_account_id, _last_synced_at DESC);
|
|
73
|
+
|
|
74
|
+
-- Payment Intents
|
|
75
|
+
CREATE INDEX IF NOT EXISTS idx_payment_intents_account_id
|
|
76
|
+
ON stripe.payment_intents (_account_id);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_payment_intents_account_last_synced
|
|
78
|
+
ON stripe.payment_intents (_account_id, _last_synced_at DESC);
|
|
79
|
+
|
|
80
|
+
-- Payment Methods
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_payment_methods_account_id
|
|
82
|
+
ON stripe.payment_methods (_account_id);
|
|
83
|
+
CREATE INDEX IF NOT EXISTS idx_payment_methods_account_last_synced
|
|
84
|
+
ON stripe.payment_methods (_account_id, _last_synced_at DESC);
|
|
85
|
+
|
|
86
|
+
-- Setup Intents
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_setup_intents_account_id
|
|
88
|
+
ON stripe.setup_intents (_account_id);
|
|
89
|
+
CREATE INDEX IF NOT EXISTS idx_setup_intents_account_last_synced
|
|
90
|
+
ON stripe.setup_intents (_account_id, _last_synced_at DESC);
|
|
91
|
+
|
|
92
|
+
-- Refunds
|
|
93
|
+
CREATE INDEX IF NOT EXISTS idx_refunds_account_id
|
|
94
|
+
ON stripe.refunds (_account_id);
|
|
95
|
+
CREATE INDEX IF NOT EXISTS idx_refunds_account_last_synced
|
|
96
|
+
ON stripe.refunds (_account_id, _last_synced_at DESC);
|
|
97
|
+
|
|
98
|
+
-- Checkout Sessions
|
|
99
|
+
CREATE INDEX IF NOT EXISTS idx_checkout_sessions_account_id
|
|
100
|
+
ON stripe.checkout_sessions (_account_id);
|
|
101
|
+
CREATE INDEX IF NOT EXISTS idx_checkout_sessions_account_last_synced
|
|
102
|
+
ON stripe.checkout_sessions (_account_id, _last_synced_at DESC);
|
|
103
|
+
|
|
104
|
+
-- Disputes
|
|
105
|
+
CREATE INDEX IF NOT EXISTS idx_disputes_account_id
|
|
106
|
+
ON stripe.disputes (_account_id);
|
|
107
|
+
CREATE INDEX IF NOT EXISTS idx_disputes_account_last_synced
|
|
108
|
+
ON stripe.disputes (_account_id, _last_synced_at DESC);
|
|
109
|
+
|
|
110
|
+
-- Early Fraud Warnings
|
|
111
|
+
CREATE INDEX IF NOT EXISTS idx_early_fraud_warnings_account_id
|
|
112
|
+
ON stripe.early_fraud_warnings (_account_id);
|
|
113
|
+
CREATE INDEX IF NOT EXISTS idx_early_fraud_warnings_account_last_synced
|
|
114
|
+
ON stripe.early_fraud_warnings (_account_id, _last_synced_at DESC);
|
|
115
|
+
|
|
116
|
+
-- Tax IDs
|
|
117
|
+
CREATE INDEX IF NOT EXISTS idx_tax_ids_account_id
|
|
118
|
+
ON stripe.tax_ids (_account_id);
|
|
119
|
+
CREATE INDEX IF NOT EXISTS idx_tax_ids_account_last_synced
|
|
120
|
+
ON stripe.tax_ids (_account_id, _last_synced_at DESC);
|