@paymentsdb/sync-engine 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-FEW6TDVA.js → chunk-DZMKGCU5.js} +95 -28
- package/dist/{chunk-ATMQZPFA.js → chunk-HXSDJSKR.js} +9 -3
- package/dist/chunk-LB4HG4Q6.js +48413 -0
- package/dist/chunk-Q3AGYUHN.js +477 -0
- package/dist/cli/index.cjs +44709 -345
- package/dist/cli/index.js +17 -11
- package/dist/cli/lib.cjs +44696 -338
- package/dist/cli/lib.d.cts +1 -0
- package/dist/cli/lib.d.ts +1 -0
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +44537 -315
- package/dist/index.d.cts +79 -5
- package/dist/index.d.ts +79 -5
- package/dist/index.js +2 -2
- package/dist/migrations/0062_sigma_query_runs.sql +8 -0
- package/dist/migrations/0064_add_account_id_indexes.sql +120 -0
- package/dist/supabase/index.cjs +92 -14
- package/dist/supabase/index.d.cts +9 -2
- package/dist/supabase/index.d.ts +9 -2
- package/dist/supabase/index.js +4 -2
- package/package.json +9 -3
- package/dist/chunk-4DX3CQZ6.js +0 -406
- package/dist/chunk-IMZLSEZY.js +0 -4196
- /package/dist/migrations/{0062_balance_transactions.sql → 0063_balance_transactions.sql} +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -33,7 +33,7 @@ declare class PostgresClient {
|
|
|
33
33
|
}>(entries: T[], table: string): Promise<T[]>;
|
|
34
34
|
upsertManyWithTimestampProtection<T extends {
|
|
35
35
|
[Key: string]: any;
|
|
36
|
-
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
|
|
36
|
+
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
|
|
37
37
|
private cleanseArrayField;
|
|
38
38
|
findMissingEntries(table: string, ids: string[]): Promise<string[]>;
|
|
39
39
|
upsertAccount(accountData: {
|
|
@@ -102,9 +102,10 @@ declare class PostgresClient {
|
|
|
102
102
|
cancelStaleRuns(accountId: string): Promise<void>;
|
|
103
103
|
/**
|
|
104
104
|
* Get or create a sync run for this account.
|
|
105
|
-
* Returns existing run if one is active, otherwise creates new one.
|
|
105
|
+
* Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
|
|
106
106
|
* Auto-cancels stale runs before checking.
|
|
107
107
|
*
|
|
108
|
+
* @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
|
|
108
109
|
* @returns RunKey with isNew flag, or null if constraint violation (race condition)
|
|
109
110
|
*/
|
|
110
111
|
getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
@@ -114,8 +115,9 @@ declare class PostgresClient {
|
|
|
114
115
|
} | null>;
|
|
115
116
|
/**
|
|
116
117
|
* Get the active sync run for an account (if any).
|
|
118
|
+
* @param triggeredBy - If provided, only returns run matching this triggeredBy value
|
|
117
119
|
*/
|
|
118
|
-
getActiveSyncRun(accountId: string): Promise<{
|
|
120
|
+
getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
119
121
|
accountId: string;
|
|
120
122
|
runStartedAt: Date;
|
|
121
123
|
} | null>;
|
|
@@ -129,6 +131,10 @@ declare class PostgresClient {
|
|
|
129
131
|
maxConcurrent: number;
|
|
130
132
|
closedAt: Date | null;
|
|
131
133
|
} | null>;
|
|
134
|
+
/**
|
|
135
|
+
* Ensure a sync run has at least the requested max_concurrent value.
|
|
136
|
+
*/
|
|
137
|
+
ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
|
|
132
138
|
/**
|
|
133
139
|
* Close a sync run (mark as done).
|
|
134
140
|
* Status (complete/error) is derived from object run states.
|
|
@@ -179,6 +185,11 @@ declare class PostgresClient {
|
|
|
179
185
|
* For non-numeric cursors, just sets the value directly.
|
|
180
186
|
*/
|
|
181
187
|
updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
188
|
+
setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
189
|
+
/**
|
|
190
|
+
* List object names for a run by status, optionally filtered to a subset.
|
|
191
|
+
*/
|
|
192
|
+
listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
|
|
182
193
|
/**
|
|
183
194
|
* Get the highest cursor from previous syncs for an object type.
|
|
184
195
|
* Uses only completed object runs.
|
|
@@ -195,6 +206,11 @@ declare class PostgresClient {
|
|
|
195
206
|
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
196
207
|
*/
|
|
197
208
|
getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
209
|
+
/**
|
|
210
|
+
* Get the most recent cursor for an object run before the given run.
|
|
211
|
+
* This returns the raw cursor value without interpretation.
|
|
212
|
+
*/
|
|
213
|
+
getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
198
214
|
/**
|
|
199
215
|
* Delete all sync runs and object runs for an account.
|
|
200
216
|
* Useful for testing or resetting sync state.
|
|
@@ -243,6 +259,12 @@ type SigmaCursorSpec = {
|
|
|
243
259
|
version: 1;
|
|
244
260
|
columns: SigmaCursorColumnSpec[];
|
|
245
261
|
};
|
|
262
|
+
type SigmaColumnSpec = {
|
|
263
|
+
name: string;
|
|
264
|
+
sigmaType: string;
|
|
265
|
+
pgType: string;
|
|
266
|
+
primaryKey: boolean;
|
|
267
|
+
};
|
|
246
268
|
type SigmaIngestionConfig = {
|
|
247
269
|
/**
|
|
248
270
|
* The Sigma table name to query (no quoting, no schema).
|
|
@@ -258,6 +280,11 @@ type SigmaIngestionConfig = {
|
|
|
258
280
|
* Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
|
|
259
281
|
*/
|
|
260
282
|
cursor: SigmaCursorSpec;
|
|
283
|
+
/**
|
|
284
|
+
* Column metadata for the destination table.
|
|
285
|
+
* If provided, used to dynamically create/reconcile the table.
|
|
286
|
+
*/
|
|
287
|
+
columns?: SigmaColumnSpec[];
|
|
261
288
|
/** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
|
|
262
289
|
additionalWhere?: string;
|
|
263
290
|
/** Columns to SELECT (defaults to `*`). */
|
|
@@ -280,12 +307,27 @@ type StripeSyncConfig = {
|
|
|
280
307
|
databaseUrl?: string;
|
|
281
308
|
/** Stripe secret key used to authenticate requests to the Stripe API. Defaults to empty string */
|
|
282
309
|
stripeSecretKey: string;
|
|
310
|
+
/**
|
|
311
|
+
* Application name sent to Stripe in the User-Agent header via appInfo.
|
|
312
|
+
* Default: 'Stripe Sync Engine'
|
|
313
|
+
*/
|
|
314
|
+
appName?: string;
|
|
283
315
|
/**
|
|
284
316
|
* Enables syncing Stripe Sigma (reporting) tables via the Sigma API.
|
|
285
317
|
*
|
|
286
318
|
* Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
|
|
287
319
|
*/
|
|
288
320
|
enableSigma?: boolean;
|
|
321
|
+
/**
|
|
322
|
+
* Optional override for Sigma page size (per query).
|
|
323
|
+
* Useful for edge function CPU limits; lower values reduce per-invocation work.
|
|
324
|
+
*/
|
|
325
|
+
sigmaPageSizeOverride?: number;
|
|
326
|
+
/**
|
|
327
|
+
* Postgres schema name for Sigma tables.
|
|
328
|
+
* Default: "sigma"
|
|
329
|
+
*/
|
|
330
|
+
sigmaSchemaName?: string;
|
|
289
331
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
290
332
|
stripeAccountId?: string;
|
|
291
333
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
@@ -340,7 +382,7 @@ type StripeSyncConfig = {
|
|
|
340
382
|
*/
|
|
341
383
|
maxConcurrentCustomers?: number;
|
|
342
384
|
};
|
|
343
|
-
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions'
|
|
385
|
+
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
|
|
344
386
|
interface Sync {
|
|
345
387
|
synced: number;
|
|
346
388
|
}
|
|
@@ -365,6 +407,8 @@ interface SyncBackfill {
|
|
|
365
407
|
checkoutSessions?: Sync;
|
|
366
408
|
subscriptionItemChangeEventsV2Beta?: Sync;
|
|
367
409
|
exchangeRatesFromUsd?: Sync;
|
|
410
|
+
/** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
|
|
411
|
+
sigma?: Record<string, Sync>;
|
|
368
412
|
}
|
|
369
413
|
interface SyncParams {
|
|
370
414
|
created?: {
|
|
@@ -407,6 +451,8 @@ interface ProcessNextResult {
|
|
|
407
451
|
hasMore: boolean;
|
|
408
452
|
/** The sync run this processing belongs to */
|
|
409
453
|
runStartedAt: Date;
|
|
454
|
+
/** Sigma-only: whether this step started a new Sigma query run */
|
|
455
|
+
startedQuery?: boolean;
|
|
410
456
|
}
|
|
411
457
|
/**
|
|
412
458
|
* Parameters for processNext() including optional run context
|
|
@@ -497,7 +543,12 @@ declare class StripeSync {
|
|
|
497
543
|
private config;
|
|
498
544
|
stripe: Stripe;
|
|
499
545
|
postgresClient: PostgresClient;
|
|
546
|
+
private readonly resourceRegistry;
|
|
547
|
+
private get sigmaSchemaName();
|
|
500
548
|
constructor(config: StripeSyncConfig);
|
|
549
|
+
private buildResourceRegistry;
|
|
550
|
+
private isSigmaResource;
|
|
551
|
+
private sigmaResultKey;
|
|
501
552
|
/**
|
|
502
553
|
* Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
|
|
503
554
|
*/
|
|
@@ -540,7 +591,6 @@ declare class StripeSync {
|
|
|
540
591
|
}>;
|
|
541
592
|
processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
|
|
542
593
|
private readonly eventHandlers;
|
|
543
|
-
private readonly resourceRegistry;
|
|
544
594
|
processEvent(event: Stripe.Event): Promise<void>;
|
|
545
595
|
/**
|
|
546
596
|
* Returns an array of all webhook event types that this sync engine can handle.
|
|
@@ -553,6 +603,13 @@ declare class StripeSync {
|
|
|
553
603
|
* Order is determined by the `order` field in resourceRegistry.
|
|
554
604
|
*/
|
|
555
605
|
getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
606
|
+
/**
|
|
607
|
+
* Get the list of Sigma-backed object types that can be synced.
|
|
608
|
+
* Only returns sigma objects when enableSigma is true.
|
|
609
|
+
*
|
|
610
|
+
* @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
|
|
611
|
+
*/
|
|
612
|
+
getSupportedSigmaObjects(): string[];
|
|
556
613
|
private handleChargeEvent;
|
|
557
614
|
private handleCustomerDeletedEvent;
|
|
558
615
|
private handleCustomerEvent;
|
|
@@ -646,6 +703,22 @@ declare class StripeSync {
|
|
|
646
703
|
runKey: RunKey;
|
|
647
704
|
objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
648
705
|
}>;
|
|
706
|
+
private applySyncBackfillResult;
|
|
707
|
+
processUntilDoneParallel(params?: SyncParams & {
|
|
708
|
+
maxParallel?: number;
|
|
709
|
+
triggeredBy?: string;
|
|
710
|
+
continueOnError?: boolean;
|
|
711
|
+
skipInaccessibleSigmaTables?: boolean;
|
|
712
|
+
}): Promise<{
|
|
713
|
+
results: SyncBackfill;
|
|
714
|
+
totals: Record<string, number>;
|
|
715
|
+
totalSynced: number;
|
|
716
|
+
skipped: string[];
|
|
717
|
+
errors: Array<{
|
|
718
|
+
object: string;
|
|
719
|
+
message: string;
|
|
720
|
+
}>;
|
|
721
|
+
}>;
|
|
649
722
|
processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
|
|
650
723
|
/**
|
|
651
724
|
* Internal implementation of processUntilDone with an existing run.
|
|
@@ -766,6 +839,7 @@ type MigrationConfig = {
|
|
|
766
839
|
databaseUrl: string;
|
|
767
840
|
ssl?: ConnectionOptions;
|
|
768
841
|
logger?: Logger;
|
|
842
|
+
enableSigma?: boolean;
|
|
769
843
|
};
|
|
770
844
|
declare function runMigrations(config: MigrationConfig): Promise<void>;
|
|
771
845
|
|
package/dist/index.d.ts
CHANGED
|
@@ -33,7 +33,7 @@ declare class PostgresClient {
|
|
|
33
33
|
}>(entries: T[], table: string): Promise<T[]>;
|
|
34
34
|
upsertManyWithTimestampProtection<T extends {
|
|
35
35
|
[Key: string]: any;
|
|
36
|
-
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
|
|
36
|
+
}>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
|
|
37
37
|
private cleanseArrayField;
|
|
38
38
|
findMissingEntries(table: string, ids: string[]): Promise<string[]>;
|
|
39
39
|
upsertAccount(accountData: {
|
|
@@ -102,9 +102,10 @@ declare class PostgresClient {
|
|
|
102
102
|
cancelStaleRuns(accountId: string): Promise<void>;
|
|
103
103
|
/**
|
|
104
104
|
* Get or create a sync run for this account.
|
|
105
|
-
* Returns existing run if one is active, otherwise creates new one.
|
|
105
|
+
* Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
|
|
106
106
|
* Auto-cancels stale runs before checking.
|
|
107
107
|
*
|
|
108
|
+
* @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
|
|
108
109
|
* @returns RunKey with isNew flag, or null if constraint violation (race condition)
|
|
109
110
|
*/
|
|
110
111
|
getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
@@ -114,8 +115,9 @@ declare class PostgresClient {
|
|
|
114
115
|
} | null>;
|
|
115
116
|
/**
|
|
116
117
|
* Get the active sync run for an account (if any).
|
|
118
|
+
* @param triggeredBy - If provided, only returns run matching this triggeredBy value
|
|
117
119
|
*/
|
|
118
|
-
getActiveSyncRun(accountId: string): Promise<{
|
|
120
|
+
getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
|
|
119
121
|
accountId: string;
|
|
120
122
|
runStartedAt: Date;
|
|
121
123
|
} | null>;
|
|
@@ -129,6 +131,10 @@ declare class PostgresClient {
|
|
|
129
131
|
maxConcurrent: number;
|
|
130
132
|
closedAt: Date | null;
|
|
131
133
|
} | null>;
|
|
134
|
+
/**
|
|
135
|
+
* Ensure a sync run has at least the requested max_concurrent value.
|
|
136
|
+
*/
|
|
137
|
+
ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
|
|
132
138
|
/**
|
|
133
139
|
* Close a sync run (mark as done).
|
|
134
140
|
* Status (complete/error) is derived from object run states.
|
|
@@ -179,6 +185,11 @@ declare class PostgresClient {
|
|
|
179
185
|
* For non-numeric cursors, just sets the value directly.
|
|
180
186
|
*/
|
|
181
187
|
updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
188
|
+
setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
189
|
+
/**
|
|
190
|
+
* List object names for a run by status, optionally filtered to a subset.
|
|
191
|
+
*/
|
|
192
|
+
listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
|
|
182
193
|
/**
|
|
183
194
|
* Get the highest cursor from previous syncs for an object type.
|
|
184
195
|
* Uses only completed object runs.
|
|
@@ -195,6 +206,11 @@ declare class PostgresClient {
|
|
|
195
206
|
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
196
207
|
*/
|
|
197
208
|
getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
209
|
+
/**
|
|
210
|
+
* Get the most recent cursor for an object run before the given run.
|
|
211
|
+
* This returns the raw cursor value without interpretation.
|
|
212
|
+
*/
|
|
213
|
+
getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
198
214
|
/**
|
|
199
215
|
* Delete all sync runs and object runs for an account.
|
|
200
216
|
* Useful for testing or resetting sync state.
|
|
@@ -243,6 +259,12 @@ type SigmaCursorSpec = {
|
|
|
243
259
|
version: 1;
|
|
244
260
|
columns: SigmaCursorColumnSpec[];
|
|
245
261
|
};
|
|
262
|
+
type SigmaColumnSpec = {
|
|
263
|
+
name: string;
|
|
264
|
+
sigmaType: string;
|
|
265
|
+
pgType: string;
|
|
266
|
+
primaryKey: boolean;
|
|
267
|
+
};
|
|
246
268
|
type SigmaIngestionConfig = {
|
|
247
269
|
/**
|
|
248
270
|
* The Sigma table name to query (no quoting, no schema).
|
|
@@ -258,6 +280,11 @@ type SigmaIngestionConfig = {
|
|
|
258
280
|
* Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
|
|
259
281
|
*/
|
|
260
282
|
cursor: SigmaCursorSpec;
|
|
283
|
+
/**
|
|
284
|
+
* Column metadata for the destination table.
|
|
285
|
+
* If provided, used to dynamically create/reconcile the table.
|
|
286
|
+
*/
|
|
287
|
+
columns?: SigmaColumnSpec[];
|
|
261
288
|
/** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
|
|
262
289
|
additionalWhere?: string;
|
|
263
290
|
/** Columns to SELECT (defaults to `*`). */
|
|
@@ -280,12 +307,27 @@ type StripeSyncConfig = {
|
|
|
280
307
|
databaseUrl?: string;
|
|
281
308
|
/** Stripe secret key used to authenticate requests to the Stripe API. Defaults to empty string */
|
|
282
309
|
stripeSecretKey: string;
|
|
310
|
+
/**
|
|
311
|
+
* Application name sent to Stripe in the User-Agent header via appInfo.
|
|
312
|
+
* Default: 'Stripe Sync Engine'
|
|
313
|
+
*/
|
|
314
|
+
appName?: string;
|
|
283
315
|
/**
|
|
284
316
|
* Enables syncing Stripe Sigma (reporting) tables via the Sigma API.
|
|
285
317
|
*
|
|
286
318
|
* Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
|
|
287
319
|
*/
|
|
288
320
|
enableSigma?: boolean;
|
|
321
|
+
/**
|
|
322
|
+
* Optional override for Sigma page size (per query).
|
|
323
|
+
* Useful for edge function CPU limits; lower values reduce per-invocation work.
|
|
324
|
+
*/
|
|
325
|
+
sigmaPageSizeOverride?: number;
|
|
326
|
+
/**
|
|
327
|
+
* Postgres schema name for Sigma tables.
|
|
328
|
+
* Default: "sigma"
|
|
329
|
+
*/
|
|
330
|
+
sigmaSchemaName?: string;
|
|
289
331
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
290
332
|
stripeAccountId?: string;
|
|
291
333
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
@@ -340,7 +382,7 @@ type StripeSyncConfig = {
|
|
|
340
382
|
*/
|
|
341
383
|
maxConcurrentCustomers?: number;
|
|
342
384
|
};
|
|
343
|
-
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions'
|
|
385
|
+
type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'balance_transaction' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
|
|
344
386
|
interface Sync {
|
|
345
387
|
synced: number;
|
|
346
388
|
}
|
|
@@ -365,6 +407,8 @@ interface SyncBackfill {
|
|
|
365
407
|
checkoutSessions?: Sync;
|
|
366
408
|
subscriptionItemChangeEventsV2Beta?: Sync;
|
|
367
409
|
exchangeRatesFromUsd?: Sync;
|
|
410
|
+
/** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
|
|
411
|
+
sigma?: Record<string, Sync>;
|
|
368
412
|
}
|
|
369
413
|
interface SyncParams {
|
|
370
414
|
created?: {
|
|
@@ -407,6 +451,8 @@ interface ProcessNextResult {
|
|
|
407
451
|
hasMore: boolean;
|
|
408
452
|
/** The sync run this processing belongs to */
|
|
409
453
|
runStartedAt: Date;
|
|
454
|
+
/** Sigma-only: whether this step started a new Sigma query run */
|
|
455
|
+
startedQuery?: boolean;
|
|
410
456
|
}
|
|
411
457
|
/**
|
|
412
458
|
* Parameters for processNext() including optional run context
|
|
@@ -497,7 +543,12 @@ declare class StripeSync {
|
|
|
497
543
|
private config;
|
|
498
544
|
stripe: Stripe;
|
|
499
545
|
postgresClient: PostgresClient;
|
|
546
|
+
private readonly resourceRegistry;
|
|
547
|
+
private get sigmaSchemaName();
|
|
500
548
|
constructor(config: StripeSyncConfig);
|
|
549
|
+
private buildResourceRegistry;
|
|
550
|
+
private isSigmaResource;
|
|
551
|
+
private sigmaResultKey;
|
|
501
552
|
/**
|
|
502
553
|
* Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
|
|
503
554
|
*/
|
|
@@ -540,7 +591,6 @@ declare class StripeSync {
|
|
|
540
591
|
}>;
|
|
541
592
|
processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
|
|
542
593
|
private readonly eventHandlers;
|
|
543
|
-
private readonly resourceRegistry;
|
|
544
594
|
processEvent(event: Stripe.Event): Promise<void>;
|
|
545
595
|
/**
|
|
546
596
|
* Returns an array of all webhook event types that this sync engine can handle.
|
|
@@ -553,6 +603,13 @@ declare class StripeSync {
|
|
|
553
603
|
* Order is determined by the `order` field in resourceRegistry.
|
|
554
604
|
*/
|
|
555
605
|
getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
606
|
+
/**
|
|
607
|
+
* Get the list of Sigma-backed object types that can be synced.
|
|
608
|
+
* Only returns sigma objects when enableSigma is true.
|
|
609
|
+
*
|
|
610
|
+
* @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
|
|
611
|
+
*/
|
|
612
|
+
getSupportedSigmaObjects(): string[];
|
|
556
613
|
private handleChargeEvent;
|
|
557
614
|
private handleCustomerDeletedEvent;
|
|
558
615
|
private handleCustomerEvent;
|
|
@@ -646,6 +703,22 @@ declare class StripeSync {
|
|
|
646
703
|
runKey: RunKey;
|
|
647
704
|
objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
|
|
648
705
|
}>;
|
|
706
|
+
private applySyncBackfillResult;
|
|
707
|
+
processUntilDoneParallel(params?: SyncParams & {
|
|
708
|
+
maxParallel?: number;
|
|
709
|
+
triggeredBy?: string;
|
|
710
|
+
continueOnError?: boolean;
|
|
711
|
+
skipInaccessibleSigmaTables?: boolean;
|
|
712
|
+
}): Promise<{
|
|
713
|
+
results: SyncBackfill;
|
|
714
|
+
totals: Record<string, number>;
|
|
715
|
+
totalSynced: number;
|
|
716
|
+
skipped: string[];
|
|
717
|
+
errors: Array<{
|
|
718
|
+
object: string;
|
|
719
|
+
message: string;
|
|
720
|
+
}>;
|
|
721
|
+
}>;
|
|
649
722
|
processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
|
|
650
723
|
/**
|
|
651
724
|
* Internal implementation of processUntilDone with an existing run.
|
|
@@ -766,6 +839,7 @@ type MigrationConfig = {
|
|
|
766
839
|
databaseUrl: string;
|
|
767
840
|
ssl?: ConnectionOptions;
|
|
768
841
|
logger?: Logger;
|
|
842
|
+
enableSigma?: boolean;
|
|
769
843
|
};
|
|
770
844
|
declare function runMigrations(config: MigrationConfig): Promise<void>;
|
|
771
845
|
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Allow parallel sync runs per triggered_by (sigma-worker vs stripe-worker)
|
|
2
|
+
ALTER TABLE "stripe"."_sync_runs" DROP CONSTRAINT IF EXISTS one_active_run_per_account;
|
|
3
|
+
ALTER TABLE "stripe"."_sync_runs"
|
|
4
|
+
ADD CONSTRAINT one_active_run_per_account_triggered_by
|
|
5
|
+
EXCLUDE (
|
|
6
|
+
"_account_id" WITH =,
|
|
7
|
+
COALESCE(triggered_by, 'default') WITH =
|
|
8
|
+
) WHERE (closed_at IS NULL);
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
-- Add indexes on _account_id for all entity tables
|
|
2
|
+
-- This dramatically improves query performance for multi-tenant queries
|
|
3
|
+
-- like COUNT(*) and MAX(_last_synced_at) filtered by account
|
|
4
|
+
--
|
|
5
|
+
-- CRITICAL: Foreign keys do NOT automatically create indexes in PostgreSQL.
|
|
6
|
+
-- Without these indexes, queries filtering by _account_id do full table scans.
|
|
7
|
+
--
|
|
8
|
+
-- NOTE: Using regular CREATE INDEX (not CONCURRENTLY) because:
|
|
9
|
+
-- 1. pg-node-migrations runs migrations in transactions
|
|
10
|
+
-- 2. CREATE INDEX CONCURRENTLY cannot run inside a transaction
|
|
11
|
+
-- 3. For new deployments, tables are empty so this is instant
|
|
12
|
+
-- 4. For existing deployments, brief lock is acceptable during maintenance window
|
|
13
|
+
|
|
14
|
+
-- Products (catalog)
|
|
15
|
+
CREATE INDEX IF NOT EXISTS idx_products_account_id
|
|
16
|
+
ON stripe.products (_account_id);
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_products_account_last_synced
|
|
18
|
+
ON stripe.products (_account_id, _last_synced_at DESC);
|
|
19
|
+
|
|
20
|
+
-- Prices (catalog)
|
|
21
|
+
CREATE INDEX IF NOT EXISTS idx_prices_account_id
|
|
22
|
+
ON stripe.prices (_account_id);
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_prices_account_last_synced
|
|
24
|
+
ON stripe.prices (_account_id, _last_synced_at DESC);
|
|
25
|
+
|
|
26
|
+
-- Plans (catalog)
|
|
27
|
+
CREATE INDEX IF NOT EXISTS idx_plans_account_id
|
|
28
|
+
ON stripe.plans (_account_id);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_plans_account_last_synced
|
|
30
|
+
ON stripe.plans (_account_id, _last_synced_at DESC);
|
|
31
|
+
|
|
32
|
+
-- Customers
|
|
33
|
+
CREATE INDEX IF NOT EXISTS idx_customers_account_id
|
|
34
|
+
ON stripe.customers (_account_id);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS idx_customers_account_last_synced
|
|
36
|
+
ON stripe.customers (_account_id, _last_synced_at DESC);
|
|
37
|
+
|
|
38
|
+
-- Subscriptions
|
|
39
|
+
CREATE INDEX IF NOT EXISTS idx_subscriptions_account_id
|
|
40
|
+
ON stripe.subscriptions (_account_id);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_subscriptions_account_last_synced
|
|
42
|
+
ON stripe.subscriptions (_account_id, _last_synced_at DESC);
|
|
43
|
+
|
|
44
|
+
-- Subscription Schedules
|
|
45
|
+
CREATE INDEX IF NOT EXISTS idx_subscription_schedules_account_id
|
|
46
|
+
ON stripe.subscription_schedules (_account_id);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_subscription_schedules_account_last_synced
|
|
48
|
+
ON stripe.subscription_schedules (_account_id, _last_synced_at DESC);
|
|
49
|
+
|
|
50
|
+
-- Invoices
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_invoices_account_id
|
|
52
|
+
ON stripe.invoices (_account_id);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_invoices_account_last_synced
|
|
54
|
+
ON stripe.invoices (_account_id, _last_synced_at DESC);
|
|
55
|
+
|
|
56
|
+
-- Credit Notes
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_credit_notes_account_id
|
|
58
|
+
ON stripe.credit_notes (_account_id);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_credit_notes_account_last_synced
|
|
60
|
+
ON stripe.credit_notes (_account_id, _last_synced_at DESC);
|
|
61
|
+
|
|
62
|
+
-- Balance Transactions
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_balance_transactions_account_id
|
|
64
|
+
ON stripe.balance_transactions (_account_id);
|
|
65
|
+
CREATE INDEX IF NOT EXISTS idx_balance_transactions_account_last_synced
|
|
66
|
+
ON stripe.balance_transactions (_account_id, _last_synced_at DESC);
|
|
67
|
+
|
|
68
|
+
-- Charges
|
|
69
|
+
CREATE INDEX IF NOT EXISTS idx_charges_account_id
|
|
70
|
+
ON stripe.charges (_account_id);
|
|
71
|
+
CREATE INDEX IF NOT EXISTS idx_charges_account_last_synced
|
|
72
|
+
ON stripe.charges (_account_id, _last_synced_at DESC);
|
|
73
|
+
|
|
74
|
+
-- Payment Intents
|
|
75
|
+
CREATE INDEX IF NOT EXISTS idx_payment_intents_account_id
|
|
76
|
+
ON stripe.payment_intents (_account_id);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_payment_intents_account_last_synced
|
|
78
|
+
ON stripe.payment_intents (_account_id, _last_synced_at DESC);
|
|
79
|
+
|
|
80
|
+
-- Payment Methods
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_payment_methods_account_id
|
|
82
|
+
ON stripe.payment_methods (_account_id);
|
|
83
|
+
CREATE INDEX IF NOT EXISTS idx_payment_methods_account_last_synced
|
|
84
|
+
ON stripe.payment_methods (_account_id, _last_synced_at DESC);
|
|
85
|
+
|
|
86
|
+
-- Setup Intents
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_setup_intents_account_id
|
|
88
|
+
ON stripe.setup_intents (_account_id);
|
|
89
|
+
CREATE INDEX IF NOT EXISTS idx_setup_intents_account_last_synced
|
|
90
|
+
ON stripe.setup_intents (_account_id, _last_synced_at DESC);
|
|
91
|
+
|
|
92
|
+
-- Refunds
|
|
93
|
+
CREATE INDEX IF NOT EXISTS idx_refunds_account_id
|
|
94
|
+
ON stripe.refunds (_account_id);
|
|
95
|
+
CREATE INDEX IF NOT EXISTS idx_refunds_account_last_synced
|
|
96
|
+
ON stripe.refunds (_account_id, _last_synced_at DESC);
|
|
97
|
+
|
|
98
|
+
-- Checkout Sessions
|
|
99
|
+
CREATE INDEX IF NOT EXISTS idx_checkout_sessions_account_id
|
|
100
|
+
ON stripe.checkout_sessions (_account_id);
|
|
101
|
+
CREATE INDEX IF NOT EXISTS idx_checkout_sessions_account_last_synced
|
|
102
|
+
ON stripe.checkout_sessions (_account_id, _last_synced_at DESC);
|
|
103
|
+
|
|
104
|
+
-- Disputes
|
|
105
|
+
CREATE INDEX IF NOT EXISTS idx_disputes_account_id
|
|
106
|
+
ON stripe.disputes (_account_id);
|
|
107
|
+
CREATE INDEX IF NOT EXISTS idx_disputes_account_last_synced
|
|
108
|
+
ON stripe.disputes (_account_id, _last_synced_at DESC);
|
|
109
|
+
|
|
110
|
+
-- Early Fraud Warnings
|
|
111
|
+
CREATE INDEX IF NOT EXISTS idx_early_fraud_warnings_account_id
|
|
112
|
+
ON stripe.early_fraud_warnings (_account_id);
|
|
113
|
+
CREATE INDEX IF NOT EXISTS idx_early_fraud_warnings_account_last_synced
|
|
114
|
+
ON stripe.early_fraud_warnings (_account_id, _last_synced_at DESC);
|
|
115
|
+
|
|
116
|
+
-- Tax IDs
|
|
117
|
+
CREATE INDEX IF NOT EXISTS idx_tax_ids_account_id
|
|
118
|
+
ON stripe.tax_ids (_account_id);
|
|
119
|
+
CREATE INDEX IF NOT EXISTS idx_tax_ids_account_last_synced
|
|
120
|
+
ON stripe.tax_ids (_account_id, _last_synced_at DESC);
|