@tanstack/electric-db-collection 0.4.8 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/electric.cjs +751 -495
- package/dist/cjs/electric.cjs.map +1 -1
- package/dist/cjs/electric.d.cts +94 -25
- package/dist/cjs/sql-compiler.cjs +47 -95
- package/dist/cjs/sql-compiler.cjs.map +1 -1
- package/dist/esm/electric.d.ts +94 -25
- package/dist/esm/electric.js +753 -497
- package/dist/esm/electric.js.map +1 -1
- package/dist/esm/sql-compiler.js +47 -95
- package/dist/esm/sql-compiler.js.map +1 -1
- package/package.json +4 -3
- package/src/electric.ts +1176 -752
- package/src/sql-compiler.ts +74 -137
package/src/electric.ts
CHANGED
|
@@ -4,13 +4,16 @@ import {
|
|
|
4
4
|
isControlMessage,
|
|
5
5
|
isVisibleInSnapshot,
|
|
6
6
|
} from '@electric-sql/client'
|
|
7
|
-
import { Store } from '@tanstack/store'
|
|
8
7
|
import DebugModule from 'debug'
|
|
9
8
|
import {
|
|
10
9
|
DeduplicatedLoadSubset,
|
|
11
10
|
LoadSubsetOperationAbortedError,
|
|
12
11
|
and,
|
|
12
|
+
validateSyncPersistenceCapability,
|
|
13
|
+
warnOnce,
|
|
13
14
|
withCollectionConfigFactory,
|
|
15
|
+
withCollectionSyncConfigCleanup,
|
|
16
|
+
withCollectionSyncConfigFactory,
|
|
14
17
|
} from '@tanstack/db'
|
|
15
18
|
import {
|
|
16
19
|
ExpectedNumberInAwaitTxIdError,
|
|
@@ -52,7 +55,6 @@ import type {
|
|
|
52
55
|
SyncConfig,
|
|
53
56
|
SyncMode,
|
|
54
57
|
UpdateMutationFnParams,
|
|
55
|
-
UtilsRecord,
|
|
56
58
|
} from '@tanstack/db'
|
|
57
59
|
import type { StandardSchemaV1 } from '@standard-schema/spec'
|
|
58
60
|
import type {
|
|
@@ -70,8 +72,6 @@ export { isChangeMessage, isControlMessage } from '@electric-sql/client'
|
|
|
70
72
|
|
|
71
73
|
const debug = DebugModule.debug(`ts/db:electric`)
|
|
72
74
|
|
|
73
|
-
const FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS = 250
|
|
74
|
-
|
|
75
75
|
/**
|
|
76
76
|
* Symbol for internal test hooks (hidden from public API)
|
|
77
77
|
*/
|
|
@@ -100,6 +100,7 @@ type ElectricResumeState =
|
|
|
100
100
|
handle: string
|
|
101
101
|
shapeId: string
|
|
102
102
|
updatedAt: number
|
|
103
|
+
requiresTagState?: boolean
|
|
103
104
|
}
|
|
104
105
|
| {
|
|
105
106
|
kind: `reset`
|
|
@@ -112,6 +113,47 @@ type ElectricSyncMeta = {
|
|
|
112
113
|
seenTxids: Array<Txid>
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
type ElectricLifecycleEvidence = {
|
|
117
|
+
seenTxids: Set<Txid>
|
|
118
|
+
seenSnapshots: Array<PostgresSnapshot>
|
|
119
|
+
hydratedResumeState?: ElectricResumeState
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
type ElectricPendingMatch<T extends Row<unknown>> = {
|
|
123
|
+
matchFn: (message: Message<T>) => boolean
|
|
124
|
+
resolve: (value: boolean) => void
|
|
125
|
+
reject: (error: Error) => void
|
|
126
|
+
timeoutId: ReturnType<typeof setTimeout>
|
|
127
|
+
matched: boolean
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
type ElectricMatchBuffer<T extends Row<unknown>> = {
|
|
131
|
+
committedMessages: Array<Message<T>>
|
|
132
|
+
pendingMessages: Array<Message<T>>
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function exportElectricSyncMeta(
|
|
136
|
+
evidence: ElectricLifecycleEvidence,
|
|
137
|
+
): ElectricSyncMeta {
|
|
138
|
+
const resume = evidence.hydratedResumeState
|
|
139
|
+
return {
|
|
140
|
+
version: 1,
|
|
141
|
+
...(resume ? { resume } : {}),
|
|
142
|
+
seenTxids: Array.from(evidence.seenTxids).sort((a, b) => a - b),
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function importElectricSyncMeta(
|
|
147
|
+
evidence: ElectricLifecycleEvidence,
|
|
148
|
+
meta: unknown,
|
|
149
|
+
): void {
|
|
150
|
+
const parsed = parseElectricSyncMeta(meta)
|
|
151
|
+
if (!parsed) return
|
|
152
|
+
|
|
153
|
+
evidence.hydratedResumeState = parsed.resume
|
|
154
|
+
evidence.seenTxids = new Set(parsed.seenTxids)
|
|
155
|
+
}
|
|
156
|
+
|
|
115
157
|
function parseElectricResumeState(
|
|
116
158
|
value: unknown,
|
|
117
159
|
): ElectricResumeState | undefined {
|
|
@@ -134,6 +176,9 @@ function parseElectricResumeState(
|
|
|
134
176
|
handle: record.handle,
|
|
135
177
|
shapeId: record.shapeId,
|
|
136
178
|
updatedAt: record.updatedAt,
|
|
179
|
+
...(typeof record.requiresTagState === `boolean` && {
|
|
180
|
+
requiresTagState: record.requiresTagState,
|
|
181
|
+
}),
|
|
137
182
|
}
|
|
138
183
|
}
|
|
139
184
|
|
|
@@ -216,7 +261,21 @@ function getNewestElectricResumeState(
|
|
|
216
261
|
): ElectricResumeState | undefined {
|
|
217
262
|
if (!current) return incoming
|
|
218
263
|
if (!incoming) return current
|
|
219
|
-
|
|
264
|
+
if (incoming.updatedAt > current.updatedAt) return incoming
|
|
265
|
+
if (incoming.updatedAt < current.updatedAt) return current
|
|
266
|
+
|
|
267
|
+
const isSameState =
|
|
268
|
+
current.kind === incoming.kind &&
|
|
269
|
+
(current.kind === `reset` ||
|
|
270
|
+
(incoming.kind === `resume` &&
|
|
271
|
+
current.offset === incoming.offset &&
|
|
272
|
+
current.handle === incoming.handle &&
|
|
273
|
+
current.shapeId === incoming.shapeId))
|
|
274
|
+
|
|
275
|
+
// Equal timestamps have no causal ordering. Preserve an identical state,
|
|
276
|
+
// but collapse every conflict to reset so merge order cannot resurrect an
|
|
277
|
+
// offset that another source has already declared unsafe.
|
|
278
|
+
return isSameState ? current : { kind: `reset`, updatedAt: current.updatedAt }
|
|
220
279
|
}
|
|
221
280
|
|
|
222
281
|
/**
|
|
@@ -230,11 +289,11 @@ export type MatchFunction<T extends Row<unknown>> = (
|
|
|
230
289
|
/**
|
|
231
290
|
* Matching strategies for Electric synchronization
|
|
232
291
|
* Handlers can return:
|
|
233
|
-
* - Txid strategy: { txid: number | number[], timeout?: number } (
|
|
292
|
+
* - Txid strategy: { txid: number | number[], timeout?: number } (deprecated compatibility behavior)
|
|
234
293
|
* - Void (no return value) - mutation completes without waiting
|
|
235
294
|
*
|
|
236
295
|
* The optional timeout property specifies how long to wait for the txid(s) in milliseconds.
|
|
237
|
-
* If not specified, defaults to
|
|
296
|
+
* If not specified, defaults to 15000ms.
|
|
238
297
|
*/
|
|
239
298
|
export type MatchingStrategy = {
|
|
240
299
|
txid: Txid | Array<Txid>
|
|
@@ -283,13 +342,7 @@ export interface ElectricCollectionConfig<
|
|
|
283
342
|
T extends Row<unknown> = Row<unknown>,
|
|
284
343
|
TSchema extends StandardSchemaV1 = never,
|
|
285
344
|
> extends Omit<
|
|
286
|
-
BaseCollectionConfig<
|
|
287
|
-
T,
|
|
288
|
-
string | number,
|
|
289
|
-
TSchema,
|
|
290
|
-
ElectricCollectionUtils<T>,
|
|
291
|
-
any
|
|
292
|
-
>,
|
|
345
|
+
BaseCollectionConfig<T, string | number, TSchema, ElectricCollectionUtils<T>>,
|
|
293
346
|
`onInsert` | `onUpdate` | `onDelete` | `syncMode`
|
|
294
347
|
> {
|
|
295
348
|
/**
|
|
@@ -306,43 +359,82 @@ export interface ElectricCollectionConfig<
|
|
|
306
359
|
|
|
307
360
|
/**
|
|
308
361
|
* Optional asynchronous handler function called before an insert operation
|
|
362
|
+
*
|
|
363
|
+
* **IMPORTANT - Electric Synchronization:**
|
|
364
|
+
* This handler **must not resolve** until synchronization is confirmed.
|
|
365
|
+
* Await one of these synchronization utilities before the handler completes:
|
|
366
|
+
* 1. `await collection.utils.awaitTxId(txid)` (recommended for most cases)
|
|
367
|
+
* 2. `await collection.utils.awaitMatch(fn)` for custom matching logic
|
|
368
|
+
*
|
|
369
|
+
* Simply returning without waiting for sync will drop optimistic state too early, causing UI glitches.
|
|
370
|
+
*
|
|
309
371
|
* @param params Object containing transaction and collection information
|
|
310
|
-
* @returns Promise
|
|
372
|
+
* @returns Promise that should resolve after synchronization is complete
|
|
373
|
+
*
|
|
374
|
+
* **Deprecation notice:** Returning `{ txid }` from handlers is deprecated and will be removed in v1.0.
|
|
375
|
+
* Use `await collection.utils.awaitTxId(txid)` within the handler instead.
|
|
376
|
+
*
|
|
311
377
|
* @example
|
|
312
|
-
* //
|
|
313
|
-
* onInsert: async ({ transaction }) => {
|
|
378
|
+
* // Recommended: Wait for txid to sync
|
|
379
|
+
* onInsert: async ({ transaction, collection }) => {
|
|
314
380
|
* const newItem = transaction.mutations[0].modified
|
|
315
381
|
* const result = await api.todos.create({
|
|
316
382
|
* data: newItem
|
|
317
383
|
* })
|
|
318
|
-
*
|
|
384
|
+
* // Wait for txid to sync before handler completes
|
|
385
|
+
* await collection.utils.awaitTxId(result.txid)
|
|
319
386
|
* }
|
|
320
387
|
*
|
|
321
388
|
* @example
|
|
322
389
|
* // Insert handler with custom timeout
|
|
323
|
-
* onInsert: async ({ transaction }) => {
|
|
390
|
+
* onInsert: async ({ transaction, collection }) => {
|
|
391
|
+
* const newItem = transaction.mutations[0].modified
|
|
392
|
+
* const result = await api.todos.create({
|
|
393
|
+
* data: newItem
|
|
394
|
+
* })
|
|
395
|
+
* // Wait up to 10 seconds for txid
|
|
396
|
+
* await collection.utils.awaitTxId(result.txid, 10000)
|
|
397
|
+
* }
|
|
398
|
+
*
|
|
399
|
+
* @example
|
|
400
|
+
* // Insert handler with timeout error handling
|
|
401
|
+
* onInsert: async ({ transaction, collection }) => {
|
|
324
402
|
* const newItem = transaction.mutations[0].modified
|
|
325
403
|
* const result = await api.todos.create({
|
|
326
404
|
* data: newItem
|
|
327
405
|
* })
|
|
328
|
-
*
|
|
406
|
+
*
|
|
407
|
+
* try {
|
|
408
|
+
* await collection.utils.awaitTxId(result.txid, 5000)
|
|
409
|
+
* } catch (error) {
|
|
410
|
+
* // Decide sync timeout policy:
|
|
411
|
+
* // - Throw to rollback optimistic state
|
|
412
|
+
* // - Catch to keep optimistic state (eventual consistency)
|
|
413
|
+
* // - Schedule background retry
|
|
414
|
+
* console.warn('Sync timeout, keeping optimistic state:', error)
|
|
415
|
+
* // Don't throw - allow optimistic state to persist
|
|
416
|
+
* }
|
|
329
417
|
* }
|
|
330
418
|
*
|
|
331
419
|
* @example
|
|
332
|
-
* // Insert handler with multiple items
|
|
333
|
-
* onInsert: async ({ transaction }) => {
|
|
420
|
+
* // Insert handler with multiple items
|
|
421
|
+
* onInsert: async ({ transaction, collection }) => {
|
|
334
422
|
* const items = transaction.mutations.map(m => m.modified)
|
|
335
423
|
* const results = await Promise.all(
|
|
336
424
|
* items.map(item => api.todos.create({ data: item }))
|
|
337
425
|
* )
|
|
338
|
-
*
|
|
426
|
+
* // Wait for all txids to sync
|
|
427
|
+
* await Promise.all(
|
|
428
|
+
* results.map(r => collection.utils.awaitTxId(r.txid))
|
|
429
|
+
* )
|
|
339
430
|
* }
|
|
340
431
|
*
|
|
341
432
|
* @example
|
|
342
|
-
* // Use awaitMatch utility for custom matching
|
|
433
|
+
* // Alternative: Use awaitMatch utility for custom matching logic
|
|
343
434
|
* onInsert: async ({ transaction, collection }) => {
|
|
344
435
|
* const newItem = transaction.mutations[0].modified
|
|
345
436
|
* await api.todos.create({ data: newItem })
|
|
437
|
+
* // Wait for specific change to appear in sync stream
|
|
346
438
|
* await collection.utils.awaitMatch(
|
|
347
439
|
* (message) => isChangeMessage(message) &&
|
|
348
440
|
* message.headers.operation === 'insert' &&
|
|
@@ -360,24 +452,39 @@ export interface ElectricCollectionConfig<
|
|
|
360
452
|
|
|
361
453
|
/**
|
|
362
454
|
* Optional asynchronous handler function called before an update operation
|
|
455
|
+
*
|
|
456
|
+
* **IMPORTANT - Electric Synchronization:**
|
|
457
|
+
* This handler **must not resolve** until synchronization is confirmed.
|
|
458
|
+
* Await one of these synchronization utilities before the handler completes:
|
|
459
|
+
* 1. `await collection.utils.awaitTxId(txid)` (recommended for most cases)
|
|
460
|
+
* 2. `await collection.utils.awaitMatch(fn)` for custom matching logic
|
|
461
|
+
*
|
|
462
|
+
* Simply returning without waiting for sync will drop optimistic state too early, causing UI glitches.
|
|
463
|
+
*
|
|
363
464
|
* @param params Object containing transaction and collection information
|
|
364
|
-
* @returns Promise
|
|
465
|
+
* @returns Promise that should resolve after synchronization is complete
|
|
466
|
+
*
|
|
467
|
+
* **Deprecation notice:** Returning `{ txid }` from handlers is deprecated and will be removed in v1.0.
|
|
468
|
+
* Use `await collection.utils.awaitTxId(txid)` within the handler instead.
|
|
469
|
+
*
|
|
365
470
|
* @example
|
|
366
|
-
* //
|
|
367
|
-
* onUpdate: async ({ transaction }) => {
|
|
471
|
+
* // Recommended: Wait for txid to sync
|
|
472
|
+
* onUpdate: async ({ transaction, collection }) => {
|
|
368
473
|
* const { original, changes } = transaction.mutations[0]
|
|
369
474
|
* const result = await api.todos.update({
|
|
370
475
|
* where: { id: original.id },
|
|
371
476
|
* data: changes
|
|
372
477
|
* })
|
|
373
|
-
*
|
|
478
|
+
* // Wait for txid to sync before handler completes
|
|
479
|
+
* await collection.utils.awaitTxId(result.txid)
|
|
374
480
|
* }
|
|
375
481
|
*
|
|
376
482
|
* @example
|
|
377
|
-
* // Use awaitMatch utility for custom matching
|
|
483
|
+
* // Alternative: Use awaitMatch utility for custom matching logic
|
|
378
484
|
* onUpdate: async ({ transaction, collection }) => {
|
|
379
485
|
* const { original, changes } = transaction.mutations[0]
|
|
380
486
|
* await api.todos.update({ where: { id: original.id }, data: changes })
|
|
487
|
+
* // Wait for specific change to appear in sync stream
|
|
381
488
|
* await collection.utils.awaitMatch(
|
|
382
489
|
* (message) => isChangeMessage(message) &&
|
|
383
490
|
* message.headers.operation === 'update' &&
|
|
@@ -395,23 +502,38 @@ export interface ElectricCollectionConfig<
|
|
|
395
502
|
|
|
396
503
|
/**
|
|
397
504
|
* Optional asynchronous handler function called before a delete operation
|
|
505
|
+
*
|
|
506
|
+
* **IMPORTANT - Electric Synchronization:**
|
|
507
|
+
* This handler **must not resolve** until synchronization is confirmed.
|
|
508
|
+
* Await one of these synchronization utilities before the handler completes:
|
|
509
|
+
* 1. `await collection.utils.awaitTxId(txid)` (recommended for most cases)
|
|
510
|
+
* 2. `await collection.utils.awaitMatch(fn)` for custom matching logic
|
|
511
|
+
*
|
|
512
|
+
* Simply returning without waiting for sync will drop optimistic state too early, causing UI glitches.
|
|
513
|
+
*
|
|
398
514
|
* @param params Object containing transaction and collection information
|
|
399
|
-
* @returns Promise
|
|
515
|
+
* @returns Promise that should resolve after synchronization is complete
|
|
516
|
+
*
|
|
517
|
+
* **Deprecation notice:** Returning `{ txid }` from handlers is deprecated and will be removed in v1.0.
|
|
518
|
+
* Use `await collection.utils.awaitTxId(txid)` within the handler instead.
|
|
519
|
+
*
|
|
400
520
|
* @example
|
|
401
|
-
* //
|
|
402
|
-
* onDelete: async ({ transaction }) => {
|
|
521
|
+
* // Recommended: Wait for txid to sync
|
|
522
|
+
* onDelete: async ({ transaction, collection }) => {
|
|
403
523
|
* const mutation = transaction.mutations[0]
|
|
404
524
|
* const result = await api.todos.delete({
|
|
405
525
|
* id: mutation.original.id
|
|
406
526
|
* })
|
|
407
|
-
*
|
|
527
|
+
* // Wait for txid to sync before handler completes
|
|
528
|
+
* await collection.utils.awaitTxId(result.txid)
|
|
408
529
|
* }
|
|
409
530
|
*
|
|
410
531
|
* @example
|
|
411
|
-
* // Use awaitMatch utility for custom matching
|
|
532
|
+
* // Alternative: Use awaitMatch utility for custom matching logic
|
|
412
533
|
* onDelete: async ({ transaction, collection }) => {
|
|
413
534
|
* const mutation = transaction.mutations[0]
|
|
414
535
|
* await api.todos.delete({ id: mutation.original.id })
|
|
536
|
+
* // Wait for specific change to appear in sync stream
|
|
415
537
|
* await collection.utils.awaitMatch(
|
|
416
538
|
* (message) => isChangeMessage(message) &&
|
|
417
539
|
* message.headers.operation === 'delete' &&
|
|
@@ -530,6 +652,7 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
530
652
|
commit,
|
|
531
653
|
getCommitCursor,
|
|
532
654
|
waitForCommitsAfter,
|
|
655
|
+
recordSnapshotRow,
|
|
533
656
|
collectionId,
|
|
534
657
|
encodeColumnName,
|
|
535
658
|
signal,
|
|
@@ -546,6 +669,7 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
546
669
|
commit: (signal?: AbortSignal) => SyncAppliedReceipt
|
|
547
670
|
getCommitCursor: () => number
|
|
548
671
|
waitForCommitsAfter: (cursor: number) => Promise<void>
|
|
672
|
+
recordSnapshotRow?: (row: T, message: Message<T>) => void
|
|
549
673
|
collectionId?: string
|
|
550
674
|
/**
|
|
551
675
|
* Optional function to encode column names (e.g., camelCase to snake_case).
|
|
@@ -607,7 +731,9 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
607
731
|
metadata: { ...row.headers },
|
|
608
732
|
})
|
|
609
733
|
}
|
|
610
|
-
|
|
734
|
+
const applied = commit(opts.signal)
|
|
735
|
+
for (const row of rows) recordSnapshotRow?.(row.value, row)
|
|
736
|
+
if (applied !== true) await applied
|
|
611
737
|
debug(`${logPrefix}Applied snapshot with ${rows.length} rows`)
|
|
612
738
|
}
|
|
613
739
|
} catch (error) {
|
|
@@ -626,56 +752,6 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
626
752
|
|
|
627
753
|
const { cursor, where, orderBy, limit } = opts
|
|
628
754
|
|
|
629
|
-
// When the stream is already up-to-date, it may be in a long-poll wait.
|
|
630
|
-
// Forcing a disconnect-and-refresh ensures requestSnapshot gets a response
|
|
631
|
-
// from a fresh server round-trip rather than waiting for the current poll to end.
|
|
632
|
-
// Some native fetch implementations (notably React Native/Expo) may not abort
|
|
633
|
-
// long-poll requests promptly. Bound the wait so on-demand live queries don't
|
|
634
|
-
// remain loading until the long-poll naturally times out.
|
|
635
|
-
// If the refresh fails or times out, we fall through to requestSnapshot which
|
|
636
|
-
// still works.
|
|
637
|
-
if (stream.isUpToDate) {
|
|
638
|
-
let timeoutId: ReturnType<typeof setTimeout> | undefined
|
|
639
|
-
const abortSignals = [signal, opts.signal].filter(
|
|
640
|
-
(candidate): candidate is AbortSignal => candidate !== undefined,
|
|
641
|
-
)
|
|
642
|
-
let rejectAbort: (reason: unknown) => void = () => {}
|
|
643
|
-
const aborted = new Promise<never>((_resolve, reject) => {
|
|
644
|
-
rejectAbort = reject
|
|
645
|
-
})
|
|
646
|
-
const abort = (event: Event) =>
|
|
647
|
-
rejectAbort(abortReason(event.currentTarget as AbortSignal))
|
|
648
|
-
for (const abortSignal of abortSignals) {
|
|
649
|
-
abortSignal.addEventListener(`abort`, abort, { once: true })
|
|
650
|
-
}
|
|
651
|
-
try {
|
|
652
|
-
await Promise.race([
|
|
653
|
-
stream.forceDisconnectAndRefresh(),
|
|
654
|
-
new Promise<void>((resolve) => {
|
|
655
|
-
timeoutId = setTimeout(
|
|
656
|
-
resolve,
|
|
657
|
-
FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS,
|
|
658
|
-
)
|
|
659
|
-
}),
|
|
660
|
-
aborted,
|
|
661
|
-
])
|
|
662
|
-
} catch (error) {
|
|
663
|
-
if (signal.aborted || opts.signal?.aborted) throw error
|
|
664
|
-
if (handleSnapshotError(error, `forceDisconnectAndRefresh`)) {
|
|
665
|
-
return
|
|
666
|
-
}
|
|
667
|
-
debug(
|
|
668
|
-
`${logPrefix}forceDisconnectAndRefresh failed, proceeding to requestSnapshot: %o`,
|
|
669
|
-
error,
|
|
670
|
-
)
|
|
671
|
-
} finally {
|
|
672
|
-
clearTimeout(timeoutId)
|
|
673
|
-
for (const abortSignal of abortSignals) {
|
|
674
|
-
abortSignal.removeEventListener(`abort`, abort)
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
|
|
679
755
|
throwIfAborted()
|
|
680
756
|
|
|
681
757
|
// Upstream limitation: ShapeStream.requestSnapshot() publishes its rows
|
|
@@ -746,11 +822,251 @@ export type AwaitMatchFn<T extends Row<unknown>> = (
|
|
|
746
822
|
*/
|
|
747
823
|
export interface ElectricCollectionUtils<
|
|
748
824
|
T extends Row<unknown> = Row<unknown>,
|
|
749
|
-
>
|
|
825
|
+
> {
|
|
826
|
+
// Keep this interface closed: extending UtilsRecord would make every
|
|
827
|
+
// nonexistent adapter utility appear as `any`.
|
|
750
828
|
awaitTxId: AwaitTxIdFn
|
|
751
829
|
awaitMatch: AwaitMatchFn<T>
|
|
752
830
|
}
|
|
753
831
|
|
|
832
|
+
/** Owns evidence and pending work for one materialized Collection. */
|
|
833
|
+
class ElectricLifecycle<T extends Row<unknown>> {
|
|
834
|
+
private readonly evidence: ElectricLifecycleEvidence = {
|
|
835
|
+
seenTxids: new Set(),
|
|
836
|
+
seenSnapshots: [],
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
private readonly pendingMatches = new Map<number, ElectricPendingMatch<T>>()
|
|
840
|
+
private readonly pendingTxidWaits = new Map<
|
|
841
|
+
number,
|
|
842
|
+
{
|
|
843
|
+
txId: Txid
|
|
844
|
+
resolve: (value: boolean) => void
|
|
845
|
+
reject: (error: Error) => void
|
|
846
|
+
timeoutId: ReturnType<typeof setTimeout>
|
|
847
|
+
}
|
|
848
|
+
>()
|
|
849
|
+
private matchBuffer: ElectricMatchBuffer<T> = {
|
|
850
|
+
committedMessages: [],
|
|
851
|
+
pendingMessages: [],
|
|
852
|
+
}
|
|
853
|
+
private nextWaiterId = 0
|
|
854
|
+
private epoch = 0
|
|
855
|
+
private active = false
|
|
856
|
+
|
|
857
|
+
constructor(private readonly collectionId?: string) {}
|
|
858
|
+
|
|
859
|
+
readonly utils: ElectricCollectionUtils<T> = {
|
|
860
|
+
awaitTxId: (txId, timeout) => this.awaitTxId(txId, timeout),
|
|
861
|
+
awaitMatch: (matchFn, timeout) => this.awaitMatch(matchFn, timeout),
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
start(): number {
|
|
865
|
+
if (this.active) this.retire()
|
|
866
|
+
this.active = true
|
|
867
|
+
this.epoch++
|
|
868
|
+
this.matchBuffer = { committedMessages: [], pendingMessages: [] }
|
|
869
|
+
return this.epoch
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
isActive(epoch: number): boolean {
|
|
873
|
+
return this.active && this.epoch === epoch
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
retire(epoch?: number): void {
|
|
877
|
+
if (epoch !== undefined && !this.isActive(epoch)) return
|
|
878
|
+
this.active = false
|
|
879
|
+
this.epoch++
|
|
880
|
+
|
|
881
|
+
for (const match of this.pendingMatches.values()) {
|
|
882
|
+
clearTimeout(match.timeoutId)
|
|
883
|
+
match.reject(new StreamAbortedError(this.collectionId))
|
|
884
|
+
}
|
|
885
|
+
this.pendingMatches.clear()
|
|
886
|
+
|
|
887
|
+
for (const waiter of this.pendingTxidWaits.values()) {
|
|
888
|
+
clearTimeout(waiter.timeoutId)
|
|
889
|
+
waiter.reject(new StreamAbortedError(this.collectionId))
|
|
890
|
+
}
|
|
891
|
+
this.pendingTxidWaits.clear()
|
|
892
|
+
this.matchBuffer = { committedMessages: [], pendingMessages: [] }
|
|
893
|
+
this.evidence.hydratedResumeState = undefined
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
exportMeta(): ElectricSyncMeta {
|
|
897
|
+
return exportElectricSyncMeta(this.evidence)
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
importMeta(meta: unknown): void {
|
|
901
|
+
importElectricSyncMeta(this.evidence, meta)
|
|
902
|
+
this.resolveTxidWaiters()
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
get resumeState(): ElectricResumeState | undefined {
|
|
906
|
+
return this.evidence.hydratedResumeState
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
set resumeState(value: ElectricResumeState | undefined) {
|
|
910
|
+
this.evidence.hydratedResumeState = value
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
publishEvidence(
|
|
914
|
+
txids: ReadonlySet<Txid>,
|
|
915
|
+
snapshots: ReadonlyArray<PostgresSnapshot>,
|
|
916
|
+
): void {
|
|
917
|
+
txids.forEach((txid) => this.evidence.seenTxids.add(txid))
|
|
918
|
+
this.evidence.seenSnapshots.push(...snapshots)
|
|
919
|
+
this.resolveTxidWaiters()
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
private hasTxid(txId: Txid): boolean {
|
|
923
|
+
return (
|
|
924
|
+
this.evidence.seenTxids.has(txId) ||
|
|
925
|
+
this.evidence.seenSnapshots.some((snapshot) =>
|
|
926
|
+
isVisibleInSnapshot(txId, snapshot),
|
|
927
|
+
)
|
|
928
|
+
)
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
private resolveTxidWaiters(): void {
|
|
932
|
+
for (const [waitId, waiter] of this.pendingTxidWaits) {
|
|
933
|
+
if (!this.hasTxid(waiter.txId)) continue
|
|
934
|
+
clearTimeout(waiter.timeoutId)
|
|
935
|
+
this.pendingTxidWaits.delete(waitId)
|
|
936
|
+
waiter.resolve(true)
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
private async awaitTxId(
|
|
941
|
+
txId: Txid,
|
|
942
|
+
timeout: number = 15000,
|
|
943
|
+
): Promise<boolean> {
|
|
944
|
+
debug(
|
|
945
|
+
`${this.collectionId ? `[${this.collectionId}] ` : ``}awaitTxId called with txid %d`,
|
|
946
|
+
txId,
|
|
947
|
+
)
|
|
948
|
+
if (typeof txId !== `number`) {
|
|
949
|
+
throw new ExpectedNumberInAwaitTxIdError(typeof txId, this.collectionId)
|
|
950
|
+
}
|
|
951
|
+
if (this.hasTxid(txId)) return true
|
|
952
|
+
|
|
953
|
+
return new Promise((resolve, reject) => {
|
|
954
|
+
const waitId = this.nextWaiterId++
|
|
955
|
+
const timeoutId = setTimeout(() => {
|
|
956
|
+
this.pendingTxidWaits.delete(waitId)
|
|
957
|
+
reject(new TimeoutWaitingForTxIdError(txId, this.collectionId))
|
|
958
|
+
}, timeout)
|
|
959
|
+
this.pendingTxidWaits.set(waitId, {
|
|
960
|
+
txId,
|
|
961
|
+
resolve,
|
|
962
|
+
reject,
|
|
963
|
+
timeoutId,
|
|
964
|
+
})
|
|
965
|
+
})
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
private async awaitMatch(
|
|
969
|
+
matchFn: MatchFunction<T>,
|
|
970
|
+
timeout: number = 15000,
|
|
971
|
+
): Promise<boolean> {
|
|
972
|
+
debug(
|
|
973
|
+
`${this.collectionId ? `[${this.collectionId}] ` : ``}awaitMatch called with custom function`,
|
|
974
|
+
)
|
|
975
|
+
|
|
976
|
+
for (const message of this.matchBuffer.committedMessages) {
|
|
977
|
+
if (!matchFn(message)) continue
|
|
978
|
+
return true
|
|
979
|
+
}
|
|
980
|
+
for (const message of this.matchBuffer.pendingMessages) {
|
|
981
|
+
if (matchFn(message)) return this.registerMatch(matchFn, timeout, true)
|
|
982
|
+
}
|
|
983
|
+
return this.registerMatch(matchFn, timeout, false)
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
private registerMatch(
|
|
987
|
+
matchFn: MatchFunction<T>,
|
|
988
|
+
timeout: number,
|
|
989
|
+
matched: boolean,
|
|
990
|
+
): Promise<boolean> {
|
|
991
|
+
return new Promise((resolve, reject) => {
|
|
992
|
+
const matchId = this.nextWaiterId++
|
|
993
|
+
const timeoutId = setTimeout(() => {
|
|
994
|
+
this.pendingMatches.delete(matchId)
|
|
995
|
+
reject(new TimeoutWaitingForMatchError(this.collectionId))
|
|
996
|
+
}, timeout)
|
|
997
|
+
this.pendingMatches.set(matchId, {
|
|
998
|
+
matchFn,
|
|
999
|
+
resolve,
|
|
1000
|
+
reject,
|
|
1001
|
+
timeoutId,
|
|
1002
|
+
matched,
|
|
1003
|
+
})
|
|
1004
|
+
})
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
beginMatchGeneration(messages: ReadonlyArray<Message<T>>): void {
|
|
1008
|
+
if (!messages.some(isMustRefetchMessage)) return
|
|
1009
|
+
this.matchBuffer = { committedMessages: [], pendingMessages: [] }
|
|
1010
|
+
for (const match of this.pendingMatches.values()) match.matched = false
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
observeMatchMessage(message: Message<T>): void {
|
|
1014
|
+
if (
|
|
1015
|
+
isChangeMessage(message) ||
|
|
1016
|
+
isMoveOutMessage(message) ||
|
|
1017
|
+
isMoveInMessage(message)
|
|
1018
|
+
) {
|
|
1019
|
+
this.matchBuffer.pendingMessages.push(message)
|
|
1020
|
+
let overflow =
|
|
1021
|
+
this.matchBuffer.committedMessages.length +
|
|
1022
|
+
this.matchBuffer.pendingMessages.length -
|
|
1023
|
+
1000
|
|
1024
|
+
if (overflow > 0) {
|
|
1025
|
+
const committedOverflow = Math.min(
|
|
1026
|
+
overflow,
|
|
1027
|
+
this.matchBuffer.committedMessages.length,
|
|
1028
|
+
)
|
|
1029
|
+
this.matchBuffer.committedMessages.splice(0, committedOverflow)
|
|
1030
|
+
overflow -= committedOverflow
|
|
1031
|
+
if (overflow > 0) {
|
|
1032
|
+
this.matchBuffer.pendingMessages.splice(0, overflow)
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
const epoch = this.epoch
|
|
1038
|
+
for (const [matchId, match] of this.pendingMatches) {
|
|
1039
|
+
if (match.matched) continue
|
|
1040
|
+
try {
|
|
1041
|
+
match.matched = match.matchFn(message)
|
|
1042
|
+
} catch (error) {
|
|
1043
|
+
clearTimeout(match.timeoutId)
|
|
1044
|
+
this.pendingMatches.delete(matchId)
|
|
1045
|
+
match.reject(error instanceof Error ? error : new Error(String(error)))
|
|
1046
|
+
}
|
|
1047
|
+
// Reentry can register replacement-session waiters in this same Map.
|
|
1048
|
+
if (!this.isActive(epoch)) return
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
commitMatches(): void {
|
|
1053
|
+
this.matchBuffer.committedMessages.push(...this.matchBuffer.pendingMessages)
|
|
1054
|
+
this.matchBuffer.pendingMessages = []
|
|
1055
|
+
if (this.matchBuffer.committedMessages.length > 1000) {
|
|
1056
|
+
this.matchBuffer.committedMessages.splice(
|
|
1057
|
+
0,
|
|
1058
|
+
this.matchBuffer.committedMessages.length - 1000,
|
|
1059
|
+
)
|
|
1060
|
+
}
|
|
1061
|
+
for (const [matchId, match] of this.pendingMatches) {
|
|
1062
|
+
if (!match.matched) continue
|
|
1063
|
+
clearTimeout(match.timeoutId)
|
|
1064
|
+
this.pendingMatches.delete(matchId)
|
|
1065
|
+
match.resolve(true)
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
754
1070
|
/**
|
|
755
1071
|
* Creates Electric collection options for use with a standard Collection
|
|
756
1072
|
*
|
|
@@ -804,269 +1120,73 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
804
1120
|
utils: ElectricCollectionUtils<T>
|
|
805
1121
|
schema?: any
|
|
806
1122
|
} {
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
const hydratedResumeState = new Store<ElectricResumeState | undefined>(
|
|
810
|
-
undefined,
|
|
811
|
-
)
|
|
1123
|
+
let descriptorLifecycle = new ElectricLifecycle<T>(config.id)
|
|
1124
|
+
let utilityLifecycle = descriptorLifecycle
|
|
812
1125
|
const internalSyncMode = config.syncMode ?? `eager`
|
|
813
1126
|
const finalSyncMode =
|
|
814
1127
|
internalSyncMode === `progressive` ? `on-demand` : internalSyncMode
|
|
815
|
-
const
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
const currentBatchMessages = new Store<Array<Message<any>>>([])
|
|
830
|
-
|
|
831
|
-
// Track whether the current batch has been committed (up-to-date received)
|
|
832
|
-
// This allows awaitMatch to resolve immediately for messages from committed batches
|
|
833
|
-
const batchCommitted = new Store<boolean>(false)
|
|
834
|
-
|
|
835
|
-
/**
|
|
836
|
-
* Helper function to remove multiple matches from the pendingMatches store
|
|
837
|
-
*/
|
|
838
|
-
const removePendingMatches = (matchIds: Array<string>) => {
|
|
839
|
-
if (matchIds.length > 0) {
|
|
840
|
-
pendingMatches.setState((current) => {
|
|
841
|
-
const newMatches = new Map(current)
|
|
842
|
-
matchIds.forEach((id) => newMatches.delete(id))
|
|
843
|
-
return newMatches
|
|
844
|
-
})
|
|
845
|
-
}
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
/**
|
|
849
|
-
* Helper function to resolve and cleanup matched pending matches
|
|
850
|
-
*/
|
|
851
|
-
const resolveMatchedPendingMatches = () => {
|
|
852
|
-
const matchesToResolve: Array<string> = []
|
|
853
|
-
pendingMatches.state.forEach((match, matchId) => {
|
|
854
|
-
if (match.matched) {
|
|
855
|
-
clearTimeout(match.timeoutId)
|
|
856
|
-
match.resolve(true)
|
|
857
|
-
matchesToResolve.push(matchId)
|
|
858
|
-
debug(
|
|
859
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch resolved on up-to-date for match %s`,
|
|
860
|
-
matchId,
|
|
861
|
-
)
|
|
862
|
-
}
|
|
1128
|
+
const boundLifecycles = new WeakMap<object, ElectricLifecycle<T>>()
|
|
1129
|
+
const awaitTxId: AwaitTxIdFn = (txId, timeout) =>
|
|
1130
|
+
utilityLifecycle.utils.awaitTxId(txId, timeout)
|
|
1131
|
+
|
|
1132
|
+
const awaitMatch: AwaitMatchFn<T> = (matchFn, timeout) =>
|
|
1133
|
+
utilityLifecycle.utils.awaitMatch(matchFn, timeout)
|
|
1134
|
+
|
|
1135
|
+
const createSync = () =>
|
|
1136
|
+
createElectricSync<T>(config.shapeOptions, {
|
|
1137
|
+
getLifecycle: (collection) =>
|
|
1138
|
+
boundLifecycles.get(collection) ?? descriptorLifecycle,
|
|
1139
|
+
syncMode: internalSyncMode,
|
|
1140
|
+
collectionId: config.id,
|
|
1141
|
+
testHooks: config[ELECTRIC_TEST_HOOKS],
|
|
863
1142
|
})
|
|
864
|
-
|
|
865
|
-
}
|
|
866
|
-
const sync = createElectricSync<T>(config.shapeOptions, {
|
|
867
|
-
seenTxids,
|
|
868
|
-
seenSnapshots,
|
|
869
|
-
hydratedResumeState,
|
|
870
|
-
syncMode: internalSyncMode,
|
|
871
|
-
pendingMatches,
|
|
872
|
-
currentBatchMessages,
|
|
873
|
-
batchCommitted,
|
|
874
|
-
removePendingMatches,
|
|
875
|
-
resolveMatchedPendingMatches,
|
|
876
|
-
collectionId: config.id,
|
|
877
|
-
testHooks: config[ELECTRIC_TEST_HOOKS],
|
|
878
|
-
})
|
|
879
|
-
|
|
880
|
-
/**
|
|
881
|
-
* Wait for a specific transaction ID to be synced
|
|
882
|
-
* @param txId The transaction ID to wait for as a number
|
|
883
|
-
* @param timeout Optional timeout in milliseconds (defaults to 5000ms)
|
|
884
|
-
* @returns Promise that resolves when the txId is synced
|
|
885
|
-
*/
|
|
886
|
-
const awaitTxId: AwaitTxIdFn = async (
|
|
887
|
-
txId: Txid,
|
|
888
|
-
timeout: number = 5000,
|
|
889
|
-
): Promise<boolean> => {
|
|
890
|
-
debug(
|
|
891
|
-
`${config.id ? `[${config.id}] ` : ``}awaitTxId called with txid %d`,
|
|
892
|
-
txId,
|
|
893
|
-
)
|
|
894
|
-
if (typeof txId !== `number`) {
|
|
895
|
-
throw new ExpectedNumberInAwaitTxIdError(typeof txId, config.id)
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
// First check if the txid is in the seenTxids store
|
|
899
|
-
const hasTxid = seenTxids.state.has(txId)
|
|
900
|
-
if (hasTxid) return true
|
|
901
|
-
|
|
902
|
-
// Then check if the txid is in any of the seen snapshots
|
|
903
|
-
const hasSnapshot = seenSnapshots.state.some((snapshot) =>
|
|
904
|
-
isVisibleInSnapshot(txId, snapshot),
|
|
905
|
-
)
|
|
906
|
-
if (hasSnapshot) return true
|
|
907
|
-
|
|
908
|
-
return new Promise((resolve, reject) => {
|
|
909
|
-
const cleanup = () => {
|
|
910
|
-
clearTimeout(timeoutId)
|
|
911
|
-
subSeenTxids.unsubscribe()
|
|
912
|
-
subSeenSnapshots.unsubscribe()
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
const timeoutId = setTimeout(() => {
|
|
916
|
-
cleanup()
|
|
917
|
-
reject(new TimeoutWaitingForTxIdError(txId, config.id))
|
|
918
|
-
}, timeout)
|
|
919
|
-
|
|
920
|
-
const subSeenTxids = seenTxids.subscribe(() => {
|
|
921
|
-
if (seenTxids.state.has(txId)) {
|
|
922
|
-
debug(
|
|
923
|
-
`${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o`,
|
|
924
|
-
txId,
|
|
925
|
-
)
|
|
926
|
-
cleanup()
|
|
927
|
-
resolve(true)
|
|
928
|
-
}
|
|
929
|
-
})
|
|
930
|
-
|
|
931
|
-
const subSeenSnapshots = seenSnapshots.subscribe(() => {
|
|
932
|
-
const visibleSnapshot = seenSnapshots.state.find((snapshot) =>
|
|
933
|
-
isVisibleInSnapshot(txId, snapshot),
|
|
934
|
-
)
|
|
935
|
-
if (visibleSnapshot) {
|
|
936
|
-
debug(
|
|
937
|
-
`${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o in snapshot %o`,
|
|
938
|
-
txId,
|
|
939
|
-
visibleSnapshot,
|
|
940
|
-
)
|
|
941
|
-
cleanup()
|
|
942
|
-
resolve(true)
|
|
943
|
-
}
|
|
944
|
-
})
|
|
945
|
-
})
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
/**
|
|
949
|
-
* Wait for a custom match function to find a matching message
|
|
950
|
-
* @param matchFn Function that returns true when a message matches
|
|
951
|
-
* @param timeout Optional timeout in milliseconds (defaults to 5000ms)
|
|
952
|
-
* @returns Promise that resolves when a matching message is found
|
|
953
|
-
*/
|
|
954
|
-
const awaitMatch: AwaitMatchFn<any> = async (
|
|
955
|
-
matchFn: MatchFunction<any>,
|
|
956
|
-
timeout: number = 3000,
|
|
957
|
-
): Promise<boolean> => {
|
|
958
|
-
debug(
|
|
959
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch called with custom function`,
|
|
960
|
-
)
|
|
961
|
-
|
|
962
|
-
return new Promise((resolve, reject) => {
|
|
963
|
-
const matchId = Math.random().toString(36)
|
|
964
|
-
|
|
965
|
-
const cleanupMatch = () => {
|
|
966
|
-
pendingMatches.setState((current) => {
|
|
967
|
-
const newMatches = new Map(current)
|
|
968
|
-
newMatches.delete(matchId)
|
|
969
|
-
return newMatches
|
|
970
|
-
})
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
const onTimeout = () => {
|
|
974
|
-
cleanupMatch()
|
|
975
|
-
reject(new TimeoutWaitingForMatchError(config.id))
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
const timeoutId = setTimeout(onTimeout, timeout)
|
|
979
|
-
|
|
980
|
-
// We need access to the stream messages to check against the match function
|
|
981
|
-
// This will be handled by the sync configuration
|
|
982
|
-
const checkMatch = (message: Message<any>) => {
|
|
983
|
-
if (matchFn(message)) {
|
|
984
|
-
debug(
|
|
985
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch found matching message, waiting for up-to-date`,
|
|
986
|
-
)
|
|
987
|
-
// Mark as matched but don't resolve yet - wait for up-to-date
|
|
988
|
-
pendingMatches.setState((current) => {
|
|
989
|
-
const newMatches = new Map(current)
|
|
990
|
-
const existing = newMatches.get(matchId)
|
|
991
|
-
if (existing) {
|
|
992
|
-
newMatches.set(matchId, { ...existing, matched: true })
|
|
993
|
-
}
|
|
994
|
-
return newMatches
|
|
995
|
-
})
|
|
996
|
-
return true
|
|
997
|
-
}
|
|
998
|
-
return false
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
// Check against current batch messages first to handle race conditions
|
|
1002
|
-
for (const message of currentBatchMessages.state) {
|
|
1003
|
-
if (matchFn(message)) {
|
|
1004
|
-
// If batch is committed (up-to-date already received), resolve immediately
|
|
1005
|
-
// just like awaitTxId does when it finds a txid in seenTxids
|
|
1006
|
-
if (batchCommitted.state) {
|
|
1007
|
-
debug(
|
|
1008
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in committed batch, resolving immediately`,
|
|
1009
|
-
)
|
|
1010
|
-
clearTimeout(timeoutId)
|
|
1011
|
-
resolve(true)
|
|
1012
|
-
return
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
// If batch is not yet committed, register match and wait for up-to-date
|
|
1016
|
-
debug(
|
|
1017
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in current batch, waiting for up-to-date`,
|
|
1018
|
-
)
|
|
1019
|
-
pendingMatches.setState((current) => {
|
|
1020
|
-
const newMatches = new Map(current)
|
|
1021
|
-
newMatches.set(matchId, {
|
|
1022
|
-
matchFn: checkMatch,
|
|
1023
|
-
resolve,
|
|
1024
|
-
reject,
|
|
1025
|
-
timeoutId,
|
|
1026
|
-
matched: true, // Already matched, will resolve on up-to-date
|
|
1027
|
-
})
|
|
1028
|
-
return newMatches
|
|
1029
|
-
})
|
|
1030
|
-
return
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
// Store the match function for the sync process to use
|
|
1035
|
-
// We'll add this to a pending matches store
|
|
1036
|
-
pendingMatches.setState((current) => {
|
|
1037
|
-
const newMatches = new Map(current)
|
|
1038
|
-
newMatches.set(matchId, {
|
|
1039
|
-
matchFn: checkMatch,
|
|
1040
|
-
resolve,
|
|
1041
|
-
reject,
|
|
1042
|
-
timeoutId,
|
|
1043
|
-
matched: false,
|
|
1044
|
-
})
|
|
1045
|
-
return newMatches
|
|
1046
|
-
})
|
|
1047
|
-
})
|
|
1048
|
-
}
|
|
1143
|
+
const sync = createSync()
|
|
1049
1144
|
|
|
1050
1145
|
/**
|
|
1051
1146
|
* Process matching strategy and wait for synchronization
|
|
1052
1147
|
*/
|
|
1053
1148
|
const processMatchingStrategy = async (
|
|
1054
|
-
result:
|
|
1149
|
+
result: unknown,
|
|
1150
|
+
waitForTxId: AwaitTxIdFn,
|
|
1055
1151
|
): Promise<void> => {
|
|
1152
|
+
const canHaveProperties =
|
|
1153
|
+
(typeof result === `object` && result !== null) ||
|
|
1154
|
+
typeof result === `function`
|
|
1155
|
+
|
|
1056
1156
|
// Only wait if result contains txid
|
|
1057
|
-
if (
|
|
1058
|
-
const
|
|
1157
|
+
if (canHaveProperties && `txid` in result) {
|
|
1158
|
+
const strategy = result as Exclude<MatchingStrategy, void>
|
|
1159
|
+
// Warn about deprecated return value pattern
|
|
1160
|
+
warnOnce(
|
|
1161
|
+
'electric-collection-txid-return',
|
|
1162
|
+
'[TanStack DB] DEPRECATED: Returning { txid } from mutation handlers is deprecated and will be removed in v1.0. ' +
|
|
1163
|
+
'Use `await collection.utils.awaitTxId(txid)` instead of returning { txid }. ' +
|
|
1164
|
+
'See migration guide: https://tanstack.com/db/latest/docs/collections/electric-collection#persistence-handlers--synchronization',
|
|
1165
|
+
)
|
|
1166
|
+
|
|
1167
|
+
const timeout = strategy.timeout
|
|
1059
1168
|
// Handle both single txid and array of txids
|
|
1060
|
-
if (Array.isArray(
|
|
1061
|
-
await Promise.all(
|
|
1169
|
+
if (Array.isArray(strategy.txid)) {
|
|
1170
|
+
await Promise.all(
|
|
1171
|
+
strategy.txid.map((txid) => waitForTxId(txid, timeout)),
|
|
1172
|
+
)
|
|
1062
1173
|
} else {
|
|
1063
|
-
await
|
|
1174
|
+
await waitForTxId(strategy.txid, timeout)
|
|
1064
1175
|
}
|
|
1065
1176
|
}
|
|
1066
1177
|
// If result is void/undefined, don't wait - mutation completes immediately
|
|
1067
1178
|
}
|
|
1179
|
+
const getMutationAwaitTxId = (params: unknown): AwaitTxIdFn => {
|
|
1180
|
+
const collection = (
|
|
1181
|
+
params as {
|
|
1182
|
+
collection?: { utils?: { awaitTxId?: AwaitTxIdFn } }
|
|
1183
|
+
}
|
|
1184
|
+
).collection
|
|
1185
|
+
return collection?.utils?.awaitTxId ?? awaitTxId
|
|
1186
|
+
}
|
|
1068
1187
|
|
|
1069
1188
|
// Create wrapper handlers for direct persistence operations that handle different matching strategies
|
|
1189
|
+
// These wrappers process deprecated return values but don't pass them through
|
|
1070
1190
|
const wrappedOnInsert = config.onInsert
|
|
1071
1191
|
? async (
|
|
1072
1192
|
params: InsertMutationFnParams<
|
|
@@ -1074,10 +1194,12 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
1074
1194
|
string | number,
|
|
1075
1195
|
ElectricCollectionUtils<T>
|
|
1076
1196
|
>,
|
|
1077
|
-
) => {
|
|
1197
|
+
): Promise<void> => {
|
|
1078
1198
|
const handlerResult = await config.onInsert!(params)
|
|
1079
|
-
await processMatchingStrategy(
|
|
1080
|
-
|
|
1199
|
+
await processMatchingStrategy(
|
|
1200
|
+
handlerResult,
|
|
1201
|
+
getMutationAwaitTxId(params),
|
|
1202
|
+
)
|
|
1081
1203
|
}
|
|
1082
1204
|
: undefined
|
|
1083
1205
|
|
|
@@ -1088,10 +1210,12 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
1088
1210
|
string | number,
|
|
1089
1211
|
ElectricCollectionUtils<T>
|
|
1090
1212
|
>,
|
|
1091
|
-
) => {
|
|
1213
|
+
): Promise<void> => {
|
|
1092
1214
|
const handlerResult = await config.onUpdate!(params)
|
|
1093
|
-
await processMatchingStrategy(
|
|
1094
|
-
|
|
1215
|
+
await processMatchingStrategy(
|
|
1216
|
+
handlerResult,
|
|
1217
|
+
getMutationAwaitTxId(params),
|
|
1218
|
+
)
|
|
1095
1219
|
}
|
|
1096
1220
|
: undefined
|
|
1097
1221
|
|
|
@@ -1102,10 +1226,12 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
1102
1226
|
string | number,
|
|
1103
1227
|
ElectricCollectionUtils<T>
|
|
1104
1228
|
>,
|
|
1105
|
-
) => {
|
|
1229
|
+
): Promise<void> => {
|
|
1106
1230
|
const handlerResult = await config.onDelete!(params)
|
|
1107
|
-
await processMatchingStrategy(
|
|
1108
|
-
|
|
1231
|
+
await processMatchingStrategy(
|
|
1232
|
+
handlerResult,
|
|
1233
|
+
getMutationAwaitTxId(params),
|
|
1234
|
+
)
|
|
1109
1235
|
}
|
|
1110
1236
|
: undefined
|
|
1111
1237
|
|
|
@@ -1118,37 +1244,67 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
1118
1244
|
...restConfig
|
|
1119
1245
|
} = config
|
|
1120
1246
|
|
|
1121
|
-
const
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1247
|
+
const utilityTemplate: ElectricCollectionUtils<T> = {
|
|
1248
|
+
awaitTxId,
|
|
1249
|
+
awaitMatch,
|
|
1250
|
+
}
|
|
1251
|
+
const consumeDescriptorLifecycle = (): ElectricLifecycle<T> => {
|
|
1252
|
+
const lifecycle = descriptorLifecycle
|
|
1253
|
+
descriptorLifecycle = new ElectricLifecycle<T>(config.id)
|
|
1254
|
+
utilityLifecycle = lifecycle
|
|
1255
|
+
return lifecycle
|
|
1256
|
+
}
|
|
1257
|
+
const createBoundSync = (
|
|
1258
|
+
source: SyncConfig<T>,
|
|
1259
|
+
utilities: object,
|
|
1260
|
+
): SyncConfig<T> => {
|
|
1261
|
+
const lifecycle = consumeDescriptorLifecycle()
|
|
1262
|
+
let collectionKey: object | undefined
|
|
1263
|
+
Object.assign(utilities, lifecycle.utils)
|
|
1264
|
+
|
|
1265
|
+
const boundSync: SyncConfig<T> = {
|
|
1266
|
+
...source,
|
|
1267
|
+
sync: (params) => {
|
|
1268
|
+
// A copied materialized config may delegate through an older binding.
|
|
1269
|
+
// The outermost binding owns this collection; inner wrappers only run
|
|
1270
|
+
// the source sync and must not retarget either owner's lifecycle.
|
|
1271
|
+
if (!boundLifecycles.has(params.collection)) {
|
|
1272
|
+
collectionKey = params.collection
|
|
1273
|
+
boundLifecycles.set(params.collection, lifecycle)
|
|
1137
1274
|
}
|
|
1138
|
-
|
|
1139
|
-
hydratedResumeState.setState(() => parsed.resume)
|
|
1140
|
-
seenTxids.setState(() => new Set(parsed.seenTxids))
|
|
1275
|
+
return source.sync(params)
|
|
1141
1276
|
},
|
|
1277
|
+
exportSyncMeta: () => lifecycle.exportMeta(),
|
|
1278
|
+
importSyncMeta: (meta) => lifecycle.importMeta(meta),
|
|
1279
|
+
mergeSyncMeta: mergeElectricSyncMeta,
|
|
1280
|
+
}
|
|
1281
|
+
return withCollectionSyncConfigCleanup(boundSync, () => {
|
|
1282
|
+
lifecycle.retire()
|
|
1283
|
+
if (collectionKey) boundLifecycles.delete(collectionKey)
|
|
1284
|
+
})
|
|
1285
|
+
}
|
|
1286
|
+
const syncTemplate = withCollectionSyncConfigFactory(
|
|
1287
|
+
{
|
|
1288
|
+
...sync,
|
|
1289
|
+
exportSyncMeta: () => descriptorLifecycle.exportMeta(),
|
|
1290
|
+
importSyncMeta: (meta) => descriptorLifecycle.importMeta(meta),
|
|
1142
1291
|
mergeSyncMeta: mergeElectricSyncMeta,
|
|
1143
1292
|
},
|
|
1293
|
+
createBoundSync,
|
|
1294
|
+
)
|
|
1295
|
+
const options = {
|
|
1296
|
+
...restConfig,
|
|
1297
|
+
syncMode: finalSyncMode,
|
|
1298
|
+
sync: syncTemplate,
|
|
1144
1299
|
onInsert: wrappedOnInsert,
|
|
1145
1300
|
onUpdate: wrappedOnUpdate,
|
|
1146
1301
|
onDelete: wrappedOnDelete,
|
|
1147
|
-
utils:
|
|
1148
|
-
awaitTxId,
|
|
1149
|
-
awaitMatch,
|
|
1150
|
-
},
|
|
1302
|
+
utils: utilityTemplate,
|
|
1151
1303
|
}
|
|
1304
|
+
Object.defineProperty(options, `utils`, {
|
|
1305
|
+
enumerable: true,
|
|
1306
|
+
get: () => ({ ...utilityTemplate }),
|
|
1307
|
+
})
|
|
1152
1308
|
|
|
1153
1309
|
return withCollectionConfigFactory(options, () =>
|
|
1154
1310
|
(
|
|
@@ -1166,356 +1322,349 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1166
1322
|
shapeOptions: ShapeStreamOptions<GetExtensions<T>>,
|
|
1167
1323
|
options: {
|
|
1168
1324
|
syncMode: ElectricSyncMode
|
|
1169
|
-
|
|
1170
|
-
seenSnapshots: Store<Array<PostgresSnapshot>>
|
|
1171
|
-
hydratedResumeState: Store<ElectricResumeState | undefined>
|
|
1172
|
-
pendingMatches: Store<
|
|
1173
|
-
Map<
|
|
1174
|
-
string,
|
|
1175
|
-
{
|
|
1176
|
-
matchFn: (message: Message<T>) => boolean
|
|
1177
|
-
resolve: (value: boolean) => void
|
|
1178
|
-
reject: (error: Error) => void
|
|
1179
|
-
timeoutId: ReturnType<typeof setTimeout>
|
|
1180
|
-
matched: boolean
|
|
1181
|
-
}
|
|
1182
|
-
>
|
|
1183
|
-
>
|
|
1184
|
-
currentBatchMessages: Store<Array<Message<T>>>
|
|
1185
|
-
batchCommitted: Store<boolean>
|
|
1186
|
-
removePendingMatches: (matchIds: Array<string>) => void
|
|
1187
|
-
resolveMatchedPendingMatches: () => void
|
|
1325
|
+
getLifecycle: (collection: object) => ElectricLifecycle<T>
|
|
1188
1326
|
collectionId?: string
|
|
1189
1327
|
testHooks?: ElectricTestHooks
|
|
1190
1328
|
},
|
|
1191
1329
|
): SyncConfig<T> {
|
|
1192
|
-
const {
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
// Parses a tag string into a ParsedMoveTag.
|
|
1213
|
-
// It memoizes the result parsed tag such that future calls
|
|
1214
|
-
// for the same tag string return the same ParsedMoveTag array.
|
|
1215
|
-
const parseTag = (tag: MoveTag): ParsedMoveTag => {
|
|
1216
|
-
const cachedTag = tagCache.get(tag)
|
|
1217
|
-
if (cachedTag) {
|
|
1218
|
-
return cachedTag
|
|
1330
|
+
const { getLifecycle, syncMode, collectionId, testHooks } = options
|
|
1331
|
+
|
|
1332
|
+
let relationSchema: string | undefined
|
|
1333
|
+
|
|
1334
|
+
const createTagState = () => {
|
|
1335
|
+
const tagCache = new Map<MoveTag, ParsedMoveTag>()
|
|
1336
|
+
|
|
1337
|
+
// Parses a tag string into a ParsedMoveTag.
|
|
1338
|
+
// It memoizes the result parsed tag such that future calls
|
|
1339
|
+
// for the same tag string return the same ParsedMoveTag array.
|
|
1340
|
+
const parseTag = (tag: MoveTag): ParsedMoveTag => {
|
|
1341
|
+
const cachedTag = tagCache.get(tag)
|
|
1342
|
+
if (cachedTag) {
|
|
1343
|
+
return cachedTag
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
const parsedTag = parseTagString(tag)
|
|
1347
|
+
tagCache.set(tag, parsedTag)
|
|
1348
|
+
return parsedTag
|
|
1219
1349
|
}
|
|
1220
1350
|
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1351
|
+
// Tag tracking state
|
|
1352
|
+
const rowTagSets = new Map<RowId, Set<MoveTag>>()
|
|
1353
|
+
const tagIndex: TagIndex = []
|
|
1354
|
+
let tagLength: number | undefined = undefined
|
|
1355
|
+
|
|
1356
|
+
// DNF state: active_conditions are per-row, disjunct_positions are global
|
|
1357
|
+
// (fixed by the shape's WHERE clause, derived once from the first tagged message).
|
|
1358
|
+
const rowActiveConditions = new Map<RowId, ActiveConditions>()
|
|
1359
|
+
let disjunctPositions: DisjunctPositions | undefined = undefined
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* Initialize the tag index with the correct length
|
|
1363
|
+
*/
|
|
1364
|
+
const initializeTagIndex = (length: number): void => {
|
|
1365
|
+
if (tagIndex.length < length) {
|
|
1366
|
+
// Extend the index array to the required length
|
|
1367
|
+
for (let i = tagIndex.length; i < length; i++) {
|
|
1368
|
+
tagIndex[i] = new Map()
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1225
1372
|
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1373
|
+
/**
|
|
1374
|
+
* Add tags to a row and update the tag index
|
|
1375
|
+
*/
|
|
1376
|
+
const addTagsToRow = (
|
|
1377
|
+
tags: Array<MoveTag>,
|
|
1378
|
+
rowId: RowId,
|
|
1379
|
+
rowTagSet: Set<MoveTag>,
|
|
1380
|
+
): void => {
|
|
1381
|
+
for (const tag of tags) {
|
|
1382
|
+
const parsedTag = parseTag(tag)
|
|
1383
|
+
|
|
1384
|
+
// Infer tag length from first tag
|
|
1385
|
+
if (tagLength === undefined) {
|
|
1386
|
+
tagLength = getTagLength(parsedTag)
|
|
1387
|
+
initializeTagIndex(tagLength)
|
|
1388
|
+
}
|
|
1230
1389
|
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1390
|
+
// Validate tag length matches
|
|
1391
|
+
const currentTagLength = getTagLength(parsedTag)
|
|
1392
|
+
if (currentTagLength !== tagLength) {
|
|
1393
|
+
debug(
|
|
1394
|
+
`${collectionId ? `[${collectionId}] ` : ``}Tag length mismatch: expected ${tagLength}, got ${currentTagLength}`,
|
|
1395
|
+
)
|
|
1396
|
+
continue
|
|
1397
|
+
}
|
|
1235
1398
|
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
*/
|
|
1239
|
-
const initializeTagIndex = (length: number): void => {
|
|
1240
|
-
if (tagIndex.length < length) {
|
|
1241
|
-
// Extend the index array to the required length
|
|
1242
|
-
for (let i = tagIndex.length; i < length; i++) {
|
|
1243
|
-
tagIndex[i] = new Map()
|
|
1399
|
+
rowTagSet.add(tag)
|
|
1400
|
+
addTagToIndex(parsedTag, rowId, tagIndex, tagLength)
|
|
1244
1401
|
}
|
|
1245
1402
|
}
|
|
1246
|
-
}
|
|
1247
1403
|
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
for (const tag of tags) {
|
|
1257
|
-
const parsedTag = parseTag(tag)
|
|
1258
|
-
|
|
1259
|
-
// Infer tag length from first tag
|
|
1404
|
+
/**
|
|
1405
|
+
* Remove tags from a row and update the tag index
|
|
1406
|
+
*/
|
|
1407
|
+
const removeTagsFromRow = (
|
|
1408
|
+
removedTags: Array<MoveTag>,
|
|
1409
|
+
rowId: RowId,
|
|
1410
|
+
rowTagSet: Set<MoveTag>,
|
|
1411
|
+
): void => {
|
|
1260
1412
|
if (tagLength === undefined) {
|
|
1261
|
-
|
|
1262
|
-
initializeTagIndex(tagLength)
|
|
1413
|
+
return
|
|
1263
1414
|
}
|
|
1264
1415
|
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1416
|
+
for (const tag of removedTags) {
|
|
1417
|
+
const parsedTag = parseTag(tag)
|
|
1418
|
+
rowTagSet.delete(tag)
|
|
1419
|
+
removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength)
|
|
1420
|
+
// We aggresively evict the tag from the cache
|
|
1421
|
+
// if this tag is shared with another row
|
|
1422
|
+
// and is not removed from that other row
|
|
1423
|
+
// then next time we encounter the tag it will be parsed again
|
|
1424
|
+
tagCache.delete(tag)
|
|
1272
1425
|
}
|
|
1273
|
-
|
|
1274
|
-
rowTagSet.add(tag)
|
|
1275
|
-
addTagToIndex(parsedTag, rowId, tagIndex, tagLength)
|
|
1276
1426
|
}
|
|
1277
|
-
}
|
|
1278
1427
|
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1428
|
+
/**
|
|
1429
|
+
* Process tags for a change message (add and remove tags)
|
|
1430
|
+
*/
|
|
1431
|
+
const processTagsForChangeMessage = (
|
|
1432
|
+
tags: Array<MoveTag> | undefined,
|
|
1433
|
+
removedTags: Array<MoveTag> | undefined,
|
|
1434
|
+
rowId: RowId,
|
|
1435
|
+
activeConditions?: ActiveConditions,
|
|
1436
|
+
): Set<MoveTag> => {
|
|
1437
|
+
// Initialize tag set for this row if it doesn't exist (needed for checking deletion)
|
|
1438
|
+
if (!rowTagSets.has(rowId)) {
|
|
1439
|
+
rowTagSets.set(rowId, new Set())
|
|
1440
|
+
}
|
|
1441
|
+
const rowTagSet = rowTagSets.get(rowId)!
|
|
1290
1442
|
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength)
|
|
1295
|
-
// We aggresively evict the tag from the cache
|
|
1296
|
-
// if this tag is shared with another row
|
|
1297
|
-
// and is not removed from that other row
|
|
1298
|
-
// then next time we encounter the tag it will be parsed again
|
|
1299
|
-
tagCache.delete(tag)
|
|
1300
|
-
}
|
|
1301
|
-
}
|
|
1443
|
+
// Add new tags
|
|
1444
|
+
if (tags) {
|
|
1445
|
+
addTagsToRow(tags, rowId, rowTagSet)
|
|
1302
1446
|
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
rowId: RowId,
|
|
1310
|
-
activeConditions?: ActiveConditions,
|
|
1311
|
-
): Set<MoveTag> => {
|
|
1312
|
-
// Initialize tag set for this row if it doesn't exist (needed for checking deletion)
|
|
1313
|
-
if (!rowTagSets.has(rowId)) {
|
|
1314
|
-
rowTagSets.set(rowId, new Set())
|
|
1315
|
-
}
|
|
1316
|
-
const rowTagSet = rowTagSets.get(rowId)!
|
|
1447
|
+
// Derive disjunct positions once — they are fixed by the shape's WHERE clause.
|
|
1448
|
+
if (disjunctPositions === undefined) {
|
|
1449
|
+
const parsedTags = tags.map(parseTag)
|
|
1450
|
+
disjunctPositions = deriveDisjunctPositions(parsedTags)
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1317
1453
|
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1454
|
+
// Remove tags
|
|
1455
|
+
if (removedTags) {
|
|
1456
|
+
removeTagsFromRow(removedTags, rowId, rowTagSet)
|
|
1457
|
+
}
|
|
1321
1458
|
|
|
1322
|
-
//
|
|
1323
|
-
if (
|
|
1324
|
-
|
|
1325
|
-
disjunctPositions = deriveDisjunctPositions(parsedTags)
|
|
1459
|
+
// Store active conditions if provided (overwrite on re-send)
|
|
1460
|
+
if (activeConditions && activeConditions.length > 0) {
|
|
1461
|
+
rowActiveConditions.set(rowId, [...activeConditions])
|
|
1326
1462
|
}
|
|
1327
|
-
}
|
|
1328
1463
|
|
|
1329
|
-
|
|
1330
|
-
if (removedTags) {
|
|
1331
|
-
removeTagsFromRow(removedTags, rowId, rowTagSet)
|
|
1464
|
+
return rowTagSet
|
|
1332
1465
|
}
|
|
1333
1466
|
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1467
|
+
/**
|
|
1468
|
+
* Clear all tag tracking state (used when truncating)
|
|
1469
|
+
*/
|
|
1470
|
+
const clearTagTrackingState = (): void => {
|
|
1471
|
+
rowTagSets.clear()
|
|
1472
|
+
tagIndex.length = 0
|
|
1473
|
+
tagLength = undefined
|
|
1474
|
+
rowActiveConditions.clear()
|
|
1475
|
+
disjunctPositions = undefined
|
|
1337
1476
|
}
|
|
1338
1477
|
|
|
1339
|
-
|
|
1340
|
-
|
|
1478
|
+
/**
|
|
1479
|
+
* Remove all tags for a row from both the tag set and the index
|
|
1480
|
+
* Used when a row is deleted
|
|
1481
|
+
*/
|
|
1482
|
+
const clearTagsForRow = (rowId: RowId): void => {
|
|
1483
|
+
if (tagLength === undefined) {
|
|
1484
|
+
return
|
|
1485
|
+
}
|
|
1341
1486
|
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
rowTagSets.clear()
|
|
1347
|
-
tagIndex.length = 0
|
|
1348
|
-
tagLength = undefined
|
|
1349
|
-
rowActiveConditions.clear()
|
|
1350
|
-
disjunctPositions = undefined
|
|
1351
|
-
}
|
|
1487
|
+
const rowTagSet = rowTagSets.get(rowId)
|
|
1488
|
+
if (!rowTagSet) {
|
|
1489
|
+
return
|
|
1490
|
+
}
|
|
1352
1491
|
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1492
|
+
// Remove each tag from the index
|
|
1493
|
+
for (const tag of rowTagSet) {
|
|
1494
|
+
const parsedTag = parseTag(tag)
|
|
1495
|
+
const currentTagLength = getTagLength(parsedTag)
|
|
1496
|
+
if (currentTagLength === tagLength) {
|
|
1497
|
+
removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength)
|
|
1498
|
+
}
|
|
1499
|
+
tagCache.delete(tag)
|
|
1500
|
+
}
|
|
1361
1501
|
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1502
|
+
// Remove the row from the tag sets map
|
|
1503
|
+
rowTagSets.delete(rowId)
|
|
1504
|
+
rowActiveConditions.delete(rowId)
|
|
1365
1505
|
}
|
|
1366
1506
|
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1507
|
+
/**
|
|
1508
|
+
* Remove matching tags from a row based on a pattern
|
|
1509
|
+
* Returns true if the row should be deleted (no longer visible)
|
|
1510
|
+
*/
|
|
1511
|
+
const removeMatchingTagsFromRow = (
|
|
1512
|
+
rowId: RowId,
|
|
1513
|
+
pattern: MovePattern,
|
|
1514
|
+
): boolean => {
|
|
1515
|
+
const rowTagSet = rowTagSets.get(rowId)
|
|
1516
|
+
if (!rowTagSet) {
|
|
1517
|
+
return false
|
|
1373
1518
|
}
|
|
1374
|
-
tagCache.delete(tag)
|
|
1375
|
-
}
|
|
1376
|
-
|
|
1377
|
-
// Remove the row from the tag sets map
|
|
1378
|
-
rowTagSets.delete(rowId)
|
|
1379
|
-
rowActiveConditions.delete(rowId)
|
|
1380
|
-
}
|
|
1381
1519
|
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1520
|
+
// DNF mode: check visibility using active conditions.
|
|
1521
|
+
// Tag index entries are preserved so that move-in can re-activate positions.
|
|
1522
|
+
const activeConditions = rowActiveConditions.get(rowId)
|
|
1523
|
+
if (activeConditions && disjunctPositions) {
|
|
1524
|
+
// Set the condition at this pattern's position to false
|
|
1525
|
+
activeConditions[pattern.pos] = false
|
|
1526
|
+
|
|
1527
|
+
if (!rowVisible(activeConditions, disjunctPositions)) {
|
|
1528
|
+
// Row is no longer visible — clean up all state including tag index
|
|
1529
|
+
for (const tag of rowTagSet) {
|
|
1530
|
+
const parsedTag = parseTag(tag)
|
|
1531
|
+
removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength!)
|
|
1532
|
+
tagCache.delete(tag)
|
|
1533
|
+
}
|
|
1534
|
+
rowTagSets.delete(rowId)
|
|
1535
|
+
rowActiveConditions.delete(rowId)
|
|
1536
|
+
return true
|
|
1537
|
+
}
|
|
1538
|
+
return false
|
|
1539
|
+
}
|
|
1394
1540
|
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
if (!rowVisible(activeConditions, disjunctPositions)) {
|
|
1403
|
-
// Row is no longer visible — clean up all state including tag index
|
|
1404
|
-
for (const tag of rowTagSet) {
|
|
1405
|
-
const parsedTag = parseTag(tag)
|
|
1541
|
+
// Simple shape (no subquery dependencies — server sends no active_conditions):
|
|
1542
|
+
// Remove matching tags and delete if tag set is empty
|
|
1543
|
+
for (const tag of rowTagSet) {
|
|
1544
|
+
const parsedTag = parseTag(tag)
|
|
1545
|
+
if (tagMatchesPattern(parsedTag, pattern)) {
|
|
1546
|
+
rowTagSet.delete(tag)
|
|
1406
1547
|
removeTagFromIndex(parsedTag, rowId, tagIndex, tagLength!)
|
|
1407
|
-
tagCache.delete(tag)
|
|
1408
1548
|
}
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
if (rowTagSet.size === 0) {
|
|
1409
1552
|
rowTagSets.delete(rowId)
|
|
1410
|
-
rowActiveConditions.delete(rowId)
|
|
1411
1553
|
return true
|
|
1412
1554
|
}
|
|
1555
|
+
|
|
1413
1556
|
return false
|
|
1414
1557
|
}
|
|
1415
1558
|
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1559
|
+
/**
|
|
1560
|
+
* Process move-out event: remove matching tags from rows and delete rows with empty tag sets
|
|
1561
|
+
*/
|
|
1562
|
+
const processMoveOutEvent = (
|
|
1563
|
+
patterns: Array<MovePattern>,
|
|
1564
|
+
begin: () => void,
|
|
1565
|
+
write: (message: ChangeMessageOrDeleteKeyMessage<T>) => void,
|
|
1566
|
+
transactionStarted: boolean,
|
|
1567
|
+
onDelete: (rowId: RowId) => void,
|
|
1568
|
+
): boolean => {
|
|
1569
|
+
if (tagLength === undefined) {
|
|
1570
|
+
debug(
|
|
1571
|
+
`${collectionId ? `[${collectionId}] ` : ``}Received move-out message but no tag length set yet, ignoring`,
|
|
1572
|
+
)
|
|
1573
|
+
return transactionStarted
|
|
1423
1574
|
}
|
|
1424
|
-
}
|
|
1425
|
-
|
|
1426
|
-
if (rowTagSet.size === 0) {
|
|
1427
|
-
rowTagSets.delete(rowId)
|
|
1428
|
-
return true
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
return false
|
|
1432
|
-
}
|
|
1433
1575
|
|
|
1434
|
-
|
|
1435
|
-
* Process move-out event: remove matching tags from rows and delete rows with empty tag sets
|
|
1436
|
-
*/
|
|
1437
|
-
const processMoveOutEvent = (
|
|
1438
|
-
patterns: Array<MovePattern>,
|
|
1439
|
-
begin: () => void,
|
|
1440
|
-
write: (message: ChangeMessageOrDeleteKeyMessage<T>) => void,
|
|
1441
|
-
transactionStarted: boolean,
|
|
1442
|
-
): boolean => {
|
|
1443
|
-
if (tagLength === undefined) {
|
|
1444
|
-
debug(
|
|
1445
|
-
`${collectionId ? `[${collectionId}] ` : ``}Received move-out message but no tag length set yet, ignoring`,
|
|
1446
|
-
)
|
|
1447
|
-
return transactionStarted
|
|
1448
|
-
}
|
|
1576
|
+
let txStarted = transactionStarted
|
|
1449
1577
|
|
|
1450
|
-
|
|
1578
|
+
// Process all patterns and collect rows to delete
|
|
1579
|
+
for (const pattern of patterns) {
|
|
1580
|
+
// Find all rows that match this pattern
|
|
1581
|
+
const affectedRowIds = findRowsMatchingPattern(pattern, tagIndex)
|
|
1451
1582
|
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1583
|
+
for (const rowId of affectedRowIds) {
|
|
1584
|
+
if (removeMatchingTagsFromRow(rowId, pattern)) {
|
|
1585
|
+
// Delete rows with empty tag sets
|
|
1586
|
+
if (!txStarted) {
|
|
1587
|
+
begin()
|
|
1588
|
+
txStarted = true
|
|
1589
|
+
}
|
|
1456
1590
|
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
txStarted = true
|
|
1591
|
+
write({
|
|
1592
|
+
type: `delete`,
|
|
1593
|
+
key: rowId,
|
|
1594
|
+
})
|
|
1595
|
+
onDelete(rowId)
|
|
1463
1596
|
}
|
|
1464
|
-
|
|
1465
|
-
write({
|
|
1466
|
-
type: `delete`,
|
|
1467
|
-
key: rowId,
|
|
1468
|
-
})
|
|
1469
1597
|
}
|
|
1470
1598
|
}
|
|
1471
|
-
}
|
|
1472
1599
|
|
|
1473
|
-
|
|
1474
|
-
}
|
|
1475
|
-
|
|
1476
|
-
/**
|
|
1477
|
-
* Process move-in event: re-activate conditions for rows matching the patterns.
|
|
1478
|
-
* This is a silent operation — no messages are emitted to the collection.
|
|
1479
|
-
*/
|
|
1480
|
-
const processMoveInEvent = (patterns: Array<MovePattern>): void => {
|
|
1481
|
-
if (tagLength === undefined) {
|
|
1482
|
-
debug(
|
|
1483
|
-
`${collectionId ? `[${collectionId}] ` : ``}Received move-in message but no tag length set yet, ignoring`,
|
|
1484
|
-
)
|
|
1485
|
-
return
|
|
1600
|
+
return txStarted
|
|
1486
1601
|
}
|
|
1487
1602
|
|
|
1488
|
-
|
|
1489
|
-
|
|
1603
|
+
/**
|
|
1604
|
+
* Process move-in event: re-activate conditions for rows matching the patterns.
|
|
1605
|
+
* This is a silent operation — no messages are emitted to the collection.
|
|
1606
|
+
*/
|
|
1607
|
+
const processMoveInEvent = (patterns: Array<MovePattern>): void => {
|
|
1608
|
+
if (tagLength === undefined) {
|
|
1609
|
+
debug(
|
|
1610
|
+
`${collectionId ? `[${collectionId}] ` : ``}Received move-in message but no tag length set yet, ignoring`,
|
|
1611
|
+
)
|
|
1612
|
+
return
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
for (const pattern of patterns) {
|
|
1616
|
+
const affectedRowIds = findRowsMatchingPattern(pattern, tagIndex)
|
|
1490
1617
|
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1618
|
+
for (const rowId of affectedRowIds) {
|
|
1619
|
+
const activeConditions = rowActiveConditions.get(rowId)
|
|
1620
|
+
if (activeConditions) {
|
|
1621
|
+
activeConditions[pattern.pos] = true
|
|
1622
|
+
}
|
|
1495
1623
|
}
|
|
1496
1624
|
}
|
|
1497
1625
|
}
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
/**
|
|
1501
|
-
* Get the sync metadata for insert operations
|
|
1502
|
-
* @returns Record containing relation information
|
|
1503
|
-
*/
|
|
1504
|
-
const getSyncMetadata = (): Record<string, unknown> => {
|
|
1505
|
-
// Use the stored schema if available, otherwise default to 'public'
|
|
1506
|
-
const schema = relationSchema.state || `public`
|
|
1507
1626
|
|
|
1508
1627
|
return {
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1628
|
+
hasTags: () => tagLength !== undefined,
|
|
1629
|
+
processTagsForChangeMessage,
|
|
1630
|
+
clearTagTrackingState,
|
|
1631
|
+
clearTagsForRow,
|
|
1632
|
+
processMoveOutEvent,
|
|
1633
|
+
processMoveInEvent,
|
|
1512
1634
|
}
|
|
1513
1635
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1636
|
+
// Tags belong to a collection, and survive a compatible resume of that
|
|
1637
|
+
// collection. Reusing an options descriptor must not share visibility.
|
|
1638
|
+
const collectionTags = new WeakMap<
|
|
1639
|
+
object,
|
|
1640
|
+
ReturnType<typeof createTagState>
|
|
1641
|
+
>()
|
|
1516
1642
|
|
|
1517
1643
|
return {
|
|
1644
|
+
getSyncMetadata: () => ({
|
|
1645
|
+
relation: shapeOptions.params?.table
|
|
1646
|
+
? [relationSchema || `public`, shapeOptions.params.table]
|
|
1647
|
+
: undefined,
|
|
1648
|
+
}),
|
|
1518
1649
|
sync: (params: Parameters<SyncConfig<T>[`sync`]>[0]) => {
|
|
1650
|
+
const retainsTagState = collectionTags.has(params.collection)
|
|
1651
|
+
let tagState = collectionTags.get(params.collection)
|
|
1652
|
+
if (!tagState) {
|
|
1653
|
+
tagState = createTagState()
|
|
1654
|
+
collectionTags.set(params.collection, tagState)
|
|
1655
|
+
}
|
|
1656
|
+
const {
|
|
1657
|
+
processTagsForChangeMessage,
|
|
1658
|
+
clearTagTrackingState,
|
|
1659
|
+
clearTagsForRow,
|
|
1660
|
+
processMoveOutEvent,
|
|
1661
|
+
processMoveInEvent,
|
|
1662
|
+
} = tagState
|
|
1663
|
+
const lifecycle = getLifecycle(params.collection)
|
|
1664
|
+
const lifecycleEpoch = lifecycle.start()
|
|
1665
|
+
const isActiveLifecycle = () => lifecycle.isActive(lifecycleEpoch)
|
|
1666
|
+
Object.assign(params.collection.utils, lifecycle.utils)
|
|
1667
|
+
|
|
1519
1668
|
const {
|
|
1520
1669
|
begin,
|
|
1521
1670
|
write,
|
|
@@ -1551,9 +1700,20 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1551
1700
|
return parseElectricResumeState(persistedResumeState)
|
|
1552
1701
|
}
|
|
1553
1702
|
|
|
1703
|
+
const persistence =
|
|
1704
|
+
metadata === undefined
|
|
1705
|
+
? null
|
|
1706
|
+
: validateSyncPersistenceCapability(metadata.persistence)
|
|
1707
|
+
const hydrateBaseline = persistence?.hydrateBaseline
|
|
1708
|
+
const resumeSnapshot = persistence?.resumeSnapshot
|
|
1709
|
+
const certifyResumeSnapshot = resumeSnapshot?.certify
|
|
1710
|
+
const getKeySetEvidence = resumeSnapshot?.getKeySetEvidence
|
|
1711
|
+
const expectCurrentCommit = resumeSnapshot?.expectCurrentCommit
|
|
1712
|
+
const persistedKeySetEvidence = getKeySetEvidence?.()
|
|
1713
|
+
|
|
1554
1714
|
const persistedResumeState = getNewestElectricResumeState(
|
|
1555
1715
|
readPersistedResumeState(),
|
|
1556
|
-
|
|
1716
|
+
lifecycle.resumeState,
|
|
1557
1717
|
)
|
|
1558
1718
|
const shapeIdentity = getStableShapeIdentity({
|
|
1559
1719
|
url: shapeOptions.url,
|
|
@@ -1562,11 +1722,48 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1562
1722
|
const hasIncompatiblePersistedResume =
|
|
1563
1723
|
persistedResumeState?.kind === `resume` &&
|
|
1564
1724
|
persistedResumeState.shapeId !== shapeIdentity
|
|
1725
|
+
// A pre-ledger `unknown` baseline cannot justify a non-initial cursor.
|
|
1726
|
+
// One fresh replacement establishes consistent evidence for later resumes.
|
|
1727
|
+
const lacksCompletePersistedKeySet =
|
|
1728
|
+
persistedResumeState?.kind === `resume` &&
|
|
1729
|
+
getKeySetEvidence !== undefined &&
|
|
1730
|
+
persistedKeySetEvidence?.status !== `consistent`
|
|
1731
|
+
const needsFullSnapshot =
|
|
1732
|
+
shapeOptions.offset === undefined &&
|
|
1733
|
+
shapeOptions.handle === undefined &&
|
|
1734
|
+
persistedResumeState !== undefined &&
|
|
1735
|
+
(persistedResumeState.kind === `reset` ||
|
|
1736
|
+
lacksCompletePersistedKeySet ||
|
|
1737
|
+
(!retainsTagState && persistedResumeState.requiresTagState !== false))
|
|
1565
1738
|
const canUsePersistedResume =
|
|
1566
1739
|
shapeOptions.offset === undefined &&
|
|
1567
1740
|
shapeOptions.handle === undefined &&
|
|
1568
1741
|
persistedResumeState?.kind === `resume` &&
|
|
1569
|
-
!hasIncompatiblePersistedResume
|
|
1742
|
+
!hasIncompatiblePersistedResume &&
|
|
1743
|
+
// Cached rows do not contain authoritative tag/active-condition state.
|
|
1744
|
+
// Only a complete adapter ledger can justify a persisted cursor.
|
|
1745
|
+
!needsFullSnapshot
|
|
1746
|
+
const hasExplicitResumeOffset =
|
|
1747
|
+
shapeOptions.offset !== undefined && shapeOptions.offset !== `-1`
|
|
1748
|
+
if (!canUsePersistedResume && !hasExplicitResumeOffset) {
|
|
1749
|
+
clearTagTrackingState()
|
|
1750
|
+
}
|
|
1751
|
+
const receivesCompleteRows = shapeOptions.params?.replica === `full`
|
|
1752
|
+
const requiresKeySetCertification =
|
|
1753
|
+
canUsePersistedResume && certifyResumeSnapshot !== undefined
|
|
1754
|
+
// Eager and progressive streams that start after the initial offset can
|
|
1755
|
+
// only apply partial updates when the local materialization is complete.
|
|
1756
|
+
const requiresCompleteResume =
|
|
1757
|
+
syncMode !== `on-demand` &&
|
|
1758
|
+
(canUsePersistedResume ||
|
|
1759
|
+
(hasExplicitResumeOffset && !receivesCompleteRows))
|
|
1760
|
+
// A fresh snapshot replaces its hydrated cache; omitting the old offset
|
|
1761
|
+
// alone would merge rows that no longer exist on the server.
|
|
1762
|
+
let freshSnapshotPending =
|
|
1763
|
+
(syncMode === `eager` || needsFullSnapshot) &&
|
|
1764
|
+
!canUsePersistedResume &&
|
|
1765
|
+
!hasExplicitResumeOffset &&
|
|
1766
|
+
hydrateBaseline !== undefined
|
|
1570
1767
|
|
|
1571
1768
|
// Wrap markReady to wait for test hook in progressive mode
|
|
1572
1769
|
let progressiveReadyGate: Promise<void> | null = null
|
|
@@ -1609,28 +1806,25 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1609
1806
|
}
|
|
1610
1807
|
}
|
|
1611
1808
|
|
|
1612
|
-
// Cleanup pending matches on abort
|
|
1613
1809
|
abortController.signal.addEventListener(`abort`, () => {
|
|
1614
|
-
|
|
1615
|
-
current.forEach((match) => {
|
|
1616
|
-
clearTimeout(match.timeoutId)
|
|
1617
|
-
match.reject(new StreamAbortedError())
|
|
1618
|
-
})
|
|
1619
|
-
return new Map() // Clear all pending matches
|
|
1620
|
-
})
|
|
1810
|
+
lifecycle.retire(lifecycleEpoch)
|
|
1621
1811
|
})
|
|
1622
1812
|
|
|
1623
1813
|
const stream = new ShapeStream({
|
|
1624
1814
|
...shapeOptions,
|
|
1625
|
-
//
|
|
1626
|
-
|
|
1815
|
+
// Recovery needs a complete snapshot even for on-demand shapes.
|
|
1816
|
+
// Normal on-demand startup still subscribes to changes only.
|
|
1817
|
+
log:
|
|
1818
|
+
syncMode === `on-demand` && !needsFullSnapshot
|
|
1819
|
+
? `changes_only`
|
|
1820
|
+
: undefined,
|
|
1627
1821
|
// In on-demand mode, we only need the changes from the point of time the collection was created
|
|
1628
1822
|
// so we default to `now` when there is no saved offset.
|
|
1629
1823
|
offset:
|
|
1630
1824
|
shapeOptions.offset ??
|
|
1631
1825
|
(canUsePersistedResume
|
|
1632
1826
|
? (persistedResumeState.offset as Offset)
|
|
1633
|
-
: syncMode === `on-demand`
|
|
1827
|
+
: syncMode === `on-demand` && !needsFullSnapshot
|
|
1634
1828
|
? `now`
|
|
1635
1829
|
: undefined),
|
|
1636
1830
|
handle:
|
|
@@ -1670,21 +1864,80 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1670
1864
|
// resume starts from an already-committed stream offset, so the next
|
|
1671
1865
|
// up-to-date message must not run the initial atomic swap again.
|
|
1672
1866
|
let hasReceivedUpToDate =
|
|
1673
|
-
syncMode === `progressive` &&
|
|
1867
|
+
syncMode === `progressive` && requiresCompleteResume
|
|
1868
|
+
// A must-refetch starts a new snapshot generation. Until its up-to-date
|
|
1869
|
+
// commit is applied, old Collection keys cannot make an update valid and
|
|
1870
|
+
// the durable resume marker must remain reset.
|
|
1871
|
+
let isResettingSnapshot = false
|
|
1872
|
+
let resetGeneration = 0
|
|
1674
1873
|
|
|
1675
1874
|
// Progressive mode state
|
|
1676
1875
|
// Helper to determine if we're buffering the initial sync
|
|
1677
1876
|
const isBufferingInitialSync = () =>
|
|
1678
|
-
syncMode === `progressive` &&
|
|
1877
|
+
syncMode === `progressive` &&
|
|
1878
|
+
!hasReceivedUpToDate &&
|
|
1879
|
+
!isResettingSnapshot
|
|
1679
1880
|
const bufferedMessages: Array<Message<T>> = [] // Buffer change messages during initial sync
|
|
1881
|
+
// Progressive subset rows are provisional: the initial atomic swap
|
|
1882
|
+
// normally discards them. Retain one only when a later stream update
|
|
1883
|
+
// needs that full row as its baseline.
|
|
1884
|
+
const progressiveSnapshotRows = new Map<string | number, Message<T>>()
|
|
1885
|
+
|
|
1886
|
+
// Presence spans the open source transaction and persisted callbacks
|
|
1887
|
+
// whose FIFO receipts have not applied yet. Later callbacks must see
|
|
1888
|
+
// those staged writes even though persistence has not replayed them into
|
|
1889
|
+
// core. Once the newest receipt applies, syncedData is authoritative.
|
|
1890
|
+
const pendingPresence = new Map<string | number, boolean>()
|
|
1891
|
+
let usesBaseline = true
|
|
1892
|
+
let pendingPresenceRevision = 0
|
|
1893
|
+
|
|
1894
|
+
const recordPendingPresence = (
|
|
1895
|
+
rowId: string | number,
|
|
1896
|
+
present: boolean,
|
|
1897
|
+
) => {
|
|
1898
|
+
pendingPresenceRevision++
|
|
1899
|
+
pendingPresence.set(rowId, present)
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
const recordPendingTruncate = () => {
|
|
1903
|
+
pendingPresenceRevision++
|
|
1904
|
+
pendingPresence.clear()
|
|
1905
|
+
progressiveSnapshotRows.clear()
|
|
1906
|
+
usesBaseline = false
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
const commitSourceTransaction = (): SyncAppliedReceipt => {
|
|
1910
|
+
const committedPresenceRevision = pendingPresenceRevision
|
|
1911
|
+
const applied = commit()
|
|
1912
|
+
if (metadata?.persistence) {
|
|
1913
|
+
const retireAppliedPresence = () => {
|
|
1914
|
+
if (pendingPresenceRevision === committedPresenceRevision) {
|
|
1915
|
+
pendingPresence.clear()
|
|
1916
|
+
usesBaseline = true
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
if (applied === true) retireAppliedPresence()
|
|
1920
|
+
else void applied.then(retireAppliedPresence, () => undefined)
|
|
1921
|
+
}
|
|
1922
|
+
return applied
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
const beginSourceTransaction = () => {
|
|
1926
|
+
if (metadata?.persistence) begin({ immediate: true })
|
|
1927
|
+
else begin()
|
|
1928
|
+
}
|
|
1680
1929
|
|
|
1681
1930
|
// Track keys that have been synced to handle overlapping subset queries.
|
|
1682
1931
|
// When multiple subset queries return the same row, the server sends `insert`
|
|
1683
1932
|
// for each response. We convert subsequent inserts to updates to avoid
|
|
1684
1933
|
// duplicate key errors when the row's data has changed between requests.
|
|
1685
1934
|
const syncedKeys = new Set<string | number>()
|
|
1935
|
+
let resumeInvalid = false
|
|
1686
1936
|
|
|
1687
1937
|
const stageResumeMetadata = () => {
|
|
1938
|
+
if (!isActiveLifecycle() || resumeInvalid) {
|
|
1939
|
+
return
|
|
1940
|
+
}
|
|
1688
1941
|
const shapeHandle = stream.shapeHandle
|
|
1689
1942
|
const lastOffset = stream.lastOffset
|
|
1690
1943
|
if (!shapeHandle || lastOffset === `-1`) {
|
|
@@ -1697,27 +1950,39 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1697
1950
|
handle: shapeHandle,
|
|
1698
1951
|
shapeId: shapeIdentity,
|
|
1699
1952
|
updatedAt: Date.now(),
|
|
1953
|
+
requiresTagState: tagState.hasTags(),
|
|
1700
1954
|
}
|
|
1701
|
-
|
|
1955
|
+
lifecycle.resumeState = resumeState
|
|
1702
1956
|
metadata?.collection.set(`electric:resume`, resumeState)
|
|
1703
1957
|
}
|
|
1704
1958
|
|
|
1705
|
-
const commitResetResumeMetadataImmediately = (
|
|
1959
|
+
const commitResetResumeMetadataImmediately = (
|
|
1960
|
+
expectInResumeSnapshot = false,
|
|
1961
|
+
) => {
|
|
1706
1962
|
const resetState: ElectricResumeState = {
|
|
1707
1963
|
kind: `reset`,
|
|
1708
1964
|
updatedAt: Date.now(),
|
|
1709
1965
|
}
|
|
1710
|
-
|
|
1966
|
+
lifecycle.resumeState = resetState
|
|
1711
1967
|
|
|
1712
1968
|
if (metadata) {
|
|
1713
1969
|
begin({ immediate: true })
|
|
1714
1970
|
metadata.collection.set(`electric:resume`, resetState)
|
|
1971
|
+
if (expectInResumeSnapshot) {
|
|
1972
|
+
expectCurrentCommit?.()
|
|
1973
|
+
}
|
|
1715
1974
|
commit()
|
|
1716
1975
|
}
|
|
1717
1976
|
}
|
|
1718
1977
|
|
|
1719
|
-
if (
|
|
1720
|
-
|
|
1978
|
+
if (
|
|
1979
|
+
hasIncompatiblePersistedResume ||
|
|
1980
|
+
(needsFullSnapshot && persistedResumeState.kind === `resume`)
|
|
1981
|
+
) {
|
|
1982
|
+
// This reset is part of the current runtime's startup decision. The
|
|
1983
|
+
// persisted wrapper may commit it before loading the atomic baseline,
|
|
1984
|
+
// so carry ownership of exactly this generation into certification.
|
|
1985
|
+
commitResetResumeMetadataImmediately(true)
|
|
1721
1986
|
}
|
|
1722
1987
|
|
|
1723
1988
|
/**
|
|
@@ -1788,6 +2053,10 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1788
2053
|
commit,
|
|
1789
2054
|
getCommitCursor: () => commitSequence,
|
|
1790
2055
|
waitForCommitsAfter,
|
|
2056
|
+
recordSnapshotRow: (row, message) => {
|
|
2057
|
+
recordPendingPresence(collection.getKeyFromItem(row), true)
|
|
2058
|
+
progressiveSnapshotRows.set(collection.getKeyFromItem(row), message)
|
|
2059
|
+
},
|
|
1791
2060
|
collectionId,
|
|
1792
2061
|
// Pass the columnMapper's encode function to transform column names
|
|
1793
2062
|
// (e.g., camelCase to snake_case) when compiling SQL for subset queries
|
|
@@ -1796,32 +2065,113 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1796
2065
|
signal: abortController.signal,
|
|
1797
2066
|
})
|
|
1798
2067
|
|
|
1799
|
-
|
|
2068
|
+
const resumeKeysPromise =
|
|
2069
|
+
requiresCompleteResume || freshSnapshotPending
|
|
2070
|
+
? hydrateBaseline
|
|
2071
|
+
? (async () => {
|
|
2072
|
+
await hydrateBaseline()
|
|
2073
|
+
const currentKeySetEvidence = getKeySetEvidence?.()
|
|
2074
|
+
if (
|
|
2075
|
+
canUsePersistedResume &&
|
|
2076
|
+
currentKeySetEvidence?.status !== `consistent`
|
|
2077
|
+
) {
|
|
2078
|
+
throw new Error(
|
|
2079
|
+
`Electric persisted resume baseline could not be certified during hydration`,
|
|
2080
|
+
)
|
|
2081
|
+
}
|
|
2082
|
+
})()
|
|
2083
|
+
: undefined
|
|
2084
|
+
: requiresKeySetCertification
|
|
2085
|
+
? (async () => {
|
|
2086
|
+
await certifyResumeSnapshot()
|
|
2087
|
+
const currentKeySetEvidence = getKeySetEvidence?.()
|
|
2088
|
+
if (currentKeySetEvidence?.status !== `consistent`) {
|
|
2089
|
+
throw new Error(
|
|
2090
|
+
`Electric persisted resume baseline could not be certified`,
|
|
2091
|
+
)
|
|
2092
|
+
}
|
|
2093
|
+
})()
|
|
2094
|
+
: undefined
|
|
2095
|
+
let areResumeKeysReady = !resumeKeysPromise
|
|
2096
|
+
const pendingResumeBatches: Array<Array<Message<T>>> = []
|
|
2097
|
+
let unsubscribeStream: () => void = () => {}
|
|
2098
|
+
|
|
2099
|
+
const invalidateResume = () => {
|
|
2100
|
+
resumeInvalid = true
|
|
2101
|
+
if (transactionStarted) {
|
|
2102
|
+
const cancellation = new AbortController()
|
|
2103
|
+
cancellation.abort()
|
|
2104
|
+
commit(cancellation.signal)
|
|
2105
|
+
transactionStarted = false
|
|
2106
|
+
}
|
|
2107
|
+
syncedKeys.clear()
|
|
2108
|
+
newTxids.clear()
|
|
2109
|
+
newSnapshots.length = 0
|
|
2110
|
+
commitResetResumeMetadataImmediately()
|
|
2111
|
+
streamErrorVersion++
|
|
2112
|
+
unsubscribeStream()
|
|
2113
|
+
abortController.abort()
|
|
2114
|
+
markError(
|
|
2115
|
+
new Error(
|
|
2116
|
+
`Electric resume state referenced an unseen row; a full snapshot is required`,
|
|
2117
|
+
),
|
|
2118
|
+
)
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
const processMessages = (messages: Array<Message<T>>): void => {
|
|
2122
|
+
if (!isActiveLifecycle() || resumeInvalid) {
|
|
2123
|
+
return
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
if (freshSnapshotPending) {
|
|
2127
|
+
freshSnapshotPending = false
|
|
2128
|
+
beginSourceTransaction()
|
|
2129
|
+
transactionStarted = true
|
|
2130
|
+
truncate()
|
|
2131
|
+
recordPendingTruncate()
|
|
2132
|
+
syncedKeys.clear()
|
|
2133
|
+
clearTagTrackingState()
|
|
2134
|
+
isResettingSnapshot = true
|
|
2135
|
+
resetGeneration++
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
// Without persistence, core owns pending source transactions and can
|
|
2139
|
+
// rebuild this callback's presence overlay from them. Persistence owns
|
|
2140
|
+
// queued source transactions until their FIFO turn, so retain the
|
|
2141
|
+
// overlay across callbacks instead. A queued truncate still fences off
|
|
2142
|
+
// the previous snapshot below.
|
|
2143
|
+
if (!metadata?.persistence && !transactionStarted) {
|
|
2144
|
+
pendingPresence.clear()
|
|
2145
|
+
usesBaseline = true
|
|
2146
|
+
for (const pending of collection._state.pendingSyncedTransactions) {
|
|
2147
|
+
if (pending.truncate) {
|
|
2148
|
+
pendingPresence.clear()
|
|
2149
|
+
usesBaseline = false
|
|
2150
|
+
}
|
|
2151
|
+
for (const operation of pending.operations) {
|
|
2152
|
+
pendingPresence.set(operation.key, operation.type !== `delete`)
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
for (const message of bufferedMessages) {
|
|
2156
|
+
if (isChangeMessage(message)) {
|
|
2157
|
+
pendingPresence.set(
|
|
2158
|
+
collection.getKeyFromItem(message.value),
|
|
2159
|
+
message.headers.operation !== `delete`,
|
|
2160
|
+
)
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
|
|
1800
2165
|
// Track commit point type - up-to-date takes precedence as it also triggers progressive mode atomic swap
|
|
1801
2166
|
let commitPoint: `up-to-date` | `subset-end` | null = null
|
|
1802
2167
|
|
|
1803
|
-
|
|
1804
|
-
// to find even if multiple batches arrive before awaitMatch is called.
|
|
1805
|
-
// The buffer is naturally limited by MAX_BATCH_MESSAGES (oldest messages are dropped).
|
|
1806
|
-
// Reset batchCommitted since we're starting a new batch
|
|
1807
|
-
batchCommitted.setState(() => false)
|
|
2168
|
+
lifecycle.beginMatchGeneration(messages)
|
|
1808
2169
|
|
|
1809
2170
|
for (const message of messages) {
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
isMoveInMessage(message)
|
|
1815
|
-
) {
|
|
1816
|
-
currentBatchMessages.setState((currentBuffer) => {
|
|
1817
|
-
const newBuffer = [...currentBuffer, message]
|
|
1818
|
-
// Limit buffer size for safety
|
|
1819
|
-
if (newBuffer.length > MAX_BATCH_MESSAGES) {
|
|
1820
|
-
newBuffer.splice(0, newBuffer.length - MAX_BATCH_MESSAGES)
|
|
1821
|
-
}
|
|
1822
|
-
return newBuffer
|
|
1823
|
-
})
|
|
1824
|
-
}
|
|
2171
|
+
lifecycle.observeMatchMessage(message)
|
|
2172
|
+
// A match predicate can synchronously clean up and restart sync.
|
|
2173
|
+
// Nothing after that boundary belongs to the replacement session.
|
|
2174
|
+
if (!isActiveLifecycle()) return
|
|
1825
2175
|
|
|
1826
2176
|
// Check for txids in the message and add them to our store
|
|
1827
2177
|
// Skip during buffered initial sync in progressive mode (txids will be extracted during atomic swap)
|
|
@@ -1834,45 +2184,53 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1834
2184
|
message.headers.txids?.forEach((txid) => newTxids.add(txid))
|
|
1835
2185
|
}
|
|
1836
2186
|
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
matchesToRemove.push(matchId)
|
|
1851
|
-
debug(`matchFn error: %o`, err)
|
|
2187
|
+
if (isChangeMessage(message)) {
|
|
2188
|
+
const rowId = collection.getKeyFromItem(message.value)
|
|
2189
|
+
const operation = message.headers.operation
|
|
2190
|
+
const hasKnownRow =
|
|
2191
|
+
pendingPresence.get(rowId) ??
|
|
2192
|
+
(usesBaseline && collection._state.syncedData.has(rowId))
|
|
2193
|
+
if (operation === `update` && !hasKnownRow) {
|
|
2194
|
+
// Validate after all earlier events, including tag move-outs.
|
|
2195
|
+
// Cancel staged writes before publishing any part of an invalid
|
|
2196
|
+
// resumed callback; the next lifecycle must take a full snapshot.
|
|
2197
|
+
if (requiresCompleteResume && !isResettingSnapshot) {
|
|
2198
|
+
invalidateResume()
|
|
2199
|
+
return
|
|
1852
2200
|
}
|
|
2201
|
+
if (!receivesCompleteRows) continue
|
|
1853
2202
|
}
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
2203
|
+
recordPendingPresence(rowId, operation !== `delete`)
|
|
2204
|
+
if (operation !== `update`) {
|
|
2205
|
+
progressiveSnapshotRows.delete(rowId)
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
1858
2208
|
|
|
1859
2209
|
if (isChangeMessage(message)) {
|
|
1860
2210
|
// Check if the message contains schema information
|
|
1861
2211
|
const schema = message.headers.schema
|
|
1862
2212
|
if (schema && typeof schema === `string`) {
|
|
1863
2213
|
// Store the schema for future use if it's a valid string
|
|
1864
|
-
relationSchema
|
|
2214
|
+
relationSchema = schema
|
|
1865
2215
|
}
|
|
1866
2216
|
|
|
1867
2217
|
// In buffered initial sync of progressive mode, buffer messages instead of writing
|
|
1868
2218
|
// EXCEPTION: If a transaction is already started (e.g., from must-refetch), write
|
|
1869
2219
|
// directly to it instead of buffering. This prevents orphan transactions.
|
|
1870
2220
|
if (isBufferingInitialSync() && !transactionStarted) {
|
|
2221
|
+
if (message.headers.operation === `update`) {
|
|
2222
|
+
const rowId = collection.getKeyFromItem(message.value)
|
|
2223
|
+
const snapshotRow = progressiveSnapshotRows.get(rowId)
|
|
2224
|
+
if (snapshotRow) {
|
|
2225
|
+
bufferedMessages.push(snapshotRow)
|
|
2226
|
+
progressiveSnapshotRows.delete(rowId)
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
1871
2229
|
bufferedMessages.push(message)
|
|
1872
2230
|
} else {
|
|
1873
2231
|
// Normal processing: write changes immediately
|
|
1874
2232
|
if (!transactionStarted) {
|
|
1875
|
-
|
|
2233
|
+
beginSourceTransaction()
|
|
1876
2234
|
transactionStarted = true
|
|
1877
2235
|
}
|
|
1878
2236
|
|
|
@@ -1904,9 +2262,13 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1904
2262
|
// Normal processing: process move-out immediately
|
|
1905
2263
|
transactionStarted = processMoveOutEvent(
|
|
1906
2264
|
message.headers.patterns,
|
|
1907
|
-
|
|
2265
|
+
beginSourceTransaction,
|
|
1908
2266
|
write,
|
|
1909
2267
|
transactionStarted,
|
|
2268
|
+
(rowId) => {
|
|
2269
|
+
recordPendingPresence(rowId, false)
|
|
2270
|
+
syncedKeys.delete(rowId)
|
|
2271
|
+
},
|
|
1910
2272
|
)
|
|
1911
2273
|
}
|
|
1912
2274
|
} else if (isMoveInMessage(message)) {
|
|
@@ -1926,7 +2288,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1926
2288
|
|
|
1927
2289
|
// Start a transaction and truncate the collection
|
|
1928
2290
|
if (!transactionStarted) {
|
|
1929
|
-
|
|
2291
|
+
beginSourceTransaction()
|
|
1930
2292
|
transactionStarted = true
|
|
1931
2293
|
}
|
|
1932
2294
|
|
|
@@ -1937,6 +2299,9 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1937
2299
|
|
|
1938
2300
|
// Clear synced keys tracking since we're starting fresh
|
|
1939
2301
|
syncedKeys.clear()
|
|
2302
|
+
recordPendingTruncate()
|
|
2303
|
+
isResettingSnapshot = true
|
|
2304
|
+
resetGeneration++
|
|
1940
2305
|
|
|
1941
2306
|
// Reset the loadSubset deduplication state since we're starting fresh
|
|
1942
2307
|
// This ensures that previously loaded predicates don't prevent refetching after truncate
|
|
@@ -1946,12 +2311,24 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1946
2311
|
commitPoint = null
|
|
1947
2312
|
hasReceivedUpToDate = false // Reset for progressive mode (isBufferingInitialSync will reflect this)
|
|
1948
2313
|
bufferedMessages.length = 0 // Clear buffered messages
|
|
2314
|
+
progressiveSnapshotRows.clear()
|
|
1949
2315
|
}
|
|
1950
2316
|
}
|
|
1951
2317
|
|
|
2318
|
+
// A subset completion cannot publish a partial cold-recovery snapshot.
|
|
2319
|
+
if (
|
|
2320
|
+
needsFullSnapshot &&
|
|
2321
|
+
isResettingSnapshot &&
|
|
2322
|
+
commitPoint === `subset-end`
|
|
2323
|
+
)
|
|
2324
|
+
return
|
|
2325
|
+
|
|
1952
2326
|
if (commitPoint !== null) {
|
|
1953
2327
|
let applied: SyncAppliedReceipt = true
|
|
1954
2328
|
const wasBufferingInitialSync = isBufferingInitialSync()
|
|
2329
|
+
const finishesReset =
|
|
2330
|
+
isResettingSnapshot && commitPoint === `up-to-date`
|
|
2331
|
+
const finishingResetGeneration = resetGeneration
|
|
1955
2332
|
// PROGRESSIVE MODE: Atomic swap on first up-to-date (not subset-end)
|
|
1956
2333
|
// EXCEPTION: Skip atomic swap if a transaction is already started (e.g., from must-refetch).
|
|
1957
2334
|
// In that case, do a normal commit to properly close the existing transaction.
|
|
@@ -1965,7 +2342,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1965
2342
|
)
|
|
1966
2343
|
|
|
1967
2344
|
// Start atomic swap transaction
|
|
1968
|
-
|
|
2345
|
+
beginSourceTransaction()
|
|
1969
2346
|
|
|
1970
2347
|
// Truncate to clear all snapshot data
|
|
1971
2348
|
truncate()
|
|
@@ -1996,7 +2373,13 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1996
2373
|
bufferedMsg.headers.patterns,
|
|
1997
2374
|
begin,
|
|
1998
2375
|
write,
|
|
1999
|
-
|
|
2376
|
+
// The swap already opened a transaction, even though the
|
|
2377
|
+
// normal-stream transactionStarted flag is still false.
|
|
2378
|
+
true,
|
|
2379
|
+
(rowId) => {
|
|
2380
|
+
recordPendingPresence(rowId, false)
|
|
2381
|
+
syncedKeys.delete(rowId)
|
|
2382
|
+
},
|
|
2000
2383
|
)
|
|
2001
2384
|
} else if (isMoveInMessage(bufferedMsg)) {
|
|
2002
2385
|
// Process buffered move-in messages during atomic swap
|
|
@@ -2006,11 +2389,12 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
2006
2389
|
|
|
2007
2390
|
// Commit the atomic swap
|
|
2008
2391
|
stageResumeMetadata()
|
|
2009
|
-
applied =
|
|
2392
|
+
applied = commitSourceTransaction()
|
|
2010
2393
|
|
|
2011
2394
|
// Exit buffering phase by marking that we've received up-to-date
|
|
2012
2395
|
// isBufferingInitialSync() will now return false
|
|
2013
2396
|
bufferedMessages.length = 0
|
|
2397
|
+
progressiveSnapshotRows.clear()
|
|
2014
2398
|
|
|
2015
2399
|
debug(
|
|
2016
2400
|
`${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Atomic swap complete, now in normal sync mode`,
|
|
@@ -2019,13 +2403,15 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
2019
2403
|
// Normal mode or on-demand: commit transaction if one was started
|
|
2020
2404
|
// Both up-to-date and subset-end trigger a commit
|
|
2021
2405
|
if (transactionStarted) {
|
|
2022
|
-
|
|
2023
|
-
|
|
2406
|
+
if (!isResettingSnapshot || finishesReset) {
|
|
2407
|
+
stageResumeMetadata()
|
|
2408
|
+
}
|
|
2409
|
+
applied = commitSourceTransaction()
|
|
2024
2410
|
transactionStarted = false
|
|
2025
2411
|
} else if (commitPoint === `up-to-date` && metadata) {
|
|
2026
|
-
|
|
2412
|
+
beginSourceTransaction()
|
|
2027
2413
|
stageResumeMetadata()
|
|
2028
|
-
applied =
|
|
2414
|
+
applied = commitSourceTransaction()
|
|
2029
2415
|
}
|
|
2030
2416
|
}
|
|
2031
2417
|
const readyErrorVersion = streamErrorVersion
|
|
@@ -2035,51 +2421,90 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
2035
2421
|
void applied.then(
|
|
2036
2422
|
() =>
|
|
2037
2423
|
wrappedMarkReady(wasBufferingInitialSync, readyErrorVersion),
|
|
2038
|
-
() =>
|
|
2424
|
+
(error: unknown) => {
|
|
2425
|
+
if (!isActiveLifecycle() || abortController.signal.aborted) {
|
|
2426
|
+
return
|
|
2427
|
+
}
|
|
2428
|
+
streamErrorVersion++
|
|
2429
|
+
unsubscribeStream()
|
|
2430
|
+
abortController.abort()
|
|
2431
|
+
if (collection.status !== `error`) markError(error)
|
|
2432
|
+
},
|
|
2039
2433
|
)
|
|
2040
2434
|
}
|
|
2041
2435
|
|
|
2436
|
+
if (finishesReset) {
|
|
2437
|
+
const finishReset = () => {
|
|
2438
|
+
if (resetGeneration === finishingResetGeneration) {
|
|
2439
|
+
isResettingSnapshot = false
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
if (applied === true) {
|
|
2443
|
+
finishReset()
|
|
2444
|
+
} else {
|
|
2445
|
+
void applied.then(finishReset, () => undefined)
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2042
2449
|
// Track that we've received the first up-to-date for progressive mode
|
|
2043
2450
|
if (commitPoint === `up-to-date`) {
|
|
2044
2451
|
hasReceivedUpToDate = true
|
|
2045
2452
|
}
|
|
2046
2453
|
|
|
2047
|
-
//
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
)
|
|
2055
|
-
}
|
|
2056
|
-
newTxids.forEach((txid) => clonedSeen.add(txid))
|
|
2057
|
-
newTxids.clear()
|
|
2058
|
-
return clonedSeen
|
|
2059
|
-
})
|
|
2060
|
-
|
|
2061
|
-
// Always commit snapshots when we receive up-to-date, regardless of transaction state
|
|
2062
|
-
seenSnapshots.setState((currentSnapshots) => {
|
|
2063
|
-
const seen = [...currentSnapshots, ...newSnapshots]
|
|
2064
|
-
newSnapshots.forEach((snapshot) =>
|
|
2065
|
-
debug(
|
|
2066
|
-
`${collectionId ? `[${collectionId}] ` : ``}new snapshot synced from pg %o`,
|
|
2067
|
-
snapshot,
|
|
2068
|
-
),
|
|
2454
|
+
// Stream evidence is the acknowledgement boundary used by mutation
|
|
2455
|
+
// handlers. It must publish before a parked applied receipt or the
|
|
2456
|
+
// optimistic transaction and its acknowledgement can deadlock.
|
|
2457
|
+
if (newTxids.size > 0) {
|
|
2458
|
+
debug(
|
|
2459
|
+
`${collectionId ? `[${collectionId}] ` : ``}new txids synced from pg %O`,
|
|
2460
|
+
Array.from(newTxids),
|
|
2069
2461
|
)
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2462
|
+
}
|
|
2463
|
+
newSnapshots.forEach((snapshot) =>
|
|
2464
|
+
debug(
|
|
2465
|
+
`${collectionId ? `[${collectionId}] ` : ``}new snapshot synced from pg %o`,
|
|
2466
|
+
snapshot,
|
|
2467
|
+
),
|
|
2468
|
+
)
|
|
2469
|
+
lifecycle.publishEvidence(newTxids, newSnapshots)
|
|
2470
|
+
newTxids.clear()
|
|
2471
|
+
newSnapshots.length = 0
|
|
2472
|
+
lifecycle.commitMatches()
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2078
2475
|
|
|
2079
|
-
|
|
2476
|
+
unsubscribeStream = stream.subscribe((messages: Array<Message<T>>) => {
|
|
2477
|
+
if (!areResumeKeysReady) {
|
|
2478
|
+
pendingResumeBatches.push([...messages])
|
|
2479
|
+
return
|
|
2080
2480
|
}
|
|
2481
|
+
processMessages(messages)
|
|
2081
2482
|
})
|
|
2082
2483
|
|
|
2484
|
+
if (!areResumeKeysReady && resumeKeysPromise) {
|
|
2485
|
+
void resumeKeysPromise.then(
|
|
2486
|
+
() => {
|
|
2487
|
+
if (abortController.signal.aborted) return
|
|
2488
|
+
|
|
2489
|
+
areResumeKeysReady = true
|
|
2490
|
+
|
|
2491
|
+
const queuedBatches = pendingResumeBatches.splice(0)
|
|
2492
|
+
queuedBatches.forEach(processMessages)
|
|
2493
|
+
},
|
|
2494
|
+
(error: unknown) => {
|
|
2495
|
+
if (abortController.signal.aborted) return
|
|
2496
|
+
|
|
2497
|
+
pendingResumeBatches.length = 0
|
|
2498
|
+
resumeInvalid = true
|
|
2499
|
+
commitResetResumeMetadataImmediately()
|
|
2500
|
+
streamErrorVersion++
|
|
2501
|
+
unsubscribeStream()
|
|
2502
|
+
abortController.abort()
|
|
2503
|
+
markError(error)
|
|
2504
|
+
},
|
|
2505
|
+
)
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2083
2508
|
// Return the deduplicated loadSubset if available (on-demand or progressive mode)
|
|
2084
2509
|
// The loadSubset method is auto-bound, so it can be safely returned directly
|
|
2085
2510
|
return {
|
|
@@ -2093,13 +2518,12 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
2093
2518
|
unsubscribeStream()
|
|
2094
2519
|
// Abort the abort controller to stop the stream
|
|
2095
2520
|
abortController.abort()
|
|
2521
|
+
pendingResumeBatches.length = 0
|
|
2096
2522
|
// Reset deduplication tracking so collection can load fresh data if restarted
|
|
2097
2523
|
loadSubsetDedupe?.reset()
|
|
2098
|
-
|
|
2524
|
+
lifecycle.retire(lifecycleEpoch)
|
|
2099
2525
|
},
|
|
2100
2526
|
}
|
|
2101
2527
|
},
|
|
2102
|
-
// Expose the getSyncMetadata function
|
|
2103
|
-
getSyncMetadata,
|
|
2104
2528
|
}
|
|
2105
2529
|
}
|