@tanstack/electric-db-collection 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/electric.cjs +45 -4
- package/dist/cjs/electric.cjs.map +1 -1
- package/dist/cjs/errors.cjs.map +1 -1
- package/dist/cjs/index.cjs +9 -0
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +1 -1
- package/dist/cjs/sql-compiler.cjs +92 -0
- package/dist/cjs/sql-compiler.cjs.map +1 -1
- package/dist/esm/electric.js +46 -5
- package/dist/esm/electric.js.map +1 -1
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/index.js +4 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/sql-compiler.js +92 -0
- package/dist/esm/sql-compiler.js.map +1 -1
- package/package.json +33 -33
- package/src/electric.ts +133 -60
- package/src/errors.ts +1 -1
- package/src/index.ts +4 -2
- package/src/sql-compiler.ts +169 -10
package/package.json
CHANGED
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tanstack/electric-db-collection",
|
|
3
|
+
"version": "0.2.14",
|
|
3
4
|
"description": "ElectricSQL collection for TanStack DB",
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
"@tanstack/db": "0.5.11"
|
|
11
|
-
},
|
|
12
|
-
"devDependencies": {
|
|
13
|
-
"@types/debug": "^4.1.12",
|
|
14
|
-
"@types/pg": "^8.15.6",
|
|
15
|
-
"@vitest/coverage-istanbul": "^3.2.4",
|
|
16
|
-
"pg": "^8.16.3"
|
|
5
|
+
"author": "Kyle Mathews",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/TanStack/db.git",
|
|
10
|
+
"directory": "packages/electric-db-collection"
|
|
17
11
|
},
|
|
12
|
+
"homepage": "https://tanstack.com/db",
|
|
13
|
+
"keywords": [
|
|
14
|
+
"electric",
|
|
15
|
+
"sql",
|
|
16
|
+
"optimistic",
|
|
17
|
+
"typescript"
|
|
18
|
+
],
|
|
19
|
+
"type": "module",
|
|
20
|
+
"main": "dist/cjs/index.cjs",
|
|
21
|
+
"module": "dist/esm/index.js",
|
|
22
|
+
"types": "dist/esm/index.d.ts",
|
|
18
23
|
"exports": {
|
|
19
24
|
".": {
|
|
20
25
|
"import": {
|
|
@@ -28,34 +33,29 @@
|
|
|
28
33
|
},
|
|
29
34
|
"./package.json": "./package.json"
|
|
30
35
|
},
|
|
36
|
+
"sideEffects": false,
|
|
31
37
|
"files": [
|
|
32
38
|
"dist",
|
|
33
39
|
"src"
|
|
34
40
|
],
|
|
35
|
-
"
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
"
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"@electric-sql/client": "1.2.0",
|
|
43
|
+
"@standard-schema/spec": "^1.0.0",
|
|
44
|
+
"@tanstack/store": "^0.8.0",
|
|
45
|
+
"debug": "^4.4.3",
|
|
46
|
+
"@tanstack/db": "0.5.12"
|
|
47
|
+
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"@types/debug": "^4.1.12",
|
|
50
|
+
"@types/pg": "^8.15.6",
|
|
51
|
+
"@vitest/coverage-istanbul": "^3.2.4",
|
|
52
|
+
"pg": "^8.16.3"
|
|
43
53
|
},
|
|
44
|
-
"homepage": "https://tanstack.com/db",
|
|
45
|
-
"keywords": [
|
|
46
|
-
"electric",
|
|
47
|
-
"sql",
|
|
48
|
-
"optimistic",
|
|
49
|
-
"typescript"
|
|
50
|
-
],
|
|
51
|
-
"sideEffects": false,
|
|
52
|
-
"type": "module",
|
|
53
|
-
"types": "dist/esm/index.d.ts",
|
|
54
54
|
"scripts": {
|
|
55
55
|
"build": "vite build",
|
|
56
56
|
"dev": "vite build --watch",
|
|
57
57
|
"lint": "eslint . --fix",
|
|
58
|
-
"test": "
|
|
59
|
-
"test:e2e": "
|
|
58
|
+
"test": "vitest run",
|
|
59
|
+
"test:e2e": "vitest run --config vitest.e2e.config.ts"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/src/electric.ts
CHANGED
|
@@ -3,17 +3,17 @@ import {
|
|
|
3
3
|
isChangeMessage,
|
|
4
4
|
isControlMessage,
|
|
5
5
|
isVisibleInSnapshot,
|
|
6
|
-
} from
|
|
7
|
-
import { Store } from
|
|
8
|
-
import DebugModule from
|
|
9
|
-
import { DeduplicatedLoadSubset } from
|
|
6
|
+
} from '@electric-sql/client'
|
|
7
|
+
import { Store } from '@tanstack/store'
|
|
8
|
+
import DebugModule from 'debug'
|
|
9
|
+
import { DeduplicatedLoadSubset, and } from '@tanstack/db'
|
|
10
10
|
import {
|
|
11
11
|
ExpectedNumberInAwaitTxIdError,
|
|
12
12
|
StreamAbortedError,
|
|
13
13
|
TimeoutWaitingForMatchError,
|
|
14
14
|
TimeoutWaitingForTxIdError,
|
|
15
|
-
} from
|
|
16
|
-
import { compileSQL } from
|
|
15
|
+
} from './errors'
|
|
16
|
+
import { compileSQL } from './sql-compiler'
|
|
17
17
|
import type {
|
|
18
18
|
BaseCollectionConfig,
|
|
19
19
|
CollectionConfig,
|
|
@@ -24,8 +24,8 @@ import type {
|
|
|
24
24
|
SyncMode,
|
|
25
25
|
UpdateMutationFnParams,
|
|
26
26
|
UtilsRecord,
|
|
27
|
-
} from
|
|
28
|
-
import type { StandardSchemaV1 } from
|
|
27
|
+
} from '@tanstack/db'
|
|
28
|
+
import type { StandardSchemaV1 } from '@standard-schema/spec'
|
|
29
29
|
import type {
|
|
30
30
|
ControlMessage,
|
|
31
31
|
GetExtensions,
|
|
@@ -33,10 +33,10 @@ import type {
|
|
|
33
33
|
PostgresSnapshot,
|
|
34
34
|
Row,
|
|
35
35
|
ShapeStreamOptions,
|
|
36
|
-
} from
|
|
36
|
+
} from '@electric-sql/client'
|
|
37
37
|
|
|
38
38
|
// Re-export for user convenience in custom match functions
|
|
39
|
-
export { isChangeMessage, isControlMessage } from
|
|
39
|
+
export { isChangeMessage, isControlMessage } from '@electric-sql/client'
|
|
40
40
|
|
|
41
41
|
const debug = DebugModule.debug(`ts/db:electric`)
|
|
42
42
|
|
|
@@ -66,7 +66,7 @@ export type Txid = number
|
|
|
66
66
|
* indicating if the mutation has been synchronized
|
|
67
67
|
*/
|
|
68
68
|
export type MatchFunction<T extends Row<unknown>> = (
|
|
69
|
-
message: Message<T
|
|
69
|
+
message: Message<T>,
|
|
70
70
|
) => boolean
|
|
71
71
|
|
|
72
72
|
/**
|
|
@@ -197,7 +197,7 @@ export interface ElectricCollectionConfig<
|
|
|
197
197
|
T,
|
|
198
198
|
string | number,
|
|
199
199
|
ElectricCollectionUtils<T>
|
|
200
|
-
|
|
200
|
+
>,
|
|
201
201
|
) => Promise<MatchingStrategy>
|
|
202
202
|
|
|
203
203
|
/**
|
|
@@ -232,7 +232,7 @@ export interface ElectricCollectionConfig<
|
|
|
232
232
|
T,
|
|
233
233
|
string | number,
|
|
234
234
|
ElectricCollectionUtils<T>
|
|
235
|
-
|
|
235
|
+
>,
|
|
236
236
|
) => Promise<MatchingStrategy>
|
|
237
237
|
|
|
238
238
|
/**
|
|
@@ -266,24 +266,24 @@ export interface ElectricCollectionConfig<
|
|
|
266
266
|
T,
|
|
267
267
|
string | number,
|
|
268
268
|
ElectricCollectionUtils<T>
|
|
269
|
-
|
|
269
|
+
>,
|
|
270
270
|
) => Promise<MatchingStrategy>
|
|
271
271
|
}
|
|
272
272
|
|
|
273
273
|
function isUpToDateMessage<T extends Row<unknown>>(
|
|
274
|
-
message: Message<T
|
|
274
|
+
message: Message<T>,
|
|
275
275
|
): message is ControlMessage & { up_to_date: true } {
|
|
276
276
|
return isControlMessage(message) && message.headers.control === `up-to-date`
|
|
277
277
|
}
|
|
278
278
|
|
|
279
279
|
function isMustRefetchMessage<T extends Row<unknown>>(
|
|
280
|
-
message: Message<T
|
|
280
|
+
message: Message<T>,
|
|
281
281
|
): message is ControlMessage & { headers: { control: `must-refetch` } } {
|
|
282
282
|
return isControlMessage(message) && message.headers.control === `must-refetch`
|
|
283
283
|
}
|
|
284
284
|
|
|
285
285
|
function isSnapshotEndMessage<T extends Row<unknown>>(
|
|
286
|
-
message: Message<T
|
|
286
|
+
message: Message<T>,
|
|
287
287
|
): message is SnapshotEndMessage {
|
|
288
288
|
return isControlMessage(message) && message.headers.control === `snapshot-end`
|
|
289
289
|
}
|
|
@@ -298,7 +298,7 @@ function parseSnapshotMessage(message: SnapshotEndMessage): PostgresSnapshot {
|
|
|
298
298
|
|
|
299
299
|
// Check if a message contains txids in its headers
|
|
300
300
|
function hasTxids<T extends Row<unknown>>(
|
|
301
|
-
message: Message<T
|
|
301
|
+
message: Message<T>,
|
|
302
302
|
): message is Message<T> & { headers: { txids?: Array<Txid> } } {
|
|
303
303
|
return `txids` in message.headers && Array.isArray(message.headers.txids)
|
|
304
304
|
}
|
|
@@ -307,7 +307,12 @@ function hasTxids<T extends Row<unknown>>(
|
|
|
307
307
|
* Creates a deduplicated loadSubset handler for progressive/on-demand modes
|
|
308
308
|
* Returns null for eager mode, or a DeduplicatedLoadSubset instance for other modes.
|
|
309
309
|
* Handles fetching snapshots in progressive mode during buffering phase,
|
|
310
|
-
* and requesting snapshots in on-demand mode
|
|
310
|
+
* and requesting snapshots in on-demand mode.
|
|
311
|
+
*
|
|
312
|
+
* When cursor expressions are provided (whereFrom/whereCurrent), makes two
|
|
313
|
+
* requestSnapshot calls:
|
|
314
|
+
* - One for whereFrom (rows > cursor) with limit
|
|
315
|
+
* - One for whereCurrent (rows = cursor, for tie-breaking) without limit
|
|
311
316
|
*/
|
|
312
317
|
function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
313
318
|
stream,
|
|
@@ -347,7 +352,7 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
347
352
|
// and completed the atomic swap while waiting for the snapshot
|
|
348
353
|
if (!isBufferingInitialSync()) {
|
|
349
354
|
debug(
|
|
350
|
-
`${collectionId ? `[${collectionId}] ` : ``}Ignoring snapshot - sync completed while fetching
|
|
355
|
+
`${collectionId ? `[${collectionId}] ` : ``}Ignoring snapshot - sync completed while fetching`,
|
|
351
356
|
)
|
|
352
357
|
return
|
|
353
358
|
}
|
|
@@ -367,13 +372,13 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
367
372
|
commit()
|
|
368
373
|
|
|
369
374
|
debug(
|
|
370
|
-
`${collectionId ? `[${collectionId}] ` : ``}Applied snapshot with ${rows.length} rows
|
|
375
|
+
`${collectionId ? `[${collectionId}] ` : ``}Applied snapshot with ${rows.length} rows`,
|
|
371
376
|
)
|
|
372
377
|
}
|
|
373
378
|
} catch (error) {
|
|
374
379
|
debug(
|
|
375
380
|
`${collectionId ? `[${collectionId}] ` : ``}Error fetching snapshot: %o`,
|
|
376
|
-
error
|
|
381
|
+
error,
|
|
377
382
|
)
|
|
378
383
|
throw error
|
|
379
384
|
}
|
|
@@ -382,8 +387,50 @@ function createLoadSubsetDedupe<T extends Row<unknown>>({
|
|
|
382
387
|
return
|
|
383
388
|
} else {
|
|
384
389
|
// On-demand mode: use requestSnapshot
|
|
385
|
-
|
|
386
|
-
|
|
390
|
+
// When cursor is provided, make two calls:
|
|
391
|
+
// 1. whereCurrent (all ties, no limit)
|
|
392
|
+
// 2. whereFrom (rows > cursor, with limit)
|
|
393
|
+
const { cursor, where, orderBy, limit } = opts
|
|
394
|
+
|
|
395
|
+
if (cursor) {
|
|
396
|
+
// Make parallel requests for cursor-based pagination
|
|
397
|
+
const promises: Array<Promise<unknown>> = []
|
|
398
|
+
|
|
399
|
+
// Request 1: All rows matching whereCurrent (ties at boundary, no limit)
|
|
400
|
+
// Combine main where with cursor.whereCurrent
|
|
401
|
+
const whereCurrentOpts: LoadSubsetOptions = {
|
|
402
|
+
where: where ? and(where, cursor.whereCurrent) : cursor.whereCurrent,
|
|
403
|
+
orderBy,
|
|
404
|
+
// No limit - get all ties
|
|
405
|
+
}
|
|
406
|
+
const whereCurrentParams = compileSQL<T>(whereCurrentOpts)
|
|
407
|
+
promises.push(stream.requestSnapshot(whereCurrentParams))
|
|
408
|
+
|
|
409
|
+
debug(
|
|
410
|
+
`${collectionId ? `[${collectionId}] ` : ``}Requesting cursor.whereCurrent snapshot (all ties)`,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
// Request 2: Rows matching whereFrom (rows > cursor, with limit)
|
|
414
|
+
// Combine main where with cursor.whereFrom
|
|
415
|
+
const whereFromOpts: LoadSubsetOptions = {
|
|
416
|
+
where: where ? and(where, cursor.whereFrom) : cursor.whereFrom,
|
|
417
|
+
orderBy,
|
|
418
|
+
limit,
|
|
419
|
+
}
|
|
420
|
+
const whereFromParams = compileSQL<T>(whereFromOpts)
|
|
421
|
+
promises.push(stream.requestSnapshot(whereFromParams))
|
|
422
|
+
|
|
423
|
+
debug(
|
|
424
|
+
`${collectionId ? `[${collectionId}] ` : ``}Requesting cursor.whereFrom snapshot (with limit ${limit})`,
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
// Wait for both requests to complete
|
|
428
|
+
await Promise.all(promises)
|
|
429
|
+
} else {
|
|
430
|
+
// No cursor - standard single request
|
|
431
|
+
const snapshotParams = compileSQL<T>(opts)
|
|
432
|
+
await stream.requestSnapshot(snapshotParams)
|
|
433
|
+
}
|
|
387
434
|
}
|
|
388
435
|
}
|
|
389
436
|
|
|
@@ -400,7 +447,7 @@ export type AwaitTxIdFn = (txId: Txid, timeout?: number) => Promise<boolean>
|
|
|
400
447
|
*/
|
|
401
448
|
export type AwaitMatchFn<T extends Row<unknown>> = (
|
|
402
449
|
matchFn: MatchFunction<T>,
|
|
403
|
-
timeout?: number
|
|
450
|
+
timeout?: number,
|
|
404
451
|
) => Promise<boolean>
|
|
405
452
|
|
|
406
453
|
/**
|
|
@@ -427,7 +474,7 @@ export interface ElectricCollectionUtils<
|
|
|
427
474
|
export function electricCollectionOptions<T extends StandardSchemaV1>(
|
|
428
475
|
config: ElectricCollectionConfig<InferSchemaOutput<T>, T> & {
|
|
429
476
|
schema: T
|
|
430
|
-
}
|
|
477
|
+
},
|
|
431
478
|
): Omit<CollectionConfig<InferSchemaOutput<T>, string | number, T>, `utils`> & {
|
|
432
479
|
id?: string
|
|
433
480
|
utils: ElectricCollectionUtils<InferSchemaOutput<T>>
|
|
@@ -438,7 +485,7 @@ export function electricCollectionOptions<T extends StandardSchemaV1>(
|
|
|
438
485
|
export function electricCollectionOptions<T extends Row<unknown>>(
|
|
439
486
|
config: ElectricCollectionConfig<T> & {
|
|
440
487
|
schema?: never // prohibit schema
|
|
441
|
-
}
|
|
488
|
+
},
|
|
442
489
|
): Omit<CollectionConfig<T, string | number>, `utils`> & {
|
|
443
490
|
id?: string
|
|
444
491
|
utils: ElectricCollectionUtils<T>
|
|
@@ -446,7 +493,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
446
493
|
}
|
|
447
494
|
|
|
448
495
|
export function electricCollectionOptions<T extends Row<unknown>>(
|
|
449
|
-
config: ElectricCollectionConfig<T, any
|
|
496
|
+
config: ElectricCollectionConfig<T, any>,
|
|
450
497
|
): Omit<
|
|
451
498
|
CollectionConfig<T, string | number, any, ElectricCollectionUtils<T>>,
|
|
452
499
|
`utils`
|
|
@@ -476,6 +523,10 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
476
523
|
// Buffer messages since last up-to-date to handle race conditions
|
|
477
524
|
const currentBatchMessages = new Store<Array<Message<any>>>([])
|
|
478
525
|
|
|
526
|
+
// Track whether the current batch has been committed (up-to-date received)
|
|
527
|
+
// This allows awaitMatch to resolve immediately for messages from committed batches
|
|
528
|
+
const batchCommitted = new Store<boolean>(false)
|
|
529
|
+
|
|
479
530
|
/**
|
|
480
531
|
* Helper function to remove multiple matches from the pendingMatches store
|
|
481
532
|
*/
|
|
@@ -501,7 +552,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
501
552
|
matchesToResolve.push(matchId)
|
|
502
553
|
debug(
|
|
503
554
|
`${config.id ? `[${config.id}] ` : ``}awaitMatch resolved on up-to-date for match %s`,
|
|
504
|
-
matchId
|
|
555
|
+
matchId,
|
|
505
556
|
)
|
|
506
557
|
}
|
|
507
558
|
})
|
|
@@ -513,6 +564,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
513
564
|
syncMode: internalSyncMode,
|
|
514
565
|
pendingMatches,
|
|
515
566
|
currentBatchMessages,
|
|
567
|
+
batchCommitted,
|
|
516
568
|
removePendingMatches,
|
|
517
569
|
resolveMatchedPendingMatches,
|
|
518
570
|
collectionId: config.id,
|
|
@@ -527,11 +579,11 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
527
579
|
*/
|
|
528
580
|
const awaitTxId: AwaitTxIdFn = async (
|
|
529
581
|
txId: Txid,
|
|
530
|
-
timeout: number = 5000
|
|
582
|
+
timeout: number = 5000,
|
|
531
583
|
): Promise<boolean> => {
|
|
532
584
|
debug(
|
|
533
585
|
`${config.id ? `[${config.id}] ` : ``}awaitTxId called with txid %d`,
|
|
534
|
-
txId
|
|
586
|
+
txId,
|
|
535
587
|
)
|
|
536
588
|
if (typeof txId !== `number`) {
|
|
537
589
|
throw new ExpectedNumberInAwaitTxIdError(typeof txId, config.id)
|
|
@@ -543,7 +595,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
543
595
|
|
|
544
596
|
// Then check if the txid is in any of the seen snapshots
|
|
545
597
|
const hasSnapshot = seenSnapshots.state.some((snapshot) =>
|
|
546
|
-
isVisibleInSnapshot(txId, snapshot)
|
|
598
|
+
isVisibleInSnapshot(txId, snapshot),
|
|
547
599
|
)
|
|
548
600
|
if (hasSnapshot) return true
|
|
549
601
|
|
|
@@ -558,7 +610,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
558
610
|
if (seenTxids.state.has(txId)) {
|
|
559
611
|
debug(
|
|
560
612
|
`${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o`,
|
|
561
|
-
txId
|
|
613
|
+
txId,
|
|
562
614
|
)
|
|
563
615
|
clearTimeout(timeoutId)
|
|
564
616
|
unsubscribeSeenTxids()
|
|
@@ -569,13 +621,13 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
569
621
|
|
|
570
622
|
const unsubscribeSeenSnapshots = seenSnapshots.subscribe(() => {
|
|
571
623
|
const visibleSnapshot = seenSnapshots.state.find((snapshot) =>
|
|
572
|
-
isVisibleInSnapshot(txId, snapshot)
|
|
624
|
+
isVisibleInSnapshot(txId, snapshot),
|
|
573
625
|
)
|
|
574
626
|
if (visibleSnapshot) {
|
|
575
627
|
debug(
|
|
576
628
|
`${config.id ? `[${config.id}] ` : ``}awaitTxId found match for txid %o in snapshot %o`,
|
|
577
629
|
txId,
|
|
578
|
-
visibleSnapshot
|
|
630
|
+
visibleSnapshot,
|
|
579
631
|
)
|
|
580
632
|
clearTimeout(timeoutId)
|
|
581
633
|
unsubscribeSeenSnapshots()
|
|
@@ -594,10 +646,10 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
594
646
|
*/
|
|
595
647
|
const awaitMatch: AwaitMatchFn<any> = async (
|
|
596
648
|
matchFn: MatchFunction<any>,
|
|
597
|
-
timeout: number = 3000
|
|
649
|
+
timeout: number = 3000,
|
|
598
650
|
): Promise<boolean> => {
|
|
599
651
|
debug(
|
|
600
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch called with custom function
|
|
652
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch called with custom function`,
|
|
601
653
|
)
|
|
602
654
|
|
|
603
655
|
return new Promise((resolve, reject) => {
|
|
@@ -623,7 +675,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
623
675
|
const checkMatch = (message: Message<any>) => {
|
|
624
676
|
if (matchFn(message)) {
|
|
625
677
|
debug(
|
|
626
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch found matching message, waiting for up-to-date
|
|
678
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch found matching message, waiting for up-to-date`,
|
|
627
679
|
)
|
|
628
680
|
// Mark as matched but don't resolve yet - wait for up-to-date
|
|
629
681
|
pendingMatches.setState((current) => {
|
|
@@ -642,10 +694,21 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
642
694
|
// Check against current batch messages first to handle race conditions
|
|
643
695
|
for (const message of currentBatchMessages.state) {
|
|
644
696
|
if (matchFn(message)) {
|
|
697
|
+
// If batch is committed (up-to-date already received), resolve immediately
|
|
698
|
+
// just like awaitTxId does when it finds a txid in seenTxids
|
|
699
|
+
if (batchCommitted.state) {
|
|
700
|
+
debug(
|
|
701
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in committed batch, resolving immediately`,
|
|
702
|
+
)
|
|
703
|
+
clearTimeout(timeoutId)
|
|
704
|
+
resolve(true)
|
|
705
|
+
return
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
// If batch is not yet committed, register match and wait for up-to-date
|
|
645
709
|
debug(
|
|
646
|
-
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in current batch, waiting for up-to-date
|
|
710
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in current batch, waiting for up-to-date`,
|
|
647
711
|
)
|
|
648
|
-
// Register match as already matched
|
|
649
712
|
pendingMatches.setState((current) => {
|
|
650
713
|
const newMatches = new Map(current)
|
|
651
714
|
newMatches.set(matchId, {
|
|
@@ -653,7 +716,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
653
716
|
resolve,
|
|
654
717
|
reject,
|
|
655
718
|
timeoutId,
|
|
656
|
-
matched: true, // Already matched
|
|
719
|
+
matched: true, // Already matched, will resolve on up-to-date
|
|
657
720
|
})
|
|
658
721
|
return newMatches
|
|
659
722
|
})
|
|
@@ -681,7 +744,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
681
744
|
* Process matching strategy and wait for synchronization
|
|
682
745
|
*/
|
|
683
746
|
const processMatchingStrategy = async (
|
|
684
|
-
result: MatchingStrategy
|
|
747
|
+
result: MatchingStrategy,
|
|
685
748
|
): Promise<void> => {
|
|
686
749
|
// Only wait if result contains txid
|
|
687
750
|
if (result && `txid` in result) {
|
|
@@ -703,7 +766,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
703
766
|
any,
|
|
704
767
|
string | number,
|
|
705
768
|
ElectricCollectionUtils<T>
|
|
706
|
-
|
|
769
|
+
>,
|
|
707
770
|
) => {
|
|
708
771
|
const handlerResult = await config.onInsert!(params)
|
|
709
772
|
await processMatchingStrategy(handlerResult)
|
|
@@ -717,7 +780,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
717
780
|
any,
|
|
718
781
|
string | number,
|
|
719
782
|
ElectricCollectionUtils<T>
|
|
720
|
-
|
|
783
|
+
>,
|
|
721
784
|
) => {
|
|
722
785
|
const handlerResult = await config.onUpdate!(params)
|
|
723
786
|
await processMatchingStrategy(handlerResult)
|
|
@@ -731,7 +794,7 @@ export function electricCollectionOptions<T extends Row<unknown>>(
|
|
|
731
794
|
any,
|
|
732
795
|
string | number,
|
|
733
796
|
ElectricCollectionUtils<T>
|
|
734
|
-
|
|
797
|
+
>,
|
|
735
798
|
) => {
|
|
736
799
|
const handlerResult = await config.onDelete!(params)
|
|
737
800
|
await processMatchingStrategy(handlerResult)
|
|
@@ -784,11 +847,12 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
784
847
|
>
|
|
785
848
|
>
|
|
786
849
|
currentBatchMessages: Store<Array<Message<T>>>
|
|
850
|
+
batchCommitted: Store<boolean>
|
|
787
851
|
removePendingMatches: (matchIds: Array<string>) => void
|
|
788
852
|
resolveMatchedPendingMatches: () => void
|
|
789
853
|
collectionId?: string
|
|
790
854
|
testHooks?: ElectricTestHooks
|
|
791
|
-
}
|
|
855
|
+
},
|
|
792
856
|
): SyncConfig<T> {
|
|
793
857
|
const {
|
|
794
858
|
seenTxids,
|
|
@@ -796,6 +860,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
796
860
|
syncMode,
|
|
797
861
|
pendingMatches,
|
|
798
862
|
currentBatchMessages,
|
|
863
|
+
batchCommitted,
|
|
799
864
|
removePendingMatches,
|
|
800
865
|
resolveMatchedPendingMatches,
|
|
801
866
|
collectionId,
|
|
@@ -858,7 +923,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
858
923
|
},
|
|
859
924
|
{
|
|
860
925
|
once: true,
|
|
861
|
-
}
|
|
926
|
+
},
|
|
862
927
|
)
|
|
863
928
|
if (shapeOptions.signal.aborted) {
|
|
864
929
|
abortController.abort()
|
|
@@ -900,7 +965,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
900
965
|
`An error occurred while syncing collection: ${collection.id}, \n` +
|
|
901
966
|
`it has been marked as ready to avoid blocking apps waiting for '.preload()' to finish. \n` +
|
|
902
967
|
`You can provide an 'onError' handler on the shapeOptions to handle this error, and this message will not be logged.`,
|
|
903
|
-
errorParams
|
|
968
|
+
errorParams,
|
|
904
969
|
)
|
|
905
970
|
}
|
|
906
971
|
|
|
@@ -935,6 +1000,12 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
935
1000
|
let hasUpToDate = false
|
|
936
1001
|
let hasSnapshotEnd = false
|
|
937
1002
|
|
|
1003
|
+
// Clear the current batch buffer at the START of processing a new batch
|
|
1004
|
+
// This preserves messages from the previous batch until new ones arrive,
|
|
1005
|
+
// allowing awaitMatch to find messages even if called after up-to-date
|
|
1006
|
+
currentBatchMessages.setState(() => [])
|
|
1007
|
+
batchCommitted.setState(() => false)
|
|
1008
|
+
|
|
938
1009
|
for (const message of messages) {
|
|
939
1010
|
// Add message to current batch buffer (for race condition handling)
|
|
940
1011
|
if (isChangeMessage(message)) {
|
|
@@ -965,7 +1036,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
965
1036
|
// If matchFn throws, clean up and reject the promise
|
|
966
1037
|
clearTimeout(match.timeoutId)
|
|
967
1038
|
match.reject(
|
|
968
|
-
err instanceof Error ? err : new Error(String(err))
|
|
1039
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
969
1040
|
)
|
|
970
1041
|
matchesToRemove.push(matchId)
|
|
971
1042
|
debug(`matchFn error: %o`, err)
|
|
@@ -1013,7 +1084,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1013
1084
|
hasUpToDate = true
|
|
1014
1085
|
} else if (isMustRefetchMessage(message)) {
|
|
1015
1086
|
debug(
|
|
1016
|
-
`${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate
|
|
1087
|
+
`${collectionId ? `[${collectionId}] ` : ``}Received must-refetch message, starting transaction with truncate`,
|
|
1017
1088
|
)
|
|
1018
1089
|
|
|
1019
1090
|
// Start a transaction and truncate the collection
|
|
@@ -1040,7 +1111,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1040
1111
|
// PROGRESSIVE MODE: Atomic swap on first up-to-date
|
|
1041
1112
|
if (isBufferingInitialSync() && hasUpToDate) {
|
|
1042
1113
|
debug(
|
|
1043
|
-
`${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Performing atomic swap with ${bufferedMessages.length} buffered messages
|
|
1114
|
+
`${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Performing atomic swap with ${bufferedMessages.length} buffered messages`,
|
|
1044
1115
|
)
|
|
1045
1116
|
|
|
1046
1117
|
// Start atomic swap transaction
|
|
@@ -1063,7 +1134,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1063
1134
|
// Extract txids from buffered messages (will be committed to store after transaction)
|
|
1064
1135
|
if (hasTxids(bufferedMsg)) {
|
|
1065
1136
|
bufferedMsg.headers.txids?.forEach((txid) =>
|
|
1066
|
-
newTxids.add(txid)
|
|
1137
|
+
newTxids.add(txid),
|
|
1067
1138
|
)
|
|
1068
1139
|
}
|
|
1069
1140
|
} else if (isSnapshotEndMessage(bufferedMsg)) {
|
|
@@ -1080,7 +1151,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1080
1151
|
bufferedMessages.length = 0
|
|
1081
1152
|
|
|
1082
1153
|
debug(
|
|
1083
|
-
`${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Atomic swap complete, now in normal sync mode
|
|
1154
|
+
`${collectionId ? `[${collectionId}] ` : ``}Progressive mode: Atomic swap complete, now in normal sync mode`,
|
|
1084
1155
|
)
|
|
1085
1156
|
} else {
|
|
1086
1157
|
// Normal mode or on-demand: commit transaction if one was started
|
|
@@ -1096,9 +1167,6 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1096
1167
|
}
|
|
1097
1168
|
}
|
|
1098
1169
|
|
|
1099
|
-
// Clear the current batch buffer since we're now up-to-date
|
|
1100
|
-
currentBatchMessages.setState(() => [])
|
|
1101
|
-
|
|
1102
1170
|
if (hasUpToDate || (hasSnapshotEnd && syncMode === `on-demand`)) {
|
|
1103
1171
|
// Mark the collection as ready now that sync is up to date
|
|
1104
1172
|
wrappedMarkReady(isBufferingInitialSync())
|
|
@@ -1115,7 +1183,7 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1115
1183
|
if (newTxids.size > 0) {
|
|
1116
1184
|
debug(
|
|
1117
1185
|
`${collectionId ? `[${collectionId}] ` : ``}new txids synced from pg %O`,
|
|
1118
|
-
Array.from(newTxids)
|
|
1186
|
+
Array.from(newTxids),
|
|
1119
1187
|
)
|
|
1120
1188
|
}
|
|
1121
1189
|
newTxids.forEach((txid) => clonedSeen.add(txid))
|
|
@@ -1129,14 +1197,19 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
1129
1197
|
newSnapshots.forEach((snapshot) =>
|
|
1130
1198
|
debug(
|
|
1131
1199
|
`${collectionId ? `[${collectionId}] ` : ``}new snapshot synced from pg %o`,
|
|
1132
|
-
snapshot
|
|
1133
|
-
)
|
|
1200
|
+
snapshot,
|
|
1201
|
+
),
|
|
1134
1202
|
)
|
|
1135
1203
|
newSnapshots.length = 0
|
|
1136
1204
|
return seen
|
|
1137
1205
|
})
|
|
1138
1206
|
|
|
1139
|
-
// Resolve all matched pending matches on up-to-date
|
|
1207
|
+
// Resolve all matched pending matches on up-to-date or snapshot-end in on-demand mode
|
|
1208
|
+
// Set batchCommitted BEFORE resolving to avoid timing window where late awaitMatch
|
|
1209
|
+
// calls could register as "matched" after resolver pass already ran
|
|
1210
|
+
if (hasUpToDate || (hasSnapshotEnd && syncMode === `on-demand`)) {
|
|
1211
|
+
batchCommitted.setState(() => true)
|
|
1212
|
+
}
|
|
1140
1213
|
resolveMatchedPendingMatches()
|
|
1141
1214
|
}
|
|
1142
1215
|
})
|
package/src/errors.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
export {
|
|
2
2
|
electricCollectionOptions,
|
|
3
|
+
isChangeMessage,
|
|
4
|
+
isControlMessage,
|
|
3
5
|
type ElectricCollectionConfig,
|
|
4
6
|
type ElectricCollectionUtils,
|
|
5
7
|
type Txid,
|
|
6
8
|
type AwaitTxIdFn,
|
|
7
|
-
} from
|
|
9
|
+
} from './electric'
|
|
8
10
|
|
|
9
|
-
export * from
|
|
11
|
+
export * from './errors'
|