@tanstack/db-sqlite-persistence-core 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -3
- package/dist/cjs/broadcast-coordinator.cjs +282 -43
- package/dist/cjs/broadcast-coordinator.cjs.map +1 -1
- package/dist/cjs/broadcast-coordinator.d.cts +25 -8
- package/dist/cjs/committed-tx-anchor.cjs +9 -0
- package/dist/cjs/committed-tx-anchor.cjs.map +1 -0
- package/dist/cjs/committed-tx-anchor.d.cts +1 -0
- package/dist/cjs/index.cjs +4 -0
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/persisted.cjs +1599 -313
- package/dist/cjs/persisted.cjs.map +1 -1
- package/dist/cjs/persisted.d.cts +106 -13
- package/dist/cjs/remote-subset-wire.cjs +11 -0
- package/dist/cjs/remote-subset-wire.cjs.map +1 -1
- package/dist/cjs/remote-subset-wire.d.cts +1 -0
- package/dist/cjs/sqlite-core-adapter.cjs +705 -64
- package/dist/cjs/sqlite-core-adapter.cjs.map +1 -1
- package/dist/cjs/sqlite-core-adapter.d.cts +53 -10
- package/dist/esm/broadcast-coordinator.d.ts +25 -8
- package/dist/esm/broadcast-coordinator.js +283 -44
- package/dist/esm/broadcast-coordinator.js.map +1 -1
- package/dist/esm/committed-tx-anchor.d.ts +1 -0
- package/dist/esm/committed-tx-anchor.js +9 -0
- package/dist/esm/committed-tx-anchor.js.map +1 -0
- package/dist/esm/index.js +6 -2
- package/dist/esm/persisted.d.ts +106 -13
- package/dist/esm/persisted.js +1601 -315
- package/dist/esm/persisted.js.map +1 -1
- package/dist/esm/remote-subset-wire.d.ts +1 -0
- package/dist/esm/remote-subset-wire.js +11 -0
- package/dist/esm/remote-subset-wire.js.map +1 -1
- package/dist/esm/sqlite-core-adapter.d.ts +53 -10
- package/dist/esm/sqlite-core-adapter.js +708 -67
- package/dist/esm/sqlite-core-adapter.js.map +1 -1
- package/package.json +2 -2
- package/src/broadcast-coordinator.ts +415 -62
- package/src/committed-tx-anchor.ts +13 -0
- package/src/persisted.ts +2283 -370
- package/src/remote-subset-wire.ts +12 -0
- package/src/sqlite-core-adapter.ts +891 -68
package/src/persisted.ts
CHANGED
|
@@ -5,7 +5,9 @@ import {
|
|
|
5
5
|
SYNC_PERSISTENCE_PROTOCOL,
|
|
6
6
|
SYNC_PERSISTENCE_VERSION,
|
|
7
7
|
SyncTransactionAbortedError,
|
|
8
|
+
collectionOptionsClaim,
|
|
8
9
|
compileSingleRowExpression,
|
|
10
|
+
equalPersistedSnapshotValues,
|
|
9
11
|
getLoadSubsetDemandKey,
|
|
10
12
|
safeRandomUUID,
|
|
11
13
|
toBooleanPredicate,
|
|
@@ -15,6 +17,7 @@ import {
|
|
|
15
17
|
} from '@tanstack/db'
|
|
16
18
|
import {
|
|
17
19
|
DuplicateRemoteSubsetOwnerError,
|
|
20
|
+
IndeterminateCommitError,
|
|
18
21
|
InvalidPersistedCollectionConfigError,
|
|
19
22
|
InvalidPersistedCollectionCoordinatorError,
|
|
20
23
|
InvalidPersistedStorageKeyEncodingError,
|
|
@@ -115,6 +118,7 @@ export type TxCommitted = {
|
|
|
115
118
|
} & (
|
|
116
119
|
| {
|
|
117
120
|
requiresFullReload: true
|
|
121
|
+
reconciliationReload?: true
|
|
118
122
|
}
|
|
119
123
|
| {
|
|
120
124
|
requiresFullReload: false
|
|
@@ -238,6 +242,38 @@ export type ApplyCommittedTxResponse =
|
|
|
238
242
|
path?: string | ReadonlyArray<string | number>
|
|
239
243
|
}
|
|
240
244
|
|
|
245
|
+
/** The durable version read before a source transaction enters coordination. */
|
|
246
|
+
export type CommittedTxAnchor = {
|
|
247
|
+
latestRowVersion: number
|
|
248
|
+
resetEpoch: number
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export type ReconcileCommittedTxResponse =
|
|
252
|
+
| {
|
|
253
|
+
type: `rpc:reconcileCommittedTx:res`
|
|
254
|
+
rpcId: string
|
|
255
|
+
ok: true
|
|
256
|
+
alreadyApplied: boolean
|
|
257
|
+
committed: { term: number; seq: number; rowVersion: number }
|
|
258
|
+
}
|
|
259
|
+
| {
|
|
260
|
+
type: `rpc:reconcileCommittedTx:res`
|
|
261
|
+
rpcId: string
|
|
262
|
+
ok: false
|
|
263
|
+
code: `NOT_LEADER` | `UNDETERMINED` | `PERSISTENCE_ERROR`
|
|
264
|
+
error: string
|
|
265
|
+
sourceCode?: string | number
|
|
266
|
+
path?: string | ReadonlyArray<string | number>
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export type ReconciledCommittedTx =
|
|
270
|
+
| {
|
|
271
|
+
kind: `already-applied` | `applied-now`
|
|
272
|
+
committed: { term: number; seq: number; rowVersion: number }
|
|
273
|
+
latestRowVersion: number
|
|
274
|
+
}
|
|
275
|
+
| { kind: `unknown` }
|
|
276
|
+
|
|
241
277
|
export type PullSinceRequest = {
|
|
242
278
|
type: `rpc:pullSince:req`
|
|
243
279
|
rpcId: string
|
|
@@ -319,6 +355,97 @@ export type PersistedRowScanOptions = SyncPersistenceScanOptions
|
|
|
319
355
|
|
|
320
356
|
export type PersistedKeySetEvidence = SyncPersistenceKeySetEvidence
|
|
321
357
|
|
|
358
|
+
export type PersistedCacheGenerationClaim = {
|
|
359
|
+
storageCollectionId: string
|
|
360
|
+
claimId: string
|
|
361
|
+
/** Present when the adapter expires dormant claims. */
|
|
362
|
+
expiresAtMs?: number
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** An eager Collection name or a claim-bound persisted cache generation. */
|
|
366
|
+
export type PersistedStorageTarget =
|
|
367
|
+
| { kind: `eager`; collectionId: string }
|
|
368
|
+
| {
|
|
369
|
+
kind: `managed`
|
|
370
|
+
storageCollectionId: string
|
|
371
|
+
claimId: string
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
export function eagerStorageTarget(
|
|
375
|
+
collectionId: string,
|
|
376
|
+
): PersistedStorageTarget {
|
|
377
|
+
return { kind: `eager`, collectionId }
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
export function managedStorageTarget(
|
|
381
|
+
storageCollectionId: string,
|
|
382
|
+
claimId: string,
|
|
383
|
+
): PersistedStorageTarget {
|
|
384
|
+
return { kind: `managed`, storageCollectionId, claimId }
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
export function resolvePersistedStorageTarget(
|
|
388
|
+
target: PersistedStorageTarget,
|
|
389
|
+
legacyClaimId?: string,
|
|
390
|
+
): { collectionId: string; claimId?: string } {
|
|
391
|
+
const input: unknown = target
|
|
392
|
+
if (typeof input !== `object` || input === null) {
|
|
393
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
394
|
+
`SQLite persistence requires an explicit eager or managed storage target`,
|
|
395
|
+
)
|
|
396
|
+
}
|
|
397
|
+
const fields = input as Record<string, unknown>
|
|
398
|
+
if (fields.kind === `eager`) {
|
|
399
|
+
if (
|
|
400
|
+
typeof fields.collectionId !== `string` ||
|
|
401
|
+
fields.collectionId.trim().length === 0
|
|
402
|
+
) {
|
|
403
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
404
|
+
`An eager storage target requires a nonempty collectionId`,
|
|
405
|
+
)
|
|
406
|
+
}
|
|
407
|
+
if (
|
|
408
|
+
legacyClaimId !== undefined ||
|
|
409
|
+
`claimId` in fields ||
|
|
410
|
+
`storageCollectionId` in fields
|
|
411
|
+
) {
|
|
412
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
413
|
+
`An eager storage target cannot carry a persisted cache claim`,
|
|
414
|
+
)
|
|
415
|
+
}
|
|
416
|
+
return { collectionId: fields.collectionId }
|
|
417
|
+
}
|
|
418
|
+
if (fields.kind !== `managed`) {
|
|
419
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
420
|
+
`Unknown SQLite persistence storage target kind`,
|
|
421
|
+
)
|
|
422
|
+
}
|
|
423
|
+
if (
|
|
424
|
+
typeof fields.storageCollectionId !== `string` ||
|
|
425
|
+
fields.storageCollectionId.trim().length === 0 ||
|
|
426
|
+
typeof fields.claimId !== `string` ||
|
|
427
|
+
fields.claimId.trim().length === 0
|
|
428
|
+
) {
|
|
429
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
430
|
+
`A managed storage target requires a storageCollectionId and persisted cache claim`,
|
|
431
|
+
)
|
|
432
|
+
}
|
|
433
|
+
if (`collectionId` in fields) {
|
|
434
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
435
|
+
`A managed storage target cannot carry an eager collectionId`,
|
|
436
|
+
)
|
|
437
|
+
}
|
|
438
|
+
if (legacyClaimId !== undefined && legacyClaimId !== fields.claimId) {
|
|
439
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
440
|
+
`The persisted cache claim does not match its storage target`,
|
|
441
|
+
)
|
|
442
|
+
}
|
|
443
|
+
return {
|
|
444
|
+
collectionId: fields.storageCollectionId,
|
|
445
|
+
claimId: fields.claimId,
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
322
449
|
type PersistedResumeGeneration = {
|
|
323
450
|
latestTerm: number
|
|
324
451
|
latestSeq: number
|
|
@@ -349,6 +476,8 @@ export type PersistedTx<
|
|
|
349
476
|
term: number
|
|
350
477
|
seq: number
|
|
351
478
|
rowVersion: number
|
|
479
|
+
/** Claim authorizing this transaction's persisted cache generation. */
|
|
480
|
+
cacheGenerationClaimId?: string
|
|
352
481
|
truncate?: boolean
|
|
353
482
|
mutations: Array<
|
|
354
483
|
| {
|
|
@@ -388,9 +517,12 @@ export const SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY = Symbol.for(
|
|
|
388
517
|
|
|
389
518
|
export interface PersistenceAdapter {
|
|
390
519
|
loadSubset: (
|
|
391
|
-
|
|
520
|
+
target: PersistedStorageTarget,
|
|
392
521
|
options: LoadSubsetOptions,
|
|
393
|
-
ctx?: {
|
|
522
|
+
ctx?: {
|
|
523
|
+
requiredIndexSignatures?: ReadonlyArray<string>
|
|
524
|
+
cacheGenerationClaimId?: string
|
|
525
|
+
},
|
|
394
526
|
) => Promise<
|
|
395
527
|
Array<{
|
|
396
528
|
key: string | number
|
|
@@ -399,10 +531,11 @@ export interface PersistenceAdapter {
|
|
|
399
531
|
}>
|
|
400
532
|
>
|
|
401
533
|
loadResumeSnapshot: (
|
|
402
|
-
|
|
534
|
+
target: PersistedStorageTarget,
|
|
403
535
|
ctx?: {
|
|
404
536
|
requiredIndexSignatures?: ReadonlyArray<string>
|
|
405
537
|
includeRows?: boolean
|
|
538
|
+
cacheGenerationClaimId?: string
|
|
406
539
|
},
|
|
407
540
|
) => Promise<{
|
|
408
541
|
rows: Array<{
|
|
@@ -417,21 +550,65 @@ export interface PersistenceAdapter {
|
|
|
417
550
|
latestRowVersion: number
|
|
418
551
|
resetEpoch: number
|
|
419
552
|
}>
|
|
420
|
-
applyCommittedTx: (
|
|
421
|
-
|
|
553
|
+
applyCommittedTx: (
|
|
554
|
+
target: PersistedStorageTarget,
|
|
555
|
+
tx: PersistedTx,
|
|
556
|
+
) => Promise<void>
|
|
557
|
+
claimCacheGeneration?: (
|
|
558
|
+
collectionId: string,
|
|
559
|
+
) => Promise<PersistedCacheGenerationClaim>
|
|
560
|
+
rotateCacheGeneration?: (
|
|
422
561
|
collectionId: string,
|
|
562
|
+
claimId: string,
|
|
563
|
+
resetMetadata?: { key: string; value: unknown },
|
|
564
|
+
expectedStorageCollectionId?: string,
|
|
565
|
+
) => Promise<PersistedCacheGenerationClaim>
|
|
566
|
+
releaseCacheGenerationClaim?: (claimId: string) => Promise<void>
|
|
567
|
+
renewCacheGenerationClaim?: (
|
|
568
|
+
storageCollectionId: string,
|
|
569
|
+
claimId: string,
|
|
570
|
+
) => Promise<number | undefined>
|
|
571
|
+
getCacheGenerationNow?: () => number
|
|
572
|
+
/** Atomically certify or apply one source transaction after a lost response. */
|
|
573
|
+
reconcileCommittedTx?: (
|
|
574
|
+
target: PersistedStorageTarget,
|
|
575
|
+
tx: PersistedTx,
|
|
576
|
+
anchor: CommittedTxAnchor,
|
|
577
|
+
) => Promise<ReconciledCommittedTx>
|
|
578
|
+
loadCollectionMetadata?: (
|
|
579
|
+
target: PersistedStorageTarget,
|
|
580
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
423
581
|
) => Promise<Array<{ key: string; value: unknown }>>
|
|
424
582
|
scanRows?: (
|
|
425
|
-
|
|
583
|
+
target: PersistedStorageTarget,
|
|
426
584
|
options?: PersistedRowScanOptions,
|
|
585
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
427
586
|
) => Promise<Array<PersistedScannedRow>>
|
|
428
587
|
ensureIndex: (
|
|
429
|
-
|
|
588
|
+
target: PersistedStorageTarget,
|
|
430
589
|
signature: string,
|
|
431
590
|
spec: PersistedIndexSpec,
|
|
591
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
432
592
|
) => Promise<void>
|
|
433
|
-
markIndexRemoved?: (
|
|
434
|
-
|
|
593
|
+
markIndexRemoved?: (
|
|
594
|
+
target: PersistedStorageTarget,
|
|
595
|
+
signature: string,
|
|
596
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
597
|
+
) => Promise<void>
|
|
598
|
+
getStreamPosition?: (
|
|
599
|
+
target: PersistedStorageTarget,
|
|
600
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
601
|
+
) => Promise<{
|
|
602
|
+
latestTerm: number
|
|
603
|
+
latestSeq: number
|
|
604
|
+
latestRowVersion: number
|
|
605
|
+
}>
|
|
606
|
+
/** Reserve a new durable election term before announcing Browser leadership. */
|
|
607
|
+
reserveLeadershipTerm?: (
|
|
608
|
+
target: PersistedStorageTarget,
|
|
609
|
+
observedTerm: number,
|
|
610
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
611
|
+
) => Promise<{
|
|
435
612
|
latestTerm: number
|
|
436
613
|
latestSeq: number
|
|
437
614
|
latestRowVersion: number
|
|
@@ -454,8 +631,9 @@ export interface PersistenceAdapter {
|
|
|
454
631
|
|
|
455
632
|
export type HydrationPersistenceAdapter = PersistenceAdapter & {
|
|
456
633
|
pullSince?: (
|
|
457
|
-
|
|
634
|
+
target: PersistedStorageTarget,
|
|
458
635
|
fromRowVersion: number,
|
|
636
|
+
ctx?: { cacheGenerationClaimId?: string },
|
|
459
637
|
) => Promise<PersistencePullSinceResult>
|
|
460
638
|
}
|
|
461
639
|
|
|
@@ -508,6 +686,7 @@ export interface PersistedCollectionCoordinator {
|
|
|
508
686
|
setAdapterForCollection?: (
|
|
509
687
|
collectionId: string,
|
|
510
688
|
adapter: PersistenceAdapter,
|
|
689
|
+
cacheGenerationClaimId?: string,
|
|
511
690
|
) => void
|
|
512
691
|
subscribe: (
|
|
513
692
|
collectionId: string,
|
|
@@ -539,10 +718,12 @@ export interface PersistedCollectionCoordinator {
|
|
|
539
718
|
spec: PersistedIndexSpec,
|
|
540
719
|
scopedAdapter?: HydrationPersistenceAdapter,
|
|
541
720
|
localEnsureCompleted?: boolean,
|
|
721
|
+
cacheGenerationClaimId?: string,
|
|
542
722
|
) => Promise<void>
|
|
543
723
|
requestApplyLocalMutations?: (
|
|
544
724
|
collectionId: string,
|
|
545
725
|
mutations: Array<PersistedMutationEnvelope>,
|
|
726
|
+
cacheGenerationClaimId?: string,
|
|
546
727
|
) => Promise<ApplyLocalMutationsResponse>
|
|
547
728
|
/**
|
|
548
729
|
* Applies a committed transaction through the supplied adapter when present.
|
|
@@ -555,11 +736,19 @@ export interface PersistedCollectionCoordinator {
|
|
|
555
736
|
tx: PersistedTx,
|
|
556
737
|
scopedAdapter?: HydrationPersistenceAdapter,
|
|
557
738
|
) => Promise<ApplyCommittedTxResponse>
|
|
739
|
+
/** Available only when the elected adapter can certify the durable outcome. */
|
|
740
|
+
reconcileCommittedTx?: (
|
|
741
|
+
collectionId: string,
|
|
742
|
+
tx: PersistedTx,
|
|
743
|
+
anchor: CommittedTxAnchor,
|
|
744
|
+
scopedAdapter?: HydrationPersistenceAdapter,
|
|
745
|
+
) => Promise<ReconcileCommittedTxResponse>
|
|
558
746
|
/** The scoped adapter is leader-local and is never serialized to a follower. */
|
|
559
747
|
pullSince?: (
|
|
560
748
|
collectionId: string,
|
|
561
749
|
fromRowVersion: number,
|
|
562
750
|
scopedAdapter?: HydrationPersistenceAdapter,
|
|
751
|
+
cacheGenerationClaimId?: string,
|
|
563
752
|
) => Promise<PullSinceResponse>
|
|
564
753
|
}
|
|
565
754
|
|
|
@@ -637,6 +826,7 @@ export type PersistedInitialRenderOptions = {
|
|
|
637
826
|
}
|
|
638
827
|
|
|
639
828
|
const DEFAULT_NETWORK_TIMEOUT_MS = 3_000
|
|
829
|
+
const CACHE_CLAIM_RENEW_WINDOW_MS = 60_000
|
|
640
830
|
const MAX_TIMER_TIMEOUT_MS = 2_147_483_647
|
|
641
831
|
|
|
642
832
|
function getNetworkTimeoutMs(
|
|
@@ -756,7 +946,7 @@ type SyncControlFns<T extends object, TKey extends string | number> = {
|
|
|
756
946
|
) => void)
|
|
757
947
|
| null
|
|
758
948
|
commit: ((signal?: AbortSignal) => SyncAppliedReceipt) | null
|
|
759
|
-
truncate: (() => void) | null
|
|
949
|
+
truncate: ((options?: { markReady?: boolean }) => void) | null
|
|
760
950
|
metadata: SyncMetadataApi<TKey> | null
|
|
761
951
|
markError: ((error: unknown) => void) | null
|
|
762
952
|
}
|
|
@@ -848,11 +1038,8 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator
|
|
|
848
1038
|
if (acquisitions!.size === 0) {
|
|
849
1039
|
this.remoteSubsetAcquisitions.delete(collectionId)
|
|
850
1040
|
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
} catch {
|
|
854
|
-
// Calling the owner transferred the lease even when its load rejected.
|
|
855
|
-
}
|
|
1041
|
+
// The owner may need unload to settle its load. The call itself already
|
|
1042
|
+
// transferred the lease, so waiting for load here can deadlock recovery.
|
|
856
1043
|
await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options)
|
|
857
1044
|
}
|
|
858
1045
|
|
|
@@ -870,14 +1057,9 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator
|
|
|
870
1057
|
this.remoteSubsetAcquisitions.delete(collectionId)
|
|
871
1058
|
this.remoteSubsetOwners.delete(collectionId)
|
|
872
1059
|
for (const acquisition of acquisitions?.values() ?? []) {
|
|
873
|
-
void (
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
} catch {
|
|
877
|
-
// Calling the owner transferred the lease even when its load rejected.
|
|
878
|
-
}
|
|
879
|
-
await unloadRemoteSubsetOwner(owner, acquisition.options)
|
|
880
|
-
})().catch(() => undefined)
|
|
1060
|
+
void unloadRemoteSubsetOwner(owner, acquisition.options).catch(
|
|
1061
|
+
() => undefined,
|
|
1062
|
+
)
|
|
881
1063
|
}
|
|
882
1064
|
}
|
|
883
1065
|
}
|
|
@@ -904,7 +1086,12 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator
|
|
|
904
1086
|
}
|
|
905
1087
|
|
|
906
1088
|
try {
|
|
907
|
-
await adapter.applyCommittedTx(
|
|
1089
|
+
await adapter.applyCommittedTx(
|
|
1090
|
+
tx.cacheGenerationClaimId
|
|
1091
|
+
? managedStorageTarget(collectionId, tx.cacheGenerationClaimId)
|
|
1092
|
+
: eagerStorageTarget(collectionId),
|
|
1093
|
+
tx,
|
|
1094
|
+
)
|
|
908
1095
|
} catch (error) {
|
|
909
1096
|
throw toPersistedCollectionDurabilityError(collectionId, error)
|
|
910
1097
|
}
|
|
@@ -1055,6 +1242,7 @@ type BufferedSyncTransaction<T extends object, TKey extends string | number> = {
|
|
|
1055
1242
|
truncate: boolean
|
|
1056
1243
|
internal: boolean
|
|
1057
1244
|
lifecycleGeneration: number
|
|
1245
|
+
admittedStorageCollectionId?: string
|
|
1058
1246
|
expectedResumeGenerationOwner?: symbol
|
|
1059
1247
|
signal?: AbortSignal
|
|
1060
1248
|
prependHydrationRows: (
|
|
@@ -1142,6 +1330,21 @@ class ApplyMutex {
|
|
|
1142
1330
|
}
|
|
1143
1331
|
}
|
|
1144
1332
|
|
|
1333
|
+
function matchesAnySubset(
|
|
1334
|
+
value: object,
|
|
1335
|
+
evaluators: ReadonlyArray<ReturnType<
|
|
1336
|
+
typeof compileSingleRowExpression
|
|
1337
|
+
> | null>,
|
|
1338
|
+
): boolean {
|
|
1339
|
+
return evaluators.some(
|
|
1340
|
+
(evaluator) =>
|
|
1341
|
+
!evaluator ||
|
|
1342
|
+
toBooleanPredicate(
|
|
1343
|
+
evaluator(value as Record<string, unknown>) as boolean | null,
|
|
1344
|
+
),
|
|
1345
|
+
)
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1145
1348
|
function toStableSerializable(value: unknown): unknown {
|
|
1146
1349
|
if (value == null) {
|
|
1147
1350
|
return value
|
|
@@ -1252,7 +1455,10 @@ function isTxCommittedPayload(payload: unknown): payload is TxCommitted {
|
|
|
1252
1455
|
}
|
|
1253
1456
|
|
|
1254
1457
|
if (payload.requiresFullReload) {
|
|
1255
|
-
return
|
|
1458
|
+
return (
|
|
1459
|
+
payload.reconciliationReload === undefined ||
|
|
1460
|
+
payload.reconciliationReload === true
|
|
1461
|
+
)
|
|
1256
1462
|
}
|
|
1257
1463
|
|
|
1258
1464
|
return (
|
|
@@ -1309,18 +1515,36 @@ class PersistedCollectionRuntime<
|
|
|
1309
1515
|
string,
|
|
1310
1516
|
LoadSubsetOptions
|
|
1311
1517
|
>()
|
|
1518
|
+
private readonly scopedRefreshes = new Map<LoadSubsetOptions, string>()
|
|
1519
|
+
/** A retained demand may use fresh physical request data after rotation. */
|
|
1520
|
+
private readonly rotatedAcquisitions = new Map<
|
|
1521
|
+
LoadSubsetOptions,
|
|
1522
|
+
LoadSubsetOptions
|
|
1523
|
+
>()
|
|
1524
|
+
/** An in-flight load waits for the recovery acquisition of its demand. */
|
|
1525
|
+
private readonly rotatedEnsures = new WeakMap<
|
|
1526
|
+
LoadSubsetOptions,
|
|
1527
|
+
Promise<void>
|
|
1528
|
+
>()
|
|
1312
1529
|
private readonly queuedHydrationTransactions: Array<
|
|
1313
1530
|
BufferedSyncTransaction<T, TKey>
|
|
1314
1531
|
> = []
|
|
1315
1532
|
private readonly queuedTxCommitted: Array<{
|
|
1316
1533
|
txCommitted: TxCommitted
|
|
1317
1534
|
lifecycleGeneration: number
|
|
1535
|
+
storageCollectionId: string
|
|
1318
1536
|
}> = []
|
|
1319
1537
|
private readonly requestIds = new WeakMap<LoadSubsetOptions, string>()
|
|
1320
1538
|
private readonly hydratedDemands = new Set<string | undefined>()
|
|
1321
1539
|
|
|
1322
1540
|
private collection: Collection<T, TKey, PersistedCollectionUtils> | null =
|
|
1323
1541
|
null
|
|
1542
|
+
// Cleanup ends the active sync run, but cannot transfer this runtime's owner.
|
|
1543
|
+
private boundCollection: Collection<
|
|
1544
|
+
T,
|
|
1545
|
+
TKey,
|
|
1546
|
+
PersistedCollectionUtils
|
|
1547
|
+
> | null = null
|
|
1324
1548
|
private syncControls: SyncControlFns<T, TKey> = {
|
|
1325
1549
|
begin: null,
|
|
1326
1550
|
write: null,
|
|
@@ -1351,13 +1575,32 @@ class PersistedCollectionRuntime<
|
|
|
1351
1575
|
private indexAddedUnsubscribe: (() => void) | null = null
|
|
1352
1576
|
private indexRemovedUnsubscribe: (() => void) | null = null
|
|
1353
1577
|
private remoteEnsureRetryTimer: ReturnType<typeof setTimeout> | null = null
|
|
1578
|
+
private cacheClaimRenewTimer: ReturnType<typeof setTimeout> | null = null
|
|
1579
|
+
private cacheClaimRecovery: Promise<void> | null = null
|
|
1354
1580
|
private nextRequestId = 0
|
|
1355
1581
|
private startupSettled = false
|
|
1356
1582
|
private sourceTruncateGeneration = 0
|
|
1583
|
+
/** Durable rows cannot replace source rows during scoped recovery. */
|
|
1584
|
+
private scopedRecovery = false
|
|
1585
|
+
private scopedRecoveryPending = false
|
|
1586
|
+
private scopedRecoverySequence = 0
|
|
1587
|
+
private recoverQueuedAfterLocalLoadFailure = false
|
|
1588
|
+
private cacheGenerationClaim: PersistedCacheGenerationClaim | undefined
|
|
1589
|
+
private startupCacheGenerationRotated = false
|
|
1590
|
+
private remoteSubsetOwner: RemoteSubsetOwner | undefined
|
|
1591
|
+
// Undefined before source assignment; null means the source cannot restart.
|
|
1592
|
+
private restartSourceAfterScopedRecovery:
|
|
1593
|
+
((cacheRotated: Promise<void>) => void | Promise<void>) | null | undefined
|
|
1357
1594
|
|
|
1358
1595
|
private latestTerm = 0
|
|
1359
1596
|
private latestSeq = 0
|
|
1360
1597
|
private latestRowVersion = 0
|
|
1598
|
+
// A metadata-only resume snapshot can advance latestRowVersion without
|
|
1599
|
+
// publishing rows. Gap checks must use the last publicly applied version.
|
|
1600
|
+
private publicRowVersion = 0
|
|
1601
|
+
// A reset does not label later tx notifications with their reset epoch.
|
|
1602
|
+
// Old-term payloads must be checked against durable rows even after a new term.
|
|
1603
|
+
private resetTermFence: number | undefined
|
|
1361
1604
|
private localTerm = 1
|
|
1362
1605
|
private localSeq = 0
|
|
1363
1606
|
private localRowVersion = 0
|
|
@@ -1369,8 +1612,234 @@ class PersistedCollectionRuntime<
|
|
|
1369
1612
|
private readonly syncMode: `eager` | `on-demand`,
|
|
1370
1613
|
private readonly dbName: string,
|
|
1371
1614
|
private readonly persistedReadiness?: PersistedReadinessTracker,
|
|
1615
|
+
private readonly optionsClaim?: () => void | (() => void),
|
|
1372
1616
|
) {}
|
|
1373
1617
|
|
|
1618
|
+
private get storageCollectionId(): string {
|
|
1619
|
+
return this.cacheGenerationClaim?.storageCollectionId ?? this.collectionId
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
private storageTargetFor(
|
|
1623
|
+
storageCollectionId: string = this.storageCollectionId,
|
|
1624
|
+
claimId: string | undefined = this.cacheGenerationClaim?.claimId,
|
|
1625
|
+
): PersistedStorageTarget {
|
|
1626
|
+
if (
|
|
1627
|
+
this.mode === `sync-present` &&
|
|
1628
|
+
this.syncMode === `on-demand` &&
|
|
1629
|
+
this.persistence.adapter.claimCacheGeneration
|
|
1630
|
+
) {
|
|
1631
|
+
if (!claimId) {
|
|
1632
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
1633
|
+
`A managed on-demand cache operation requires a persisted cache claim`,
|
|
1634
|
+
)
|
|
1635
|
+
}
|
|
1636
|
+
return { kind: `managed`, storageCollectionId, claimId }
|
|
1637
|
+
}
|
|
1638
|
+
return { kind: `eager`, collectionId: storageCollectionId }
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
private cacheGenerationNow(): number {
|
|
1642
|
+
return this.persistence.adapter.getCacheGenerationNow?.() ?? Date.now()
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
private hasExpiredCacheClaim(): boolean {
|
|
1646
|
+
const expiresAtMs = this.cacheGenerationClaim?.expiresAtMs
|
|
1647
|
+
return expiresAtMs !== undefined && this.cacheGenerationNow() >= expiresAtMs
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
private async claimCacheGeneration(
|
|
1651
|
+
lifecycleGeneration: number,
|
|
1652
|
+
): Promise<void> {
|
|
1653
|
+
if (this.mode !== `sync-present` || this.syncMode !== `on-demand`) return
|
|
1654
|
+
const adapter = this.persistence.adapter
|
|
1655
|
+
const capabilities = [
|
|
1656
|
+
adapter.claimCacheGeneration,
|
|
1657
|
+
adapter.rotateCacheGeneration,
|
|
1658
|
+
adapter.renewCacheGenerationClaim,
|
|
1659
|
+
adapter.releaseCacheGenerationClaim,
|
|
1660
|
+
]
|
|
1661
|
+
if (capabilities.some(Boolean) && !capabilities.every(Boolean)) {
|
|
1662
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
1663
|
+
`An on-demand managed cache generation adapter must implement claimCacheGeneration, rotateCacheGeneration, renewCacheGenerationClaim, and releaseCacheGenerationClaim together`,
|
|
1664
|
+
)
|
|
1665
|
+
}
|
|
1666
|
+
if (!adapter.claimCacheGeneration) return
|
|
1667
|
+
const claim = await adapter.claimCacheGeneration(this.collectionId)
|
|
1668
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) {
|
|
1669
|
+
await adapter.releaseCacheGenerationClaim?.(claim.claimId)
|
|
1670
|
+
return
|
|
1671
|
+
}
|
|
1672
|
+
this.cacheGenerationClaim = claim
|
|
1673
|
+
this.scheduleCacheClaimRenewal(claim)
|
|
1674
|
+
this.attachCoordinatorSubscription()
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
private clearCacheClaimRenewal(): void {
|
|
1678
|
+
if (this.cacheClaimRenewTimer !== null) {
|
|
1679
|
+
clearTimeout(this.cacheClaimRenewTimer)
|
|
1680
|
+
this.cacheClaimRenewTimer = null
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
private scheduleCacheClaimRenewal(
|
|
1685
|
+
claim: PersistedCacheGenerationClaim,
|
|
1686
|
+
): void {
|
|
1687
|
+
this.clearCacheClaimRenewal()
|
|
1688
|
+
if (
|
|
1689
|
+
claim.expiresAtMs === undefined ||
|
|
1690
|
+
!this.persistence.adapter.renewCacheGenerationClaim
|
|
1691
|
+
) {
|
|
1692
|
+
return
|
|
1693
|
+
}
|
|
1694
|
+
const remaining = claim.expiresAtMs - this.cacheGenerationNow()
|
|
1695
|
+
const delay =
|
|
1696
|
+
remaining <= 0
|
|
1697
|
+
? 1_000
|
|
1698
|
+
: Math.max(
|
|
1699
|
+
1,
|
|
1700
|
+
Math.min(CACHE_CLAIM_RENEW_WINDOW_MS, Math.floor(remaining / 2)),
|
|
1701
|
+
)
|
|
1702
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
1703
|
+
this.cacheClaimRenewTimer = setTimeout(() => {
|
|
1704
|
+
this.cacheClaimRenewTimer = null
|
|
1705
|
+
void this.ensureCacheClaimActive().catch(() => {
|
|
1706
|
+
// A failed renewal does not establish claim loss. Adapter reads and
|
|
1707
|
+
// writes still check the claim; retry the background check without
|
|
1708
|
+
// turning one transient storage error into a permanent Collection
|
|
1709
|
+
// failure. An expired claim retries at a bounded pace.
|
|
1710
|
+
if (
|
|
1711
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
1712
|
+
this.cacheGenerationClaim === claim
|
|
1713
|
+
) {
|
|
1714
|
+
this.scheduleCacheClaimRenewal(claim)
|
|
1715
|
+
}
|
|
1716
|
+
})
|
|
1717
|
+
}, delay)
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
private async ensureCacheClaimActive(): Promise<void> {
|
|
1721
|
+
if (this.cacheClaimRecovery) {
|
|
1722
|
+
await this.cacheClaimRecovery
|
|
1723
|
+
return
|
|
1724
|
+
}
|
|
1725
|
+
if (await this.renewCacheClaim(this.persistence.adapter)) return
|
|
1726
|
+
// Another caller may have started recovery while renewal was awaited.
|
|
1727
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
1728
|
+
if (!this.cacheClaimRecovery) {
|
|
1729
|
+
const recovery =
|
|
1730
|
+
this.restartSourceAfterScopedRecovery === undefined
|
|
1731
|
+
? this.reclaimStartupCacheGeneration()
|
|
1732
|
+
: this.startScopedRecovery()
|
|
1733
|
+
this.cacheClaimRecovery = recovery
|
|
1734
|
+
void recovery
|
|
1735
|
+
.finally(() => {
|
|
1736
|
+
if (this.cacheClaimRecovery === recovery) {
|
|
1737
|
+
this.cacheClaimRecovery = null
|
|
1738
|
+
}
|
|
1739
|
+
})
|
|
1740
|
+
.catch(() => undefined)
|
|
1741
|
+
}
|
|
1742
|
+
await this.cacheClaimRecovery
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
private scheduleCacheClaimRecovery(lifecycleGeneration: number): void {
|
|
1746
|
+
// Recovery may need the apply mutex held by the caller. Let that caller
|
|
1747
|
+
// release its turn before checking the claim and rotating the cache.
|
|
1748
|
+
queueMicrotask(() => {
|
|
1749
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
1750
|
+
void this.ensureCacheClaimActive()
|
|
1751
|
+
.then(() =>
|
|
1752
|
+
this.applyMutex.run(() => this.flushQueuedTxCommittedUnsafe()),
|
|
1753
|
+
)
|
|
1754
|
+
.catch((error) => {
|
|
1755
|
+
this.markTerminalFailure(error, lifecycleGeneration)
|
|
1756
|
+
})
|
|
1757
|
+
})
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
private async reclaimStartupCacheGeneration(): Promise<void> {
|
|
1761
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
1762
|
+
const previous = this.cacheGenerationClaim
|
|
1763
|
+
if (!previous) return
|
|
1764
|
+
if (!this.persistence.adapter.rotateCacheGeneration) {
|
|
1765
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
1766
|
+
`A managed on-demand cache adapter must rotate expired startup claims`,
|
|
1767
|
+
)
|
|
1768
|
+
}
|
|
1769
|
+
this.clearCacheClaimRenewal()
|
|
1770
|
+
this.coordinatorUnsubscribe?.()
|
|
1771
|
+
this.coordinatorUnsubscribe = null
|
|
1772
|
+
const rotated = await this.persistence.adapter.rotateCacheGeneration(
|
|
1773
|
+
this.collectionId,
|
|
1774
|
+
previous.claimId,
|
|
1775
|
+
undefined,
|
|
1776
|
+
previous.storageCollectionId,
|
|
1777
|
+
)
|
|
1778
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) {
|
|
1779
|
+
await this.persistence.adapter.releaseCacheGenerationClaim?.(
|
|
1780
|
+
rotated.claimId,
|
|
1781
|
+
)
|
|
1782
|
+
return
|
|
1783
|
+
}
|
|
1784
|
+
this.cacheGenerationClaim = rotated
|
|
1785
|
+
this.startupCacheGenerationRotated = true
|
|
1786
|
+
this.resetSequence++
|
|
1787
|
+
this.latestTerm = 0
|
|
1788
|
+
this.latestSeq = 0
|
|
1789
|
+
this.latestRowVersion = 0
|
|
1790
|
+
this.localTerm = 1
|
|
1791
|
+
this.localSeq = 0
|
|
1792
|
+
this.localRowVersion = 0
|
|
1793
|
+
this.persistedResumeGeneration = undefined
|
|
1794
|
+
this.persistedKeySetEvidence = { status: `incompatible` }
|
|
1795
|
+
this.scheduleCacheClaimRenewal(rotated)
|
|
1796
|
+
this.attachCoordinatorSubscription()
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
private async renewCacheClaim(
|
|
1800
|
+
adapter: HydrationPersistenceAdapter,
|
|
1801
|
+
): Promise<boolean> {
|
|
1802
|
+
const claim = this.cacheGenerationClaim
|
|
1803
|
+
if (
|
|
1804
|
+
!claim ||
|
|
1805
|
+
!adapter.renewCacheGenerationClaim ||
|
|
1806
|
+
this.scopedRecoveryPending
|
|
1807
|
+
)
|
|
1808
|
+
return true
|
|
1809
|
+
if (
|
|
1810
|
+
claim.expiresAtMs !== undefined &&
|
|
1811
|
+
claim.expiresAtMs - this.cacheGenerationNow() >
|
|
1812
|
+
CACHE_CLAIM_RENEW_WINDOW_MS
|
|
1813
|
+
) {
|
|
1814
|
+
// Durable operations check their claim at the adapter boundary. Keep
|
|
1815
|
+
// the background timer armed without writing the same live claim for
|
|
1816
|
+
// every foreground operation.
|
|
1817
|
+
if (this.cacheClaimRenewTimer === null) {
|
|
1818
|
+
this.scheduleCacheClaimRenewal(claim)
|
|
1819
|
+
}
|
|
1820
|
+
return true
|
|
1821
|
+
}
|
|
1822
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
1823
|
+
const expiresAtMs = await adapter.renewCacheGenerationClaim(
|
|
1824
|
+
claim.storageCollectionId,
|
|
1825
|
+
claim.claimId,
|
|
1826
|
+
)
|
|
1827
|
+
if (
|
|
1828
|
+
lifecycleGeneration !== this.lifecycleGeneration ||
|
|
1829
|
+
this.cacheGenerationClaim?.claimId !== claim.claimId ||
|
|
1830
|
+
this.cacheGenerationClaim.storageCollectionId !==
|
|
1831
|
+
claim.storageCollectionId
|
|
1832
|
+
) {
|
|
1833
|
+
return true
|
|
1834
|
+
}
|
|
1835
|
+
if (expiresAtMs === undefined) {
|
|
1836
|
+
return false
|
|
1837
|
+
}
|
|
1838
|
+
this.cacheGenerationClaim = { ...claim, expiresAtMs }
|
|
1839
|
+
this.scheduleCacheClaimRenewal(this.cacheGenerationClaim)
|
|
1840
|
+
return true
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1374
1843
|
setSyncControls(syncControls: SyncControlFns<T, TKey>): void {
|
|
1375
1844
|
this.advanceLifecycle()
|
|
1376
1845
|
this.syncErrorReported = false
|
|
@@ -1385,6 +1854,17 @@ class PersistedCollectionRuntime<
|
|
|
1385
1854
|
}
|
|
1386
1855
|
}
|
|
1387
1856
|
|
|
1857
|
+
setSourceRestartAfterScopedRecovery(
|
|
1858
|
+
restart:
|
|
1859
|
+
((cacheRotated: Promise<void>) => void | Promise<void>) | undefined,
|
|
1860
|
+
): void {
|
|
1861
|
+
this.restartSourceAfterScopedRecovery = restart ?? null
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
clearSourceRestartAfterScopedRecovery(): void {
|
|
1865
|
+
this.restartSourceAfterScopedRecovery = undefined
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1388
1868
|
reportSyncError(error: unknown): unknown {
|
|
1389
1869
|
const markError = this.syncControls.markError
|
|
1390
1870
|
if (this.syncErrorReported) return this.reportedSyncError
|
|
@@ -1402,9 +1882,10 @@ class PersistedCollectionRuntime<
|
|
|
1402
1882
|
|
|
1403
1883
|
registerRemoteSubsetOwner(owner: RemoteSubsetOwner): void {
|
|
1404
1884
|
this.remoteSubsetOwnerUnsubscribe?.()
|
|
1885
|
+
this.remoteSubsetOwner = owner
|
|
1405
1886
|
this.remoteSubsetOwnerUnsubscribe =
|
|
1406
1887
|
this.persistence.coordinator.registerRemoteSubsetOwner(
|
|
1407
|
-
this.
|
|
1888
|
+
this.storageCollectionId,
|
|
1408
1889
|
owner,
|
|
1409
1890
|
)
|
|
1410
1891
|
}
|
|
@@ -1438,6 +1919,31 @@ class PersistedCollectionRuntime<
|
|
|
1438
1919
|
)
|
|
1439
1920
|
}
|
|
1440
1921
|
|
|
1922
|
+
// An ensure reply settles a follower's load (law 7). A source's loadSubset
|
|
1923
|
+
// must await every commit receipt that establishes its result, so the
|
|
1924
|
+
// leader broadcasts those commits before it replies on the same channel.
|
|
1925
|
+
// They are already queued on the apply mutex; settle after they show here.
|
|
1926
|
+
private async applyLeaderCommitsBeforeReply(
|
|
1927
|
+
options: LoadSubsetOptions,
|
|
1928
|
+
): Promise<void> {
|
|
1929
|
+
try {
|
|
1930
|
+
// Commits kept while the claim was expired apply after it is active.
|
|
1931
|
+
if (this.hasExpiredCacheClaim() || this.cacheClaimRecovery) {
|
|
1932
|
+
await this.ensureCacheClaimActive()
|
|
1933
|
+
}
|
|
1934
|
+
await this.applyMutex.run(() => this.flushQueuedTxCommittedUnsafe())
|
|
1935
|
+
// A broadcast that failed to apply made the runtime terminal; its rows
|
|
1936
|
+
// are not visible, so the load must not settle successfully.
|
|
1937
|
+
this.throwIfTerminal()
|
|
1938
|
+
} catch (error) {
|
|
1939
|
+
// A terminal runtime releases the acquisition the owner granted the
|
|
1940
|
+
// load. Otherwise, as after a failed ensure, the demand retries.
|
|
1941
|
+
if (this.getCurrentTerminalFailure()) this.unloadSubset(options)
|
|
1942
|
+
else this.queueRemoteSubsetEnsure(options)
|
|
1943
|
+
throw error
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1441
1947
|
private clearSyncControls(): void {
|
|
1442
1948
|
this.syncControls = {
|
|
1443
1949
|
begin: null,
|
|
@@ -1513,21 +2019,63 @@ class PersistedCollectionRuntime<
|
|
|
1513
2019
|
return this.internalApplyDepth > 0
|
|
1514
2020
|
}
|
|
1515
2021
|
|
|
2022
|
+
isScopedRecoveryPending(): boolean {
|
|
2023
|
+
return this.scopedRecoveryPending
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
hasBoundCollection(): boolean {
|
|
2027
|
+
return this.boundCollection !== null
|
|
2028
|
+
}
|
|
2029
|
+
|
|
1516
2030
|
setCollection(
|
|
1517
2031
|
collection: Collection<T, TKey, PersistedCollectionUtils>,
|
|
2032
|
+
syncControls: SyncControlFns<T, TKey>,
|
|
1518
2033
|
): void {
|
|
2034
|
+
if (
|
|
2035
|
+
(
|
|
2036
|
+
collection.config as typeof collection.config & {
|
|
2037
|
+
[collectionOptionsClaim]?: unknown
|
|
2038
|
+
}
|
|
2039
|
+
)[collectionOptionsClaim] !== this.optionsClaim
|
|
2040
|
+
) {
|
|
2041
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
2042
|
+
`Persisted sync belongs to its options Collection`,
|
|
2043
|
+
)
|
|
2044
|
+
}
|
|
2045
|
+
if (this.boundCollection && this.boundCollection !== collection) {
|
|
2046
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
2047
|
+
`Persisted options cannot own more than one Collection`,
|
|
2048
|
+
)
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
const previousCollection = this.collection
|
|
2052
|
+
const previousControls = this.syncControls
|
|
2053
|
+
this.setSyncControls(syncControls)
|
|
1519
2054
|
if (this.collection === collection) {
|
|
2055
|
+
this.boundCollection ??= collection
|
|
1520
2056
|
return
|
|
1521
2057
|
}
|
|
1522
|
-
|
|
1523
2058
|
this.collection = collection
|
|
1524
|
-
|
|
2059
|
+
try {
|
|
2060
|
+
if (
|
|
2061
|
+
!this.persistence.adapter.claimCacheGeneration ||
|
|
2062
|
+
this.mode !== `sync-present` ||
|
|
2063
|
+
this.syncMode !== `on-demand`
|
|
2064
|
+
) {
|
|
2065
|
+
this.attachCoordinatorSubscription()
|
|
2066
|
+
}
|
|
2067
|
+
} catch (error) {
|
|
2068
|
+
this.collection = previousCollection
|
|
2069
|
+
this.syncControls = previousControls
|
|
2070
|
+
throw error
|
|
2071
|
+
}
|
|
2072
|
+
this.boundCollection ??= collection
|
|
1525
2073
|
}
|
|
1526
2074
|
|
|
1527
2075
|
getLeadershipState(): PersistedCollectionLeadershipState {
|
|
1528
2076
|
return {
|
|
1529
2077
|
nodeId: this.persistence.coordinator.getNodeId(),
|
|
1530
|
-
isLeader: this.persistence.coordinator.isLeader(this.
|
|
2078
|
+
isLeader: this.persistence.coordinator.isLeader(this.storageCollectionId),
|
|
1531
2079
|
}
|
|
1532
2080
|
}
|
|
1533
2081
|
|
|
@@ -1548,6 +2096,7 @@ class PersistedCollectionRuntime<
|
|
|
1548
2096
|
}
|
|
1549
2097
|
|
|
1550
2098
|
const lifecycleGeneration = this.lifecycleGeneration
|
|
2099
|
+
this.startupCacheGenerationRotated = false
|
|
1551
2100
|
const isRestart = this.hasAttemptedStartup
|
|
1552
2101
|
this.hasAttemptedStartup = true
|
|
1553
2102
|
let resolveStartupMetadata!: () => void
|
|
@@ -1559,6 +2108,12 @@ class PersistedCollectionRuntime<
|
|
|
1559
2108
|
void this.startupMetadataPromise.catch(() => undefined)
|
|
1560
2109
|
|
|
1561
2110
|
this.startPromise = (async () => {
|
|
2111
|
+
await this.claimCacheGeneration(lifecycleGeneration)
|
|
2112
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) {
|
|
2113
|
+
resolveStartupMetadata()
|
|
2114
|
+
return
|
|
2115
|
+
}
|
|
2116
|
+
let metadataExpired = false
|
|
1562
2117
|
const loadStartupMetadata = async (
|
|
1563
2118
|
adapter: HydrationPersistenceAdapter,
|
|
1564
2119
|
) => {
|
|
@@ -1568,9 +2123,24 @@ class PersistedCollectionRuntime<
|
|
|
1568
2123
|
}
|
|
1569
2124
|
|
|
1570
2125
|
try {
|
|
1571
|
-
|
|
2126
|
+
if (
|
|
2127
|
+
!(await this.loadStartupMetadataInternal(
|
|
2128
|
+
lifecycleGeneration,
|
|
2129
|
+
adapter,
|
|
2130
|
+
))
|
|
2131
|
+
) {
|
|
2132
|
+
metadataExpired = true
|
|
2133
|
+
return false
|
|
2134
|
+
}
|
|
1572
2135
|
return lifecycleGeneration === this.lifecycleGeneration
|
|
1573
2136
|
} catch (error) {
|
|
2137
|
+
// SQLite can reject its second claim check after a startup read
|
|
2138
|
+
// entered with a live claim. That old cache cannot authorize source
|
|
2139
|
+
// entry; rotate it after the hydration scope releases its scheduler.
|
|
2140
|
+
if (this.cacheGenerationClaim && this.hasExpiredCacheClaim()) {
|
|
2141
|
+
metadataExpired = true
|
|
2142
|
+
return false
|
|
2143
|
+
}
|
|
1574
2144
|
rejectStartupMetadata(error)
|
|
1575
2145
|
throw error
|
|
1576
2146
|
}
|
|
@@ -1587,50 +2157,82 @@ class PersistedCollectionRuntime<
|
|
|
1587
2157
|
this.persistence.adapter.runInHydrationScope !== undefined &&
|
|
1588
2158
|
(!isRestart ||
|
|
1589
2159
|
(this.persistence.adapter.isHydrationScopeScheduled?.() ?? true))
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
adapter
|
|
1597
|
-
|
|
1598
|
-
|
|
2160
|
+
for (;;) {
|
|
2161
|
+
metadataExpired = false
|
|
2162
|
+
const storageAtAttempt = this.storageCollectionId
|
|
2163
|
+
if (scheduleStartupAsOneHydrate) {
|
|
2164
|
+
startup = await this.applyMutex.run(async () => {
|
|
2165
|
+
const result = await this.runInHydrationScope(async (adapter) => {
|
|
2166
|
+
if (!(await loadStartupMetadata(adapter))) return undefined
|
|
2167
|
+
return this.startInternal(
|
|
2168
|
+
lifecycleGeneration,
|
|
2169
|
+
adapter,
|
|
2170
|
+
resolveStartupMetadata,
|
|
2171
|
+
)
|
|
2172
|
+
})
|
|
2173
|
+
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
2174
|
+
await this.flushQueuedTxCommittedUnsafe()
|
|
2175
|
+
}
|
|
2176
|
+
return result
|
|
1599
2177
|
})
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2178
|
+
} else {
|
|
2179
|
+
// Preserve the existing unscheduled-adapter lifecycle contract: a
|
|
2180
|
+
// replacement upstream may start while stale hydration is settling.
|
|
2181
|
+
if (await loadStartupMetadata(this.persistence.adapter)) {
|
|
2182
|
+
if (this.persistence.adapter.isHydrationScopeScheduled?.()) {
|
|
2183
|
+
startup = await this.applyMutex.run(async () => {
|
|
2184
|
+
const result = await this.runInHydrationScope((adapter) =>
|
|
2185
|
+
this.startInternal(
|
|
2186
|
+
lifecycleGeneration,
|
|
2187
|
+
adapter,
|
|
2188
|
+
resolveStartupMetadata,
|
|
2189
|
+
),
|
|
2190
|
+
)
|
|
2191
|
+
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
2192
|
+
await this.flushQueuedTxCommittedUnsafe()
|
|
2193
|
+
}
|
|
2194
|
+
return result
|
|
2195
|
+
})
|
|
2196
|
+
} else {
|
|
2197
|
+
startup = await this.startInternal(
|
|
2198
|
+
lifecycleGeneration,
|
|
2199
|
+
this.persistence.adapter,
|
|
2200
|
+
resolveStartupMetadata,
|
|
1617
2201
|
)
|
|
1618
2202
|
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
1619
2203
|
await this.flushQueuedTxCommittedUnsafe()
|
|
1620
2204
|
}
|
|
1621
|
-
return result
|
|
1622
|
-
})
|
|
1623
|
-
} else {
|
|
1624
|
-
startup = await this.startInternal(
|
|
1625
|
-
lifecycleGeneration,
|
|
1626
|
-
this.persistence.adapter,
|
|
1627
|
-
resolveStartupMetadata,
|
|
1628
|
-
)
|
|
1629
|
-
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
1630
|
-
await this.flushQueuedTxCommittedUnsafe()
|
|
1631
2205
|
}
|
|
1632
2206
|
}
|
|
1633
2207
|
}
|
|
2208
|
+
if (
|
|
2209
|
+
this.cacheGenerationClaim &&
|
|
2210
|
+
this.syncMode === `on-demand` &&
|
|
2211
|
+
// Hydration may have changed this through loadStartupMetadata().
|
|
2212
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
2213
|
+
!metadataExpired
|
|
2214
|
+
) {
|
|
2215
|
+
// Source entry must wait until any private rotation started during
|
|
2216
|
+
// index bootstrap has settled. The source has no restart capability
|
|
2217
|
+
// until its sync callback returns.
|
|
2218
|
+
await this.ensureCacheClaimActive()
|
|
2219
|
+
metadataExpired = storageAtAttempt !== this.storageCollectionId
|
|
2220
|
+
}
|
|
2221
|
+
if (!metadataExpired) break
|
|
2222
|
+
// A startup read cannot reacquire the claim inside its hydration
|
|
2223
|
+
// scope: claim acquisition uses the regular scheduler lane.
|
|
2224
|
+
await this.ensureCacheClaimActive()
|
|
2225
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) {
|
|
2226
|
+
resolveStartupMetadata()
|
|
2227
|
+
return
|
|
2228
|
+
}
|
|
2229
|
+
}
|
|
2230
|
+
if (
|
|
2231
|
+
this.cacheGenerationClaim &&
|
|
2232
|
+
this.syncMode === `on-demand` &&
|
|
2233
|
+
lifecycleGeneration === this.lifecycleGeneration
|
|
2234
|
+
) {
|
|
2235
|
+
resolveStartupMetadata()
|
|
1634
2236
|
}
|
|
1635
2237
|
if (
|
|
1636
2238
|
startup !== undefined &&
|
|
@@ -1674,6 +2276,7 @@ class PersistedCollectionRuntime<
|
|
|
1674
2276
|
await this.ensureStarted()
|
|
1675
2277
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
1676
2278
|
if (this.syncMode !== `on-demand`) return
|
|
2279
|
+
if (this.scopedRecovery) return
|
|
1677
2280
|
|
|
1678
2281
|
const appliedCursor = await this.applyMutex.run(async () => {
|
|
1679
2282
|
const result = await this.runInHydrationScope((adapter) =>
|
|
@@ -1684,6 +2287,11 @@ class PersistedCollectionRuntime<
|
|
|
1684
2287
|
}
|
|
1685
2288
|
return result
|
|
1686
2289
|
})
|
|
2290
|
+
if (appliedCursor === false) {
|
|
2291
|
+
// Rotation needs the apply mutex, so recover only after releasing it.
|
|
2292
|
+
await this.ensureCacheClaimActive()
|
|
2293
|
+
return
|
|
2294
|
+
}
|
|
1687
2295
|
if (
|
|
1688
2296
|
appliedCursor !== undefined &&
|
|
1689
2297
|
lifecycleGeneration === this.lifecycleGeneration
|
|
@@ -1705,15 +2313,39 @@ class PersistedCollectionRuntime<
|
|
|
1705
2313
|
this.resumeCertificationPromise = (async () => {
|
|
1706
2314
|
await this.ensureStarted()
|
|
1707
2315
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
2316
|
+
if (this.scopedRecovery) return
|
|
2317
|
+
|
|
2318
|
+
const storageCollectionId = this.storageCollectionId
|
|
2319
|
+
const claimId = this.cacheGenerationClaim?.claimId
|
|
2320
|
+
const certificationIsCurrent = () =>
|
|
2321
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
2322
|
+
!this.scopedRecovery &&
|
|
2323
|
+
storageCollectionId === this.storageCollectionId &&
|
|
2324
|
+
claimId === this.cacheGenerationClaim?.claimId
|
|
2325
|
+
let snapshot: Awaited<
|
|
2326
|
+
ReturnType<PersistenceAdapter[`loadResumeSnapshot`]>
|
|
2327
|
+
>
|
|
2328
|
+
try {
|
|
2329
|
+
snapshot = await this.persistence.adapter.loadResumeSnapshot(
|
|
2330
|
+
this.storageTargetFor(storageCollectionId, claimId),
|
|
2331
|
+
{
|
|
2332
|
+
requiredIndexSignatures: this.getRequiredIndexSignatures(),
|
|
2333
|
+
includeRows: false,
|
|
2334
|
+
cacheGenerationClaimId: claimId,
|
|
2335
|
+
},
|
|
2336
|
+
)
|
|
2337
|
+
} catch (error) {
|
|
2338
|
+
if (!certificationIsCurrent()) return
|
|
2339
|
+
if (!claimId || !this.hasExpiredCacheClaim()) throw error
|
|
2340
|
+
await this.ensureCacheClaimActive()
|
|
2341
|
+
return
|
|
2342
|
+
}
|
|
2343
|
+
if (!certificationIsCurrent()) return
|
|
2344
|
+
if (claimId && !(await this.renewCacheClaim(this.persistence.adapter))) {
|
|
2345
|
+
await this.ensureCacheClaimActive()
|
|
2346
|
+
return
|
|
2347
|
+
}
|
|
2348
|
+
if (!certificationIsCurrent()) return
|
|
1717
2349
|
this.bindResumeSnapshotEvidence(snapshot)
|
|
1718
2350
|
})()
|
|
1719
2351
|
return this.resumeCertificationPromise
|
|
@@ -1723,6 +2355,187 @@ class PersistedCollectionRuntime<
|
|
|
1723
2355
|
return this.persistedKeySetEvidence
|
|
1724
2356
|
}
|
|
1725
2357
|
|
|
2358
|
+
hasManagedCacheGeneration(): boolean {
|
|
2359
|
+
return this.cacheGenerationClaim !== undefined
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
hasRotatedStartupCacheGeneration(): boolean {
|
|
2363
|
+
return this.startupCacheGenerationRotated
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
async startScopedRecovery(resetMetadata?: {
|
|
2367
|
+
key: string
|
|
2368
|
+
value: unknown
|
|
2369
|
+
}): Promise<void> {
|
|
2370
|
+
this.throwIfTerminal()
|
|
2371
|
+
this.clearCacheClaimRenewal()
|
|
2372
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
2373
|
+
const restartSource = this.restartSourceAfterScopedRecovery
|
|
2374
|
+
// Invalidate an in-flight old-cache read before yielding to the mutex.
|
|
2375
|
+
this.scopedRecovery = true
|
|
2376
|
+
this.scopedRecoveryPending = true
|
|
2377
|
+
const recoverySequence = ++this.scopedRecoverySequence
|
|
2378
|
+
this.hydratedDemands.clear()
|
|
2379
|
+
this.resetSequence++
|
|
2380
|
+
let resolveCacheRotated: (() => void) | undefined
|
|
2381
|
+
let rejectCacheRotated: ((error: unknown) => void) | undefined
|
|
2382
|
+
let sourceRestart: Promise<void> | undefined
|
|
2383
|
+
try {
|
|
2384
|
+
if (restartSource) {
|
|
2385
|
+
const cacheRotated = new Promise<void>((resolve, reject) => {
|
|
2386
|
+
resolveCacheRotated = resolve
|
|
2387
|
+
rejectCacheRotated = reject
|
|
2388
|
+
})
|
|
2389
|
+
void cacheRotated.catch(() => undefined)
|
|
2390
|
+
// The source must retire old callbacks before this call returns.
|
|
2391
|
+
sourceRestart = Promise.resolve(restartSource(cacheRotated))
|
|
2392
|
+
void sourceRestart.catch(() => undefined)
|
|
2393
|
+
}
|
|
2394
|
+
await this.applyMutex.run(async () => {
|
|
2395
|
+
this.throwIfLifecycleReplaced(lifecycleGeneration)
|
|
2396
|
+
const claim = this.cacheGenerationClaim
|
|
2397
|
+
if (claim && this.persistence.adapter.rotateCacheGeneration) {
|
|
2398
|
+
const previousStorageId = claim.storageCollectionId
|
|
2399
|
+
const rotated = await this.persistence.adapter.rotateCacheGeneration(
|
|
2400
|
+
this.collectionId,
|
|
2401
|
+
claim.claimId,
|
|
2402
|
+
resetMetadata,
|
|
2403
|
+
previousStorageId,
|
|
2404
|
+
)
|
|
2405
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) {
|
|
2406
|
+
// Cleanup released the old claim while rotation was in flight.
|
|
2407
|
+
// The returned claim belongs to this abandoned recovery too.
|
|
2408
|
+
await this.persistence.adapter.releaseCacheGenerationClaim?.(
|
|
2409
|
+
rotated.claimId,
|
|
2410
|
+
)
|
|
2411
|
+
throw new SyncTransactionAbortedError()
|
|
2412
|
+
}
|
|
2413
|
+
for (const options of this.activeSubsets.values()) {
|
|
2414
|
+
// The retired generation has a separate coordinator route. A
|
|
2415
|
+
// paused old leader cannot hold up the new cache's demands.
|
|
2416
|
+
void this.persistence.coordinator
|
|
2417
|
+
.requestReleaseRemoteSubset(
|
|
2418
|
+
previousStorageId,
|
|
2419
|
+
this.rotatedAcquisitions.get(options) ?? options,
|
|
2420
|
+
)
|
|
2421
|
+
.catch(() => undefined)
|
|
2422
|
+
if (options.signal?.aborted) {
|
|
2423
|
+
this.rotatedAcquisitions.delete(options)
|
|
2424
|
+
continue
|
|
2425
|
+
}
|
|
2426
|
+
this.rotatedAcquisitions.set(options, {
|
|
2427
|
+
...options,
|
|
2428
|
+
refetch: true,
|
|
2429
|
+
})
|
|
2430
|
+
}
|
|
2431
|
+
this.remoteSubsetOwnerUnsubscribe?.()
|
|
2432
|
+
this.remoteSubsetOwnerUnsubscribe = null
|
|
2433
|
+
this.coordinatorUnsubscribe?.()
|
|
2434
|
+
this.coordinatorUnsubscribe = null
|
|
2435
|
+
this.cacheGenerationClaim = rotated
|
|
2436
|
+
this.latestTerm = 0
|
|
2437
|
+
this.latestSeq = 0
|
|
2438
|
+
this.latestRowVersion = 0
|
|
2439
|
+
this.publicRowVersion = 0
|
|
2440
|
+
this.resetTermFence = undefined
|
|
2441
|
+
this.localTerm = 1
|
|
2442
|
+
this.localSeq = 0
|
|
2443
|
+
this.localRowVersion = 0
|
|
2444
|
+
this.queuedTxCommitted.length = 0
|
|
2445
|
+
this.persistedResumeGeneration = undefined
|
|
2446
|
+
this.persistedKeySetEvidence = { status: `incompatible` }
|
|
2447
|
+
this.resumeCertificationPromise = null
|
|
2448
|
+
this.resumeGenerationOwner = Symbol(
|
|
2449
|
+
`persisted resume generation owner`,
|
|
2450
|
+
)
|
|
2451
|
+
this.attachCoordinatorSubscription()
|
|
2452
|
+
if (this.remoteSubsetOwner) {
|
|
2453
|
+
this.registerRemoteSubsetOwner(this.remoteSubsetOwner)
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
if (this.syncControls.begin && this.syncControls.commit) {
|
|
2457
|
+
const applied = this.withInternalApply(() => {
|
|
2458
|
+
this.syncControls.begin?.()
|
|
2459
|
+
this.syncControls.truncate?.({ markReady: false })
|
|
2460
|
+
if (resetMetadata) {
|
|
2461
|
+
this.syncControls.metadata?.collection.set(
|
|
2462
|
+
resetMetadata.key,
|
|
2463
|
+
resetMetadata.value,
|
|
2464
|
+
)
|
|
2465
|
+
}
|
|
2466
|
+
return this.syncControls.commit?.() ?? true
|
|
2467
|
+
})
|
|
2468
|
+
await whenSyncAccepted(applied)
|
|
2469
|
+
if (
|
|
2470
|
+
!this.cacheGenerationClaim &&
|
|
2471
|
+
recoverySequence === this.scopedRecoverySequence
|
|
2472
|
+
) {
|
|
2473
|
+
this.scopedRecoveryPending = false
|
|
2474
|
+
}
|
|
2475
|
+
await this.runInHydrationScope(async (adapter) => {
|
|
2476
|
+
if (this.recoverQueuedAfterLocalLoadFailure) {
|
|
2477
|
+
this.recoverQueuedAfterLocalLoadFailure = false
|
|
2478
|
+
await this.recoverBufferedTransactionsAfterLocalLoadFailureUnsafe(
|
|
2479
|
+
lifecycleGeneration,
|
|
2480
|
+
adapter,
|
|
2481
|
+
)
|
|
2482
|
+
} else {
|
|
2483
|
+
await this.flushQueuedHydrationTransactionsUnsafe(adapter)
|
|
2484
|
+
}
|
|
2485
|
+
})
|
|
2486
|
+
await this.flushQueuedTxCommittedUnsafe()
|
|
2487
|
+
}
|
|
2488
|
+
if (recoverySequence === this.scopedRecoverySequence) {
|
|
2489
|
+
// Older overlapping rotations leave quarantine in place. The final
|
|
2490
|
+
// rotation rebuilds the current index registry before releasing the
|
|
2491
|
+
// mutex; later index events can then target only its storage ID.
|
|
2492
|
+
this.scopedRecoveryPending = false
|
|
2493
|
+
if (this.cacheGenerationClaim) {
|
|
2494
|
+
const currentIndexes = this.collection?.getIndexMetadata() ?? []
|
|
2495
|
+
const completedIndexes = await this.bootstrapPersistedIndexes(
|
|
2496
|
+
currentIndexes,
|
|
2497
|
+
this.persistence.adapter,
|
|
2498
|
+
)
|
|
2499
|
+
await this.requestCoordinatorPersistedIndexes(
|
|
2500
|
+
currentIndexes,
|
|
2501
|
+
completedIndexes,
|
|
2502
|
+
)
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
})
|
|
2506
|
+
this.throwIfLifecycleReplaced(lifecycleGeneration)
|
|
2507
|
+
resolveCacheRotated?.()
|
|
2508
|
+
await sourceRestart
|
|
2509
|
+
if (
|
|
2510
|
+
recoverySequence === this.scopedRecoverySequence &&
|
|
2511
|
+
this.cacheGenerationClaim
|
|
2512
|
+
) {
|
|
2513
|
+
this.scheduleCacheClaimRenewal(this.cacheGenerationClaim)
|
|
2514
|
+
}
|
|
2515
|
+
// A newer rotation owns reacquisition of the active demands. Dispatching
|
|
2516
|
+
// them from this older recovery would fetch against its pending cache.
|
|
2517
|
+
if (recoverySequence !== this.scopedRecoverySequence) return
|
|
2518
|
+
for (const options of this.activeSubsets.values()) {
|
|
2519
|
+
if (options.signal?.aborted) continue
|
|
2520
|
+
const rotatedOptions = this.rotatedAcquisitions.get(options)
|
|
2521
|
+
if (rotatedOptions) {
|
|
2522
|
+
const ensure = this.persistence.coordinator.requestEnsureRemoteSubset(
|
|
2523
|
+
this.storageCollectionId,
|
|
2524
|
+
rotatedOptions,
|
|
2525
|
+
)
|
|
2526
|
+
this.rotatedEnsures.set(options, ensure)
|
|
2527
|
+
void ensure.catch((error) => this.reportSyncError(error))
|
|
2528
|
+
} else {
|
|
2529
|
+
this.refreshScopedSubsets()
|
|
2530
|
+
break
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
} catch (error) {
|
|
2534
|
+
rejectCacheRotated?.(error)
|
|
2535
|
+
throw this.markTerminalFailure(error, lifecycleGeneration)
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
|
|
1726
2539
|
getResumeGenerationOwner(): symbol {
|
|
1727
2540
|
return this.resumeGenerationOwner
|
|
1728
2541
|
}
|
|
@@ -1730,7 +2543,7 @@ class PersistedCollectionRuntime<
|
|
|
1730
2543
|
private async hydrateBaseline(
|
|
1731
2544
|
lifecycleGeneration: number,
|
|
1732
2545
|
adapter: HydrationPersistenceAdapter,
|
|
1733
|
-
): Promise<number | undefined> {
|
|
2546
|
+
): Promise<number | undefined | false> {
|
|
1734
2547
|
if (lifecycleGeneration !== this.lifecycleGeneration) return undefined
|
|
1735
2548
|
|
|
1736
2549
|
// The baseline shares the unconstrained demand key. Its lease is never
|
|
@@ -1738,7 +2551,7 @@ class PersistedCollectionRuntime<
|
|
|
1738
2551
|
const baseline = {}
|
|
1739
2552
|
this.activeSubsets.set(this.getSubsetKey(baseline), baseline)
|
|
1740
2553
|
const appliedCursor = this.appliedReceiptSequence
|
|
1741
|
-
await this.hydrateSubsetUnsafe(
|
|
2554
|
+
const hydrated = await this.hydrateSubsetUnsafe(
|
|
1742
2555
|
baseline,
|
|
1743
2556
|
{
|
|
1744
2557
|
lifecycleGeneration,
|
|
@@ -1746,6 +2559,7 @@ class PersistedCollectionRuntime<
|
|
|
1746
2559
|
},
|
|
1747
2560
|
adapter,
|
|
1748
2561
|
)
|
|
2562
|
+
if (!hydrated) return false
|
|
1749
2563
|
return lifecycleGeneration === this.lifecycleGeneration
|
|
1750
2564
|
? appliedCursor
|
|
1751
2565
|
: undefined
|
|
@@ -1783,17 +2597,19 @@ class PersistedCollectionRuntime<
|
|
|
1783
2597
|
)
|
|
1784
2598
|
if (lifecycleGeneration !== this.lifecycleGeneration) return undefined
|
|
1785
2599
|
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
//
|
|
1789
|
-
|
|
2600
|
+
// Eager and unmanaged sources start at the hydrate boundary so their first
|
|
2601
|
+
// transaction binds to its sequence. A managed on-demand source waits
|
|
2602
|
+
// until the scope exits and any concurrent private rotation settles.
|
|
2603
|
+
if (!this.cacheGenerationClaim || this.syncMode !== `on-demand`) {
|
|
2604
|
+
onStartupMetadataLoaded()
|
|
2605
|
+
}
|
|
1790
2606
|
const appliedCursor =
|
|
1791
2607
|
this.syncMode !== `on-demand`
|
|
1792
2608
|
? await this.hydrateBaseline(lifecycleGeneration, adapter)
|
|
1793
2609
|
: undefined
|
|
1794
2610
|
return lifecycleGeneration === this.lifecycleGeneration
|
|
1795
2611
|
? {
|
|
1796
|
-
appliedCursor,
|
|
2612
|
+
appliedCursor: appliedCursor === false ? undefined : appliedCursor,
|
|
1797
2613
|
indexBootstrapSnapshot,
|
|
1798
2614
|
completedLocalIndexSignatures,
|
|
1799
2615
|
}
|
|
@@ -1803,11 +2619,23 @@ class PersistedCollectionRuntime<
|
|
|
1803
2619
|
private async loadStartupMetadataInternal(
|
|
1804
2620
|
lifecycleGeneration: number,
|
|
1805
2621
|
adapter: HydrationPersistenceAdapter,
|
|
1806
|
-
): Promise<
|
|
1807
|
-
const
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
2622
|
+
): Promise<boolean> {
|
|
2623
|
+
const claimId = this.cacheGenerationClaim?.claimId
|
|
2624
|
+
const storageCollectionId = this.storageCollectionId
|
|
2625
|
+
const snapshot = await adapter.loadResumeSnapshot(
|
|
2626
|
+
this.storageTargetFor(storageCollectionId, claimId),
|
|
2627
|
+
{
|
|
2628
|
+
includeRows: false,
|
|
2629
|
+
cacheGenerationClaimId: claimId,
|
|
2630
|
+
},
|
|
2631
|
+
)
|
|
2632
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return false
|
|
2633
|
+
if (claimId !== this.cacheGenerationClaim?.claimId) return false
|
|
2634
|
+
if (storageCollectionId !== this.storageCollectionId) return false
|
|
2635
|
+
if (claimId && !(await this.renewCacheClaim(adapter))) return false
|
|
2636
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return false
|
|
2637
|
+
if (claimId !== this.cacheGenerationClaim?.claimId) return false
|
|
2638
|
+
if (storageCollectionId !== this.storageCollectionId) return false
|
|
1811
2639
|
this.persistedResumeGeneration = this.getResumeSnapshotGeneration(snapshot)
|
|
1812
2640
|
this.persistedKeySetEvidence = snapshot.keySet
|
|
1813
2641
|
this.observeStreamPosition(
|
|
@@ -1819,6 +2647,7 @@ class PersistedCollectionRuntime<
|
|
|
1819
2647
|
snapshot.collectionMetadata,
|
|
1820
2648
|
)
|
|
1821
2649
|
await whenSyncAccepted(applied)
|
|
2650
|
+
return true
|
|
1822
2651
|
}
|
|
1823
2652
|
|
|
1824
2653
|
private async loadCollectionMetadataSnapshot(
|
|
@@ -1828,7 +2657,9 @@ class PersistedCollectionRuntime<
|
|
|
1828
2657
|
return []
|
|
1829
2658
|
}
|
|
1830
2659
|
|
|
1831
|
-
return adapter.loadCollectionMetadata(this.
|
|
2660
|
+
return adapter.loadCollectionMetadata(this.storageTargetFor(), {
|
|
2661
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
2662
|
+
})
|
|
1832
2663
|
}
|
|
1833
2664
|
|
|
1834
2665
|
private replaceCollectionMetadataSnapshot(
|
|
@@ -1839,38 +2670,42 @@ class PersistedCollectionRuntime<
|
|
|
1839
2670
|
!this.syncControls.commit ||
|
|
1840
2671
|
!this.syncControls.metadata
|
|
1841
2672
|
) {
|
|
1842
|
-
return true
|
|
1843
|
-
}
|
|
1844
|
-
|
|
1845
|
-
const nextMetadata = new Map(
|
|
1846
|
-
collectionMetadata.map(({ key, value }) => [key, value]),
|
|
1847
|
-
)
|
|
1848
|
-
const currentKeys = this.syncControls.metadata.collection
|
|
1849
|
-
.list()
|
|
1850
|
-
.map(({ key }) => key)
|
|
2673
|
+
return true
|
|
2674
|
+
}
|
|
1851
2675
|
|
|
1852
2676
|
return this.withInternalApply(() => {
|
|
1853
2677
|
this.syncControls.begin?.()
|
|
1854
|
-
|
|
1855
|
-
currentKeys.forEach((key) => {
|
|
1856
|
-
if (!nextMetadata.has(key)) {
|
|
1857
|
-
this.syncControls.metadata?.collection.delete(key)
|
|
1858
|
-
}
|
|
1859
|
-
})
|
|
1860
|
-
|
|
1861
|
-
nextMetadata.forEach((value, key) => {
|
|
1862
|
-
this.syncControls.metadata?.collection.set(key, value)
|
|
1863
|
-
})
|
|
1864
|
-
|
|
2678
|
+
this.applyCollectionMetadataSnapshot(collectionMetadata)
|
|
1865
2679
|
return this.syncControls.commit?.() ?? true
|
|
1866
2680
|
})
|
|
1867
2681
|
}
|
|
1868
2682
|
|
|
2683
|
+
private applyCollectionMetadataSnapshot(
|
|
2684
|
+
collectionMetadata: Array<{ key: string; value: unknown }>,
|
|
2685
|
+
): void {
|
|
2686
|
+
const metadata = this.syncControls.metadata
|
|
2687
|
+
if (!metadata) return
|
|
2688
|
+
const next = new Map(
|
|
2689
|
+
collectionMetadata.map(({ key, value }) => [key, value]),
|
|
2690
|
+
)
|
|
2691
|
+
for (const { key } of metadata.collection.list()) {
|
|
2692
|
+
if (!next.has(key)) metadata.collection.delete(key)
|
|
2693
|
+
}
|
|
2694
|
+
for (const [key, value] of next) metadata.collection.set(key, value)
|
|
2695
|
+
}
|
|
2696
|
+
|
|
1869
2697
|
async loadSubset(
|
|
1870
2698
|
options: LoadSubsetOptions,
|
|
1871
2699
|
upstreamLoadSubset?: LoadSubsetFn,
|
|
2700
|
+
onHydrationStarted?: () => void,
|
|
2701
|
+
isDemandRetained?: () => boolean,
|
|
1872
2702
|
): Promise<void> {
|
|
1873
2703
|
this.throwIfTerminal()
|
|
2704
|
+
await this.ensureCacheClaimActive()
|
|
2705
|
+
// The caller can retire demand while claim renewal or rotation is awaited.
|
|
2706
|
+
// No acquisition may start after that retirement.
|
|
2707
|
+
if (options.signal?.aborted || (isDemandRetained && !isDemandRetained()))
|
|
2708
|
+
return
|
|
1874
2709
|
const lifecycleGeneration = this.lifecycleGeneration
|
|
1875
2710
|
const subsetKey = this.getSubsetKey(options)
|
|
1876
2711
|
const truncateGeneration = this.sourceTruncateGeneration
|
|
@@ -1881,28 +2716,63 @@ class PersistedCollectionRuntime<
|
|
|
1881
2716
|
this.activeSubsets.set(subsetKey, options)
|
|
1882
2717
|
const appliedCursor = this.appliedReceiptSequence
|
|
1883
2718
|
try {
|
|
1884
|
-
await this.applyMutex.run(
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
{
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
2719
|
+
const hydrated = await this.applyMutex.run(
|
|
2720
|
+
async (): Promise<boolean | `aborted`> => {
|
|
2721
|
+
if (options.signal?.aborted) return `aborted`
|
|
2722
|
+
try {
|
|
2723
|
+
return await this.runInHydrationScope<boolean | `aborted`>(
|
|
2724
|
+
(adapter) => {
|
|
2725
|
+
if (options.signal?.aborted) {
|
|
2726
|
+
return Promise.resolve(`aborted`)
|
|
2727
|
+
}
|
|
2728
|
+
onHydrationStarted?.()
|
|
2729
|
+
return this.hydrateSubsetUnsafe(
|
|
2730
|
+
options,
|
|
2731
|
+
{
|
|
2732
|
+
lifecycleGeneration,
|
|
2733
|
+
requestLocalLoadFailure: true,
|
|
2734
|
+
rejectBufferedReplayFailure: true,
|
|
2735
|
+
},
|
|
2736
|
+
adapter,
|
|
2737
|
+
)
|
|
1893
2738
|
},
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
await this.flushQueuedTxCommittedUnsafe()
|
|
2739
|
+
)
|
|
2740
|
+
} finally {
|
|
2741
|
+
if (
|
|
2742
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
2743
|
+
!this.getCurrentTerminalFailure()
|
|
2744
|
+
) {
|
|
2745
|
+
await this.flushQueuedTxCommittedUnsafe()
|
|
2746
|
+
}
|
|
1903
2747
|
}
|
|
2748
|
+
},
|
|
2749
|
+
)
|
|
2750
|
+
if (hydrated === `aborted`) {
|
|
2751
|
+
if (this.activeSubsets.get(subsetKey) === options) {
|
|
2752
|
+
this.activeSubsets.delete(subsetKey)
|
|
1904
2753
|
}
|
|
1905
|
-
|
|
2754
|
+
return
|
|
2755
|
+
}
|
|
2756
|
+
if (!hydrated) {
|
|
2757
|
+
// The old cache read is discarded. Recovery reacquires this retained
|
|
2758
|
+
// demand under the new storage ID, so await that acquisition rather
|
|
2759
|
+
// than starting another source request with old request data.
|
|
2760
|
+
await this.ensureCacheClaimActive()
|
|
2761
|
+
const rotatedOptions = this.rotatedAcquisitions.get(options)
|
|
2762
|
+
if (
|
|
2763
|
+
rotatedOptions &&
|
|
2764
|
+
this.activeSubsets.get(subsetKey) === options &&
|
|
2765
|
+
!options.signal?.aborted
|
|
2766
|
+
) {
|
|
2767
|
+
await (this.rotatedEnsures.get(options) ??
|
|
2768
|
+
this.persistence.coordinator.requestEnsureRemoteSubset(
|
|
2769
|
+
this.storageCollectionId,
|
|
2770
|
+
rotatedOptions,
|
|
2771
|
+
))
|
|
2772
|
+
await this.applyLeaderCommitsBeforeReply(options)
|
|
2773
|
+
}
|
|
2774
|
+
return
|
|
2775
|
+
}
|
|
1906
2776
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
1907
2777
|
await this.waitForAppliedReceiptsAfter(appliedCursor)
|
|
1908
2778
|
// Leadership can change while hydration waits. Validate newly remote
|
|
@@ -1924,14 +2794,20 @@ class PersistedCollectionRuntime<
|
|
|
1924
2794
|
options.signal?.aborted ||
|
|
1925
2795
|
this.activeSubsets.get(this.getSubsetKey(options)) !== options
|
|
1926
2796
|
) {
|
|
2797
|
+
if (
|
|
2798
|
+
options.signal?.aborted &&
|
|
2799
|
+
this.activeSubsets.get(subsetKey) === options
|
|
2800
|
+
) {
|
|
2801
|
+
this.activeSubsets.delete(subsetKey)
|
|
2802
|
+
}
|
|
1927
2803
|
return
|
|
1928
2804
|
}
|
|
1929
2805
|
|
|
1930
2806
|
if (this.canRouteRemoteDemandThroughCoordinator()) {
|
|
1931
2807
|
try {
|
|
1932
2808
|
await this.persistence.coordinator.requestEnsureRemoteSubset(
|
|
1933
|
-
this.
|
|
1934
|
-
options,
|
|
2809
|
+
this.storageCollectionId,
|
|
2810
|
+
this.rotatedAcquisitions.get(options) ?? options,
|
|
1935
2811
|
)
|
|
1936
2812
|
} catch (error) {
|
|
1937
2813
|
if (
|
|
@@ -1947,6 +2823,7 @@ class PersistedCollectionRuntime<
|
|
|
1947
2823
|
this.queueRemoteSubsetEnsure(options)
|
|
1948
2824
|
throw error
|
|
1949
2825
|
}
|
|
2826
|
+
await this.applyLeaderCommitsBeforeReply(options)
|
|
1950
2827
|
return
|
|
1951
2828
|
}
|
|
1952
2829
|
|
|
@@ -1974,6 +2851,7 @@ class PersistedCollectionRuntime<
|
|
|
1974
2851
|
const failure = this.getCurrentTerminalFailure()
|
|
1975
2852
|
if (failure) return Promise.reject(failure.error)
|
|
1976
2853
|
if (
|
|
2854
|
+
this.hasExpiredCacheClaim() ||
|
|
1977
2855
|
!this.startupSettled ||
|
|
1978
2856
|
this.isHydratingNow() ||
|
|
1979
2857
|
this.canRouteRemoteDemandThroughCoordinator() ||
|
|
@@ -2036,6 +2914,8 @@ class PersistedCollectionRuntime<
|
|
|
2036
2914
|
const subsetKey = this.getSubsetKey(options)
|
|
2037
2915
|
this.activeSubsets.delete(subsetKey)
|
|
2038
2916
|
this.pendingRemoteSubsetEnsures.delete(subsetKey)
|
|
2917
|
+
const remoteOptions = this.rotatedAcquisitions.get(options) ?? options
|
|
2918
|
+
this.rotatedAcquisitions.delete(options)
|
|
2039
2919
|
const demandKey = getLoadSubsetDemandKey(options)
|
|
2040
2920
|
const stillActive = Array.from(this.activeSubsets.values()).some(
|
|
2041
2921
|
(active) => getLoadSubsetDemandKey(active) === demandKey,
|
|
@@ -2043,7 +2923,7 @@ class PersistedCollectionRuntime<
|
|
|
2043
2923
|
if (!stillActive) this.hydratedDemands.delete(demandKey)
|
|
2044
2924
|
if (this.mode === `sync-present`) {
|
|
2045
2925
|
void this.persistence.coordinator
|
|
2046
|
-
.requestReleaseRemoteSubset(this.
|
|
2926
|
+
.requestReleaseRemoteSubset(this.storageCollectionId, remoteOptions)
|
|
2047
2927
|
.catch((error) => {
|
|
2048
2928
|
this.reportSyncError(error)
|
|
2049
2929
|
})
|
|
@@ -2066,10 +2946,11 @@ class PersistedCollectionRuntime<
|
|
|
2066
2946
|
|
|
2067
2947
|
async forceReloadSubset(options: LoadSubsetOptions): Promise<void> {
|
|
2068
2948
|
this.throwIfTerminal()
|
|
2949
|
+
await this.ensureCacheClaimActive()
|
|
2069
2950
|
const lifecycleGeneration = this.lifecycleGeneration
|
|
2070
2951
|
// A one-shot refresh does not acquire an enduring subscription lease.
|
|
2071
|
-
await this.applyMutex.run(async () => {
|
|
2072
|
-
await this.runInHydrationScope((adapter) =>
|
|
2952
|
+
const hydrated = await this.applyMutex.run(async () => {
|
|
2953
|
+
const result = await this.runInHydrationScope((adapter) =>
|
|
2073
2954
|
this.hydrateSubsetUnsafe(
|
|
2074
2955
|
options,
|
|
2075
2956
|
{
|
|
@@ -2082,7 +2963,9 @@ class PersistedCollectionRuntime<
|
|
|
2082
2963
|
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
2083
2964
|
await this.flushQueuedTxCommittedUnsafe()
|
|
2084
2965
|
}
|
|
2966
|
+
return result
|
|
2085
2967
|
})
|
|
2968
|
+
if (!hydrated) await this.ensureCacheClaimActive()
|
|
2086
2969
|
}
|
|
2087
2970
|
|
|
2088
2971
|
queueHydrationBufferedTransaction(
|
|
@@ -2093,6 +2976,7 @@ class PersistedCollectionRuntime<
|
|
|
2093
2976
|
transaction.rejectApplied?.(failure.error)
|
|
2094
2977
|
return
|
|
2095
2978
|
}
|
|
2979
|
+
transaction.admittedStorageCollectionId = this.storageCollectionId
|
|
2096
2980
|
this.queuedHydrationTransactions.push(transaction)
|
|
2097
2981
|
}
|
|
2098
2982
|
|
|
@@ -2101,6 +2985,7 @@ class PersistedCollectionRuntime<
|
|
|
2101
2985
|
): Promise<void> {
|
|
2102
2986
|
const failure = this.getCurrentTerminalFailure()
|
|
2103
2987
|
if (failure) return Promise.reject(failure.error)
|
|
2988
|
+
transaction.admittedStorageCollectionId = this.storageCollectionId
|
|
2104
2989
|
return this.applyMutex.run(async () => {
|
|
2105
2990
|
await this.applyBufferedSyncTransactionUnsafe(transaction)
|
|
2106
2991
|
})
|
|
@@ -2233,7 +3118,10 @@ class PersistedCollectionRuntime<
|
|
|
2233
3118
|
attempt(() => {
|
|
2234
3119
|
remoteReleases.push(
|
|
2235
3120
|
this.persistence.coordinator
|
|
2236
|
-
.requestReleaseRemoteSubset(
|
|
3121
|
+
.requestReleaseRemoteSubset(
|
|
3122
|
+
this.storageCollectionId,
|
|
3123
|
+
this.rotatedAcquisitions.get(options) ?? options,
|
|
3124
|
+
)
|
|
2237
3125
|
.catch((error) => {
|
|
2238
3126
|
this.reportSyncError(error)
|
|
2239
3127
|
}),
|
|
@@ -2247,6 +3135,7 @@ class PersistedCollectionRuntime<
|
|
|
2247
3135
|
|
|
2248
3136
|
attempt(() => this.remoteSubsetOwnerUnsubscribe?.())
|
|
2249
3137
|
this.remoteSubsetOwnerUnsubscribe = null
|
|
3138
|
+
this.remoteSubsetOwner = undefined
|
|
2250
3139
|
|
|
2251
3140
|
attempt(() => this.indexAddedUnsubscribe?.())
|
|
2252
3141
|
this.indexAddedUnsubscribe = null
|
|
@@ -2260,6 +3149,16 @@ class PersistedCollectionRuntime<
|
|
|
2260
3149
|
}
|
|
2261
3150
|
|
|
2262
3151
|
this.pendingRemoteSubsetEnsures.clear()
|
|
3152
|
+
this.rotatedAcquisitions.clear()
|
|
3153
|
+
for (const [options, storageId] of this.scopedRefreshes) {
|
|
3154
|
+
remoteReleases.push(
|
|
3155
|
+
this.persistence.coordinator.requestReleaseRemoteSubset(
|
|
3156
|
+
storageId,
|
|
3157
|
+
options,
|
|
3158
|
+
),
|
|
3159
|
+
)
|
|
3160
|
+
}
|
|
3161
|
+
this.scopedRefreshes.clear()
|
|
2263
3162
|
this.activeSubsets.clear()
|
|
2264
3163
|
this.hydratedDemands.clear()
|
|
2265
3164
|
for (const transaction of this.queuedHydrationTransactions) {
|
|
@@ -2269,6 +3168,15 @@ class PersistedCollectionRuntime<
|
|
|
2269
3168
|
this.queuedTxCommitted.length = 0
|
|
2270
3169
|
this.clearSyncControls()
|
|
2271
3170
|
this.collection = null
|
|
3171
|
+
const claim = this.cacheGenerationClaim
|
|
3172
|
+
this.cacheGenerationClaim = undefined
|
|
3173
|
+
this.clearCacheClaimRenewal()
|
|
3174
|
+
this.cacheClaimRecovery = null
|
|
3175
|
+
if (claim && this.persistence.adapter.releaseCacheGenerationClaim) {
|
|
3176
|
+
remoteReleases.push(
|
|
3177
|
+
this.persistence.adapter.releaseCacheGenerationClaim(claim.claimId),
|
|
3178
|
+
)
|
|
3179
|
+
}
|
|
2272
3180
|
return Promise.all(remoteReleases).then(() => {
|
|
2273
3181
|
if (failures.length === 1) throw failures[0]
|
|
2274
3182
|
if (failures.length > 1) {
|
|
@@ -2281,8 +3189,16 @@ class PersistedCollectionRuntime<
|
|
|
2281
3189
|
|
|
2282
3190
|
private advanceLifecycle(): void {
|
|
2283
3191
|
this.lifecycleGeneration++
|
|
3192
|
+
this.clearCacheClaimRenewal()
|
|
3193
|
+
this.cacheClaimRecovery = null
|
|
3194
|
+
this.scopedRecovery = false
|
|
3195
|
+
this.scopedRecoveryPending = false
|
|
3196
|
+
this.scopedRecoverySequence++
|
|
3197
|
+
this.recoverQueuedAfterLocalLoadFailure = false
|
|
2284
3198
|
this.persistedReadiness?.set({ status: `loading` })
|
|
2285
3199
|
this.startupSettled = false
|
|
3200
|
+
this.publicRowVersion = 0
|
|
3201
|
+
this.resetTermFence = undefined
|
|
2286
3202
|
this.hydratedDemands.clear()
|
|
2287
3203
|
this.startupMetadataPromise = null
|
|
2288
3204
|
this.startPromise = null
|
|
@@ -2317,8 +3233,10 @@ class PersistedCollectionRuntime<
|
|
|
2317
3233
|
options: LoadSubsetOptions,
|
|
2318
3234
|
adapter: HydrationPersistenceAdapter,
|
|
2319
3235
|
): Promise<Array<{ key: TKey; value: T; metadata?: unknown }>> {
|
|
2320
|
-
|
|
3236
|
+
if (this.scopedRecovery) return Promise.resolve([])
|
|
3237
|
+
return adapter.loadSubset(this.storageTargetFor(), options, {
|
|
2321
3238
|
requiredIndexSignatures: this.getRequiredIndexSignatures(),
|
|
3239
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
2322
3240
|
}) as Promise<Array<{ key: TKey; value: T; metadata?: unknown }>>
|
|
2323
3241
|
}
|
|
2324
3242
|
|
|
@@ -2329,17 +3247,36 @@ class PersistedCollectionRuntime<
|
|
|
2329
3247
|
return []
|
|
2330
3248
|
}
|
|
2331
3249
|
|
|
2332
|
-
return this.persistence.adapter.scanRows(
|
|
2333
|
-
this.
|
|
2334
|
-
|
|
2335
|
-
) as Promise<Array<PersistedScannedRow<T, TKey>>>
|
|
3250
|
+
return this.persistence.adapter.scanRows(this.storageTargetFor(), options, {
|
|
3251
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
3252
|
+
}) as Promise<Array<PersistedScannedRow<T, TKey>>>
|
|
2336
3253
|
}
|
|
2337
3254
|
|
|
2338
3255
|
async scanPersistedRows(
|
|
2339
3256
|
options?: PersistedRowScanOptions,
|
|
2340
3257
|
): Promise<Array<PersistedScannedRow<T, TKey>>> {
|
|
2341
|
-
|
|
2342
|
-
|
|
3258
|
+
for (;;) {
|
|
3259
|
+
this.throwIfTerminal()
|
|
3260
|
+
await this.ensureCacheClaimActive()
|
|
3261
|
+
const claimId = this.cacheGenerationClaim?.claimId
|
|
3262
|
+
const storageCollectionId = this.storageCollectionId
|
|
3263
|
+
const rows = await this.applyMutex.run(() =>
|
|
3264
|
+
this.scanPersistedRowsUnsafe(options),
|
|
3265
|
+
)
|
|
3266
|
+
if (this.scopedRecoveryPending) {
|
|
3267
|
+
throw new SyncTransactionAbortedError()
|
|
3268
|
+
}
|
|
3269
|
+
if (
|
|
3270
|
+
claimId !== this.cacheGenerationClaim?.claimId ||
|
|
3271
|
+
storageCollectionId !== this.storageCollectionId
|
|
3272
|
+
) {
|
|
3273
|
+
continue
|
|
3274
|
+
}
|
|
3275
|
+
if (claimId && !(await this.renewCacheClaim(this.persistence.adapter))) {
|
|
3276
|
+
continue
|
|
3277
|
+
}
|
|
3278
|
+
return rows
|
|
3279
|
+
}
|
|
2343
3280
|
}
|
|
2344
3281
|
|
|
2345
3282
|
private async hydrateSubsetUnsafe(
|
|
@@ -2351,8 +3288,9 @@ class PersistedCollectionRuntime<
|
|
|
2351
3288
|
rejectBufferedReplayFailure?: boolean
|
|
2352
3289
|
},
|
|
2353
3290
|
adapter: HydrationPersistenceAdapter,
|
|
2354
|
-
): Promise<
|
|
3291
|
+
): Promise<boolean> {
|
|
2355
3292
|
let rowsLoaded = false
|
|
3293
|
+
let boundRowVersion: number | undefined
|
|
2356
3294
|
let replayFailure: { reason: unknown } | undefined
|
|
2357
3295
|
const resetSequence = this.resetSequence
|
|
2358
3296
|
this.hydrationSequence++
|
|
@@ -2363,25 +3301,53 @@ class PersistedCollectionRuntime<
|
|
|
2363
3301
|
this.hydratingGeneration = config.lifecycleGeneration
|
|
2364
3302
|
try {
|
|
2365
3303
|
let rows: Array<{ key: TKey; value: T; metadata?: unknown }>
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
3304
|
+
let bindResumeSnapshot: (() => void) | undefined
|
|
3305
|
+
if (config.bindKeySetEvidence && !this.scopedRecovery) {
|
|
3306
|
+
const snapshot = await adapter.loadResumeSnapshot(
|
|
3307
|
+
this.storageTargetFor(),
|
|
3308
|
+
{
|
|
3309
|
+
requiredIndexSignatures: this.getRequiredIndexSignatures(),
|
|
3310
|
+
includeRows: true,
|
|
3311
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
3312
|
+
},
|
|
3313
|
+
)
|
|
2371
3314
|
rows = snapshot.rows as Array<{
|
|
2372
3315
|
key: TKey
|
|
2373
3316
|
value: T
|
|
2374
3317
|
metadata?: unknown
|
|
2375
3318
|
}>
|
|
2376
|
-
if (config.lifecycleGeneration !== this.lifecycleGeneration)
|
|
2377
|
-
|
|
2378
|
-
this.
|
|
3319
|
+
if (config.lifecycleGeneration !== this.lifecycleGeneration)
|
|
3320
|
+
return true
|
|
3321
|
+
if (resetSequence !== this.resetSequence) return true
|
|
3322
|
+
bindResumeSnapshot = () => this.bindResumeSnapshotEvidence(snapshot)
|
|
3323
|
+
boundRowVersion = snapshot.latestRowVersion
|
|
2379
3324
|
} else {
|
|
2380
3325
|
rows = await this.loadSubsetRowsUnsafe(options, adapter)
|
|
2381
3326
|
}
|
|
3327
|
+
const complete =
|
|
3328
|
+
this.syncMode === `on-demand` &&
|
|
3329
|
+
!this.scopedRecovery &&
|
|
3330
|
+
options.where === undefined &&
|
|
3331
|
+
options.limit === undefined &&
|
|
3332
|
+
options.offset === undefined &&
|
|
3333
|
+
options.cursor === undefined
|
|
3334
|
+
const validationRows = complete ? rows : undefined
|
|
2382
3335
|
rowsLoaded = true
|
|
2383
|
-
if (config.lifecycleGeneration !== this.lifecycleGeneration) return
|
|
2384
|
-
if (resetSequence !== this.resetSequence) return
|
|
3336
|
+
if (config.lifecycleGeneration !== this.lifecycleGeneration) return true
|
|
3337
|
+
if (resetSequence !== this.resetSequence) return true
|
|
3338
|
+
|
|
3339
|
+
// SQLite validated the claim at read admission. A paused sync run
|
|
3340
|
+
// can resume after expiry before its renewal timer runs, so cached
|
|
3341
|
+
// rows need fresh authority before they enter the public Collection.
|
|
3342
|
+
if (
|
|
3343
|
+
this.cacheGenerationClaim &&
|
|
3344
|
+
!(await this.renewCacheClaim(adapter))
|
|
3345
|
+
) {
|
|
3346
|
+
return false
|
|
3347
|
+
}
|
|
3348
|
+
if (config.lifecycleGeneration !== this.lifecycleGeneration) return true
|
|
3349
|
+
if (resetSequence !== this.resetSequence) return true
|
|
3350
|
+
bindResumeSnapshot?.()
|
|
2385
3351
|
|
|
2386
3352
|
if (
|
|
2387
3353
|
!config.bindKeySetEvidence ||
|
|
@@ -2390,8 +3356,14 @@ class PersistedCollectionRuntime<
|
|
|
2390
3356
|
for (const row of rows) {
|
|
2391
3357
|
hydrationContext.suppliedRowKeys.add(row.key)
|
|
2392
3358
|
}
|
|
2393
|
-
const applied = this.applyRowsToCollection(rows)
|
|
3359
|
+
const applied = this.applyRowsToCollection(rows, validationRows)
|
|
2394
3360
|
await whenSyncAccepted(applied)
|
|
3361
|
+
if (boundRowVersion !== undefined) {
|
|
3362
|
+
this.publicRowVersion = Math.max(
|
|
3363
|
+
this.publicRowVersion,
|
|
3364
|
+
boundRowVersion,
|
|
3365
|
+
)
|
|
3366
|
+
}
|
|
2395
3367
|
}
|
|
2396
3368
|
} finally {
|
|
2397
3369
|
if (
|
|
@@ -2402,11 +3374,29 @@ class PersistedCollectionRuntime<
|
|
|
2402
3374
|
}
|
|
2403
3375
|
}
|
|
2404
3376
|
|
|
2405
|
-
if (config.lifecycleGeneration !== this.lifecycleGeneration) return
|
|
2406
|
-
|
|
2407
|
-
|
|
3377
|
+
if (config.lifecycleGeneration !== this.lifecycleGeneration) return true
|
|
3378
|
+
if (!this.scopedRecoveryPending) {
|
|
3379
|
+
replayFailure =
|
|
3380
|
+
await this.flushQueuedHydrationTransactionsUnsafe(adapter)
|
|
3381
|
+
}
|
|
3382
|
+
if (config.lifecycleGeneration !== this.lifecycleGeneration) return true
|
|
2408
3383
|
} catch (error) {
|
|
3384
|
+
// A read admitted under a live claim may fail at the adapter's later
|
|
3385
|
+
// claim check after this run wakes from expiry. Its old cache has no
|
|
3386
|
+
// authority; the retained demand must reacquire under a new generation.
|
|
3387
|
+
if (
|
|
3388
|
+
config.requestLocalLoadFailure &&
|
|
3389
|
+
!rowsLoaded &&
|
|
3390
|
+
this.cacheGenerationClaim &&
|
|
3391
|
+
this.hasExpiredCacheClaim()
|
|
3392
|
+
) {
|
|
3393
|
+
return false
|
|
3394
|
+
}
|
|
2409
3395
|
if (config.requestLocalLoadFailure && !rowsLoaded) {
|
|
3396
|
+
if (this.scopedRecoveryPending) {
|
|
3397
|
+
this.recoverQueuedAfterLocalLoadFailure = true
|
|
3398
|
+
throw error
|
|
3399
|
+
}
|
|
2410
3400
|
// Keep admitting source work to the hydration queue while recovery
|
|
2411
3401
|
// reconstructs any persisted baseline required by partial updates.
|
|
2412
3402
|
// The failed subset itself owns no rows, so recovery must evaluate
|
|
@@ -2440,6 +3430,7 @@ class PersistedCollectionRuntime<
|
|
|
2440
3430
|
if (replayFailure && config.rejectBufferedReplayFailure) {
|
|
2441
3431
|
throw replayFailure.reason
|
|
2442
3432
|
}
|
|
3433
|
+
return true
|
|
2443
3434
|
}
|
|
2444
3435
|
|
|
2445
3436
|
private async recoverBufferedTransactionsAfterLocalLoadFailureUnsafe(
|
|
@@ -2495,24 +3486,26 @@ class PersistedCollectionRuntime<
|
|
|
2495
3486
|
getPresence(key) === `unknown`
|
|
2496
3487
|
) {
|
|
2497
3488
|
if (snapshotRows === undefined) {
|
|
2498
|
-
const snapshot =
|
|
2499
|
-
|
|
2500
|
-
{
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
3489
|
+
const snapshot = this.scopedRecovery
|
|
3490
|
+
? undefined
|
|
3491
|
+
: await adapter.loadResumeSnapshot(this.storageTargetFor(), {
|
|
3492
|
+
requiredIndexSignatures:
|
|
3493
|
+
this.getRequiredIndexSignatures(),
|
|
3494
|
+
includeRows: true,
|
|
3495
|
+
cacheGenerationClaimId:
|
|
3496
|
+
this.cacheGenerationClaim?.claimId,
|
|
3497
|
+
})
|
|
2505
3498
|
this.throwIfLifecycleReplaced(lifecycleGeneration)
|
|
2506
3499
|
if (transaction.signal?.aborted) break
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
snapshot.rows as Array<{
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
)
|
|
3500
|
+
const rows =
|
|
3501
|
+
snapshot && !this.scopedRecovery
|
|
3502
|
+
? (snapshot.rows as Array<{
|
|
3503
|
+
key: TKey
|
|
3504
|
+
value: T
|
|
3505
|
+
metadata?: unknown
|
|
3506
|
+
}>)
|
|
3507
|
+
: []
|
|
3508
|
+
snapshotRows = new Map(rows.map((row) => [row.key, row]))
|
|
2516
3509
|
}
|
|
2517
3510
|
const baseline = snapshotRows.get(key)
|
|
2518
3511
|
if (baseline === undefined) {
|
|
@@ -2573,6 +3566,8 @@ class PersistedCollectionRuntime<
|
|
|
2573
3566
|
|
|
2574
3567
|
private applyRowsToCollection(
|
|
2575
3568
|
rows: Array<{ key: TKey; value: T; metadata?: unknown }>,
|
|
3569
|
+
validationRows?: Array<{ key: TKey; value: T; metadata?: unknown }>,
|
|
3570
|
+
collectionMetadata?: Array<{ key: string; value: unknown }>,
|
|
2576
3571
|
): SyncAppliedReceipt {
|
|
2577
3572
|
if (
|
|
2578
3573
|
!this.syncControls.begin ||
|
|
@@ -2585,10 +3580,68 @@ class PersistedCollectionRuntime<
|
|
|
2585
3580
|
return this.withInternalApply(() => {
|
|
2586
3581
|
this.syncControls.begin?.()
|
|
2587
3582
|
|
|
3583
|
+
// A full durable snapshot validates already public keys. Only the
|
|
3584
|
+
// requested subset may add keys to the public Collection.
|
|
3585
|
+
if (validationRows) {
|
|
3586
|
+
const durableRows = new Map(validationRows.map((row) => [row.key, row]))
|
|
3587
|
+
for (const key of this.collection?.base.keys() ?? []) {
|
|
3588
|
+
if (this.collection?._hasHydratedKey(key)) continue
|
|
3589
|
+
const durable = durableRows.get(key)
|
|
3590
|
+
if (!durable) {
|
|
3591
|
+
this.syncControls.write?.({ type: `delete`, key })
|
|
3592
|
+
} else {
|
|
3593
|
+
const valueChanged = !equalPersistedSnapshotValues(
|
|
3594
|
+
this.collection?.base.get(key),
|
|
3595
|
+
durable.value,
|
|
3596
|
+
)
|
|
3597
|
+
const metadataChanged = !equalPersistedSnapshotValues(
|
|
3598
|
+
this.syncControls.metadata?.row.get(key),
|
|
3599
|
+
durable.metadata,
|
|
3600
|
+
)
|
|
3601
|
+
if (
|
|
3602
|
+
valueChanged ||
|
|
3603
|
+
(metadataChanged && durable.metadata !== undefined)
|
|
3604
|
+
) {
|
|
3605
|
+
// A persisted read supplies a complete row. In partial-update
|
|
3606
|
+
// mode, deleting it within this sync transaction removes fields
|
|
3607
|
+
// that are absent from the durable value before replacement.
|
|
3608
|
+
if (valueChanged) {
|
|
3609
|
+
this.syncControls.write?.({ type: `delete`, key })
|
|
3610
|
+
}
|
|
3611
|
+
this.syncControls.write?.({
|
|
3612
|
+
type: `update`,
|
|
3613
|
+
value: durable.value,
|
|
3614
|
+
metadata: durable.metadata as
|
|
3615
|
+
Record<string, unknown> | undefined,
|
|
3616
|
+
})
|
|
3617
|
+
}
|
|
3618
|
+
if (metadataChanged && durable.metadata === undefined) {
|
|
3619
|
+
this.syncControls.metadata?.row.delete(key)
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
}
|
|
3623
|
+
for (const key of this.collection?._syncedRowMetadataKeysOutsideBase() ??
|
|
3624
|
+
[]) {
|
|
3625
|
+
this.syncControls.metadata?.row.delete(key)
|
|
3626
|
+
}
|
|
3627
|
+
}
|
|
3628
|
+
|
|
2588
3629
|
for (const row of rows) {
|
|
2589
|
-
if (
|
|
3630
|
+
if (
|
|
3631
|
+
(validationRows && this.collection?.base.has(row.key)) ||
|
|
3632
|
+
this.collection?._hasHydratedKey(row.key)
|
|
3633
|
+
) {
|
|
2590
3634
|
continue
|
|
2591
3635
|
}
|
|
3636
|
+
if (
|
|
3637
|
+
this.collection?.base.has(row.key) &&
|
|
3638
|
+
!equalPersistedSnapshotValues(
|
|
3639
|
+
this.collection.base.get(row.key),
|
|
3640
|
+
row.value,
|
|
3641
|
+
)
|
|
3642
|
+
) {
|
|
3643
|
+
this.syncControls.write?.({ type: `delete`, key: row.key })
|
|
3644
|
+
}
|
|
2592
3645
|
this.syncControls.write?.({
|
|
2593
3646
|
type: `update`,
|
|
2594
3647
|
value: row.value,
|
|
@@ -2596,6 +3649,9 @@ class PersistedCollectionRuntime<
|
|
|
2596
3649
|
})
|
|
2597
3650
|
}
|
|
2598
3651
|
|
|
3652
|
+
if (collectionMetadata)
|
|
3653
|
+
this.applyCollectionMetadataSnapshot(collectionMetadata)
|
|
3654
|
+
|
|
2599
3655
|
return this.syncControls.commit?.() ?? true
|
|
2600
3656
|
})
|
|
2601
3657
|
}
|
|
@@ -2620,10 +3676,14 @@ class PersistedCollectionRuntime<
|
|
|
2620
3676
|
const expected = this.persistedResumeGeneration
|
|
2621
3677
|
return (
|
|
2622
3678
|
expected !== undefined &&
|
|
2623
|
-
expected.latestTerm === generation.latestTerm &&
|
|
2624
|
-
expected.latestSeq === generation.latestSeq &&
|
|
2625
3679
|
expected.latestRowVersion === generation.latestRowVersion &&
|
|
2626
|
-
expected.resetEpoch === generation.resetEpoch
|
|
3680
|
+
expected.resetEpoch === generation.resetEpoch &&
|
|
3681
|
+
((expected.latestTerm === generation.latestTerm &&
|
|
3682
|
+
expected.latestSeq === generation.latestSeq) ||
|
|
3683
|
+
// A no-write election reserves a new term at sequence zero. It does
|
|
3684
|
+
// not change the persisted rows or the source resume cursor.
|
|
3685
|
+
(generation.latestTerm > expected.latestTerm &&
|
|
3686
|
+
generation.latestSeq === 0))
|
|
2627
3687
|
)
|
|
2628
3688
|
}
|
|
2629
3689
|
|
|
@@ -2663,6 +3723,7 @@ class PersistedCollectionRuntime<
|
|
|
2663
3723
|
private replaceCollectionSnapshot(
|
|
2664
3724
|
rows: Array<{ key: TKey; value: T; metadata?: unknown }>,
|
|
2665
3725
|
collectionMetadata: Array<{ key: string; value: unknown }>,
|
|
3726
|
+
replayDemand: boolean,
|
|
2666
3727
|
): SyncAppliedReceipt {
|
|
2667
3728
|
if (
|
|
2668
3729
|
!this.syncControls.begin ||
|
|
@@ -2673,18 +3734,45 @@ class PersistedCollectionRuntime<
|
|
|
2673
3734
|
return true
|
|
2674
3735
|
}
|
|
2675
3736
|
|
|
2676
|
-
const
|
|
2677
|
-
|
|
3737
|
+
const base = this.collection?.base
|
|
3738
|
+
const hydratedKeys = new Set(
|
|
3739
|
+
Array.from(base?.keys() ?? []).filter(
|
|
3740
|
+
(key) => this.collection?._hasHydratedKey(key) === true,
|
|
3741
|
+
),
|
|
2678
3742
|
)
|
|
2679
|
-
const
|
|
2680
|
-
|
|
2681
|
-
|
|
3743
|
+
const rowKeys = new Set(rows.map((row) => row.key))
|
|
3744
|
+
// A truncate replays the Collection's demand. A follower routes that
|
|
3745
|
+
// demand to its leader, whose source loads and commits again, so a
|
|
3746
|
+
// follower replaces its rows in place, except after a reset, which owes
|
|
3747
|
+
// the replay.
|
|
3748
|
+
const followsLeader =
|
|
3749
|
+
this.canRouteRemoteDemandThroughCoordinator() &&
|
|
3750
|
+
!this.persistence.coordinator.isLeader(this.storageCollectionId)
|
|
3751
|
+
const truncate = hydratedKeys.size === 0 && (replayDemand || !followsLeader)
|
|
2682
3752
|
|
|
2683
3753
|
return this.withInternalApply(() => {
|
|
2684
3754
|
this.syncControls.begin?.()
|
|
2685
|
-
|
|
3755
|
+
if (truncate) {
|
|
3756
|
+
this.syncControls.truncate?.()
|
|
3757
|
+
} else {
|
|
3758
|
+
for (const key of base?.keys() ?? []) {
|
|
3759
|
+
if (!hydratedKeys.has(key) && !rowKeys.has(key)) {
|
|
3760
|
+
this.syncControls.write?.({ type: `delete`, key })
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
for (const key of this.collection?._syncedRowMetadataKeysOutsideBase() ??
|
|
3764
|
+
[]) {
|
|
3765
|
+
this.syncControls.metadata?.row.delete(key)
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
2686
3768
|
|
|
2687
3769
|
for (const row of rows) {
|
|
3770
|
+
if (hydratedKeys.has(row.key)) continue
|
|
3771
|
+
if (!truncate && base?.has(row.key)) {
|
|
3772
|
+
// Remove stale fields before applying a complete persisted row in
|
|
3773
|
+
// the default partial-update mode, without truncating seed keys.
|
|
3774
|
+
this.syncControls.write?.({ type: `delete`, key: row.key })
|
|
3775
|
+
}
|
|
2688
3776
|
this.syncControls.write?.({
|
|
2689
3777
|
type: `update`,
|
|
2690
3778
|
value: row.value,
|
|
@@ -2692,20 +3780,42 @@ class PersistedCollectionRuntime<
|
|
|
2692
3780
|
})
|
|
2693
3781
|
}
|
|
2694
3782
|
|
|
2695
|
-
|
|
2696
|
-
if (!nextMetadata.has(key)) {
|
|
2697
|
-
this.syncControls.metadata?.collection.delete(key)
|
|
2698
|
-
}
|
|
2699
|
-
})
|
|
2700
|
-
|
|
2701
|
-
nextMetadata.forEach((value, key) => {
|
|
2702
|
-
this.syncControls.metadata?.collection.set(key, value)
|
|
2703
|
-
})
|
|
2704
|
-
|
|
3783
|
+
this.applyCollectionMetadataSnapshot(collectionMetadata)
|
|
2705
3784
|
return this.syncControls.commit?.() ?? true
|
|
2706
3785
|
})
|
|
2707
3786
|
}
|
|
2708
3787
|
|
|
3788
|
+
private matchesCollectionSnapshot(
|
|
3789
|
+
rows: Map<TKey, { value: T; metadata?: unknown }>,
|
|
3790
|
+
collectionMetadata: Array<{ key: string; value: unknown }>,
|
|
3791
|
+
): boolean {
|
|
3792
|
+
if (!this.collection || !this.syncControls.metadata) return false
|
|
3793
|
+
const base = this.collection.base
|
|
3794
|
+
if (base.size !== rows.size) return false
|
|
3795
|
+
for (const [key, row] of rows) {
|
|
3796
|
+
if (
|
|
3797
|
+
!base.has(key) ||
|
|
3798
|
+
!equalPersistedSnapshotValues(base.get(key), row.value) ||
|
|
3799
|
+
!equalPersistedSnapshotValues(
|
|
3800
|
+
this.syncControls.metadata.row.get(key),
|
|
3801
|
+
row.metadata,
|
|
3802
|
+
)
|
|
3803
|
+
) {
|
|
3804
|
+
return false
|
|
3805
|
+
}
|
|
3806
|
+
}
|
|
3807
|
+
const currentMetadata = this.syncControls.metadata.collection.list()
|
|
3808
|
+
if (currentMetadata.length !== collectionMetadata.length) return false
|
|
3809
|
+
const expectedMetadata = new Map(
|
|
3810
|
+
collectionMetadata.map(({ key, value }) => [key, value]),
|
|
3811
|
+
)
|
|
3812
|
+
return currentMetadata.every(
|
|
3813
|
+
({ key, value }) =>
|
|
3814
|
+
expectedMetadata.has(key) &&
|
|
3815
|
+
equalPersistedSnapshotValues(expectedMetadata.get(key), value),
|
|
3816
|
+
)
|
|
3817
|
+
}
|
|
3818
|
+
|
|
2709
3819
|
private async flushQueuedHydrationTransactionsUnsafe(
|
|
2710
3820
|
adapter: HydrationPersistenceAdapter,
|
|
2711
3821
|
): Promise<{ reason: unknown } | undefined> {
|
|
@@ -2742,6 +3852,42 @@ class PersistedCollectionRuntime<
|
|
|
2742
3852
|
}> {
|
|
2743
3853
|
this.throwIfTerminal()
|
|
2744
3854
|
this.throwIfLifecycleReplaced(transaction.lifecycleGeneration)
|
|
3855
|
+
if (
|
|
3856
|
+
transaction.admittedStorageCollectionId !== undefined &&
|
|
3857
|
+
transaction.admittedStorageCollectionId !== this.storageCollectionId
|
|
3858
|
+
) {
|
|
3859
|
+
const error = new SyncTransactionAbortedError()
|
|
3860
|
+
if (!transaction.rejectApplied) throw error
|
|
3861
|
+
transaction.rejectApplied(error)
|
|
3862
|
+
return { applied: false }
|
|
3863
|
+
}
|
|
3864
|
+
if (!transaction.internal && this.scopedRecoveryPending) {
|
|
3865
|
+
const error = new SyncTransactionAbortedError()
|
|
3866
|
+
if (!transaction.rejectApplied) throw error
|
|
3867
|
+
transaction.rejectApplied(error)
|
|
3868
|
+
return { applied: false }
|
|
3869
|
+
}
|
|
3870
|
+
if (
|
|
3871
|
+
!transaction.internal &&
|
|
3872
|
+
this.cacheGenerationClaim &&
|
|
3873
|
+
(scopedAdapter ?? this.persistence.adapter).renewCacheGenerationClaim &&
|
|
3874
|
+
!(await this.renewCacheClaim(scopedAdapter ?? this.persistence.adapter))
|
|
3875
|
+
) {
|
|
3876
|
+
const error = new SyncTransactionAbortedError()
|
|
3877
|
+
// Recovery needs this apply mutex, so start it after the rejected
|
|
3878
|
+
// transaction has released its turn. Its new source snapshots use a
|
|
3879
|
+
// fresh cache generation; this commit must never be replayed into it.
|
|
3880
|
+
queueMicrotask(() => {
|
|
3881
|
+
if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3882
|
+
if (this.scopedRecoveryPending) return
|
|
3883
|
+
void this.startScopedRecovery().catch((failure) => {
|
|
3884
|
+
this.markTerminalFailure(failure, transaction.lifecycleGeneration)
|
|
3885
|
+
})
|
|
3886
|
+
})
|
|
3887
|
+
if (!transaction.rejectApplied) throw error
|
|
3888
|
+
transaction.rejectApplied(error)
|
|
3889
|
+
return { applied: false }
|
|
3890
|
+
}
|
|
2745
3891
|
const abortedBeforeApplication = transaction.signal?.aborted === true
|
|
2746
3892
|
let abortedDuringApplication = false
|
|
2747
3893
|
let applicationReturned = false
|
|
@@ -2834,16 +3980,60 @@ class PersistedCollectionRuntime<
|
|
|
2834
3980
|
return
|
|
2835
3981
|
}
|
|
2836
3982
|
|
|
3983
|
+
const anchor = this.persistedResumeGeneration && {
|
|
3984
|
+
latestRowVersion: this.latestRowVersion,
|
|
3985
|
+
resetEpoch: this.persistedResumeGeneration.resetEpoch,
|
|
3986
|
+
}
|
|
2837
3987
|
const streamPosition = this.nextLocalStreamPosition()
|
|
2838
3988
|
const tx = this.createPersistedTxFromOperations(transaction, streamPosition)
|
|
2839
|
-
const
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
3989
|
+
const storageCollectionId = this.storageCollectionId
|
|
3990
|
+
let response: ApplyCommittedTxResponse
|
|
3991
|
+
try {
|
|
3992
|
+
response = await this.persistence.coordinator.requestApplyCommittedTx(
|
|
3993
|
+
storageCollectionId,
|
|
3994
|
+
tx,
|
|
3995
|
+
// A hydration-scoped adapter is already inside the shared scheduler.
|
|
3996
|
+
// Ordinary source commits enter it before taking the writer lock.
|
|
3997
|
+
scopedAdapter,
|
|
3998
|
+
)
|
|
3999
|
+
} catch (error) {
|
|
4000
|
+
if (
|
|
4001
|
+
!(error instanceof IndeterminateCommitError) ||
|
|
4002
|
+
!anchor ||
|
|
4003
|
+
!this.persistence.coordinator.reconcileCommittedTx
|
|
4004
|
+
) {
|
|
4005
|
+
throw error
|
|
4006
|
+
}
|
|
4007
|
+
let reconciled: ReconcileCommittedTxResponse
|
|
4008
|
+
try {
|
|
4009
|
+
reconciled = await this.persistence.coordinator.reconcileCommittedTx(
|
|
4010
|
+
storageCollectionId,
|
|
4011
|
+
tx,
|
|
4012
|
+
anchor,
|
|
4013
|
+
scopedAdapter,
|
|
4014
|
+
)
|
|
4015
|
+
} catch (reconcileError) {
|
|
4016
|
+
if (reconcileError instanceof PersistedCollectionDurabilityError) {
|
|
4017
|
+
throw reconcileError
|
|
4018
|
+
}
|
|
4019
|
+
throw error
|
|
4020
|
+
}
|
|
4021
|
+
if (!reconciled.ok) {
|
|
4022
|
+
if (reconciled.code === `PERSISTENCE_ERROR`) {
|
|
4023
|
+
throw new PersistedCollectionDurabilityError(
|
|
4024
|
+
`Failed to reconcile collection "${this.collectionId}": ${reconciled.error}`,
|
|
4025
|
+
{
|
|
4026
|
+
cause: reconciled,
|
|
4027
|
+
code: reconciled.sourceCode ?? reconciled.code,
|
|
4028
|
+
path: reconciled.path,
|
|
4029
|
+
},
|
|
4030
|
+
)
|
|
4031
|
+
}
|
|
4032
|
+
throw error
|
|
4033
|
+
}
|
|
4034
|
+
this.recordDurableSourceCommit(transaction, reconciled.committed)
|
|
4035
|
+
return
|
|
4036
|
+
}
|
|
2847
4037
|
if (!response.ok) {
|
|
2848
4038
|
if (response.code === `PERSISTENCE_ERROR`) {
|
|
2849
4039
|
const error = new PersistedCollectionDurabilityError(
|
|
@@ -2860,12 +4050,22 @@ class PersistedCollectionRuntime<
|
|
|
2860
4050
|
`failed to apply external sync transaction through coordinator: ${response.error}`,
|
|
2861
4051
|
)
|
|
2862
4052
|
}
|
|
4053
|
+
this.recordDurableSourceCommit(transaction, {
|
|
4054
|
+
term: response.term,
|
|
4055
|
+
seq: response.seq,
|
|
4056
|
+
rowVersion: response.latestRowVersion,
|
|
4057
|
+
})
|
|
4058
|
+
}
|
|
4059
|
+
|
|
4060
|
+
private recordDurableSourceCommit(
|
|
4061
|
+
transaction: BufferedSyncTransaction<T, TKey>,
|
|
4062
|
+
position: { term: number; seq: number; rowVersion: number },
|
|
4063
|
+
): void {
|
|
2863
4064
|
if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return
|
|
2864
|
-
this.observeStreamPosition(
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
)
|
|
4065
|
+
this.observeStreamPosition(position.term, position.seq, position.rowVersion)
|
|
4066
|
+
if (position.rowVersion === this.publicRowVersion + 1) {
|
|
4067
|
+
this.publicRowVersion = position.rowVersion
|
|
4068
|
+
}
|
|
2869
4069
|
if (
|
|
2870
4070
|
transaction.expectedResumeGenerationOwner ===
|
|
2871
4071
|
this.resumeGenerationOwner &&
|
|
@@ -2873,9 +4073,9 @@ class PersistedCollectionRuntime<
|
|
|
2873
4073
|
) {
|
|
2874
4074
|
this.persistedResumeGeneration = {
|
|
2875
4075
|
...this.persistedResumeGeneration,
|
|
2876
|
-
latestTerm:
|
|
2877
|
-
latestSeq:
|
|
2878
|
-
latestRowVersion:
|
|
4076
|
+
latestTerm: position.term,
|
|
4077
|
+
latestSeq: position.seq,
|
|
4078
|
+
latestRowVersion: position.rowVersion,
|
|
2879
4079
|
}
|
|
2880
4080
|
}
|
|
2881
4081
|
}
|
|
@@ -2887,7 +4087,10 @@ class PersistedCollectionRuntime<
|
|
|
2887
4087
|
this.throwIfTerminal()
|
|
2888
4088
|
this.throwIfLifecycleReplaced(lifecycleGeneration)
|
|
2889
4089
|
try {
|
|
2890
|
-
await this.persistence.adapter.applyCommittedTx(
|
|
4090
|
+
await this.persistence.adapter.applyCommittedTx(
|
|
4091
|
+
this.storageTargetFor(),
|
|
4092
|
+
tx,
|
|
4093
|
+
)
|
|
2891
4094
|
} catch (error) {
|
|
2892
4095
|
throw this.markTerminalFailure(
|
|
2893
4096
|
toPersistedCollectionDurabilityError(this.collectionId, error),
|
|
@@ -2906,6 +4109,7 @@ class PersistedCollectionRuntime<
|
|
|
2906
4109
|
term: streamPosition.term,
|
|
2907
4110
|
seq: streamPosition.seq,
|
|
2908
4111
|
rowVersion: streamPosition.rowVersion,
|
|
4112
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
2909
4113
|
truncate: transaction.truncate,
|
|
2910
4114
|
mutations: transaction.operations.map((operation) =>
|
|
2911
4115
|
operation.type === `update`
|
|
@@ -2950,6 +4154,7 @@ class PersistedCollectionRuntime<
|
|
|
2950
4154
|
term: streamPosition.term,
|
|
2951
4155
|
seq: streamPosition.seq,
|
|
2952
4156
|
rowVersion: streamPosition.rowVersion,
|
|
4157
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
2953
4158
|
mutations: mutations.map((mutation) => {
|
|
2954
4159
|
if (mutation.type === `delete`) {
|
|
2955
4160
|
return {
|
|
@@ -3037,6 +4242,7 @@ class PersistedCollectionRuntime<
|
|
|
3037
4242
|
// but leave the coordinator's state.latestSeq stale, causing seq collisions
|
|
3038
4243
|
// when follower RPCs later arrive.
|
|
3039
4244
|
if (this.persistence.coordinator.requestApplyLocalMutations) {
|
|
4245
|
+
const target = this.storageTargetFor()
|
|
3040
4246
|
const envelopeMutations = mutations.map((mutation) =>
|
|
3041
4247
|
toPersistedMutationEnvelope(
|
|
3042
4248
|
mutation as unknown as PendingMutation<Record<string, unknown>>,
|
|
@@ -3045,8 +4251,9 @@ class PersistedCollectionRuntime<
|
|
|
3045
4251
|
|
|
3046
4252
|
const response =
|
|
3047
4253
|
await this.persistence.coordinator.requestApplyLocalMutations(
|
|
3048
|
-
this.
|
|
4254
|
+
this.storageCollectionId,
|
|
3049
4255
|
envelopeMutations,
|
|
4256
|
+
target.kind === `managed` ? target.claimId : undefined,
|
|
3050
4257
|
)
|
|
3051
4258
|
|
|
3052
4259
|
this.throwIfLifecycleReplaced(lifecycleGeneration)
|
|
@@ -3181,12 +4388,12 @@ class PersistedCollectionRuntime<
|
|
|
3181
4388
|
const envelope: ProtocolEnvelope<TxCommitted> = {
|
|
3182
4389
|
v: 1,
|
|
3183
4390
|
dbName: this.dbName,
|
|
3184
|
-
collectionId: this.
|
|
4391
|
+
collectionId: this.storageCollectionId,
|
|
3185
4392
|
senderId: this.persistence.coordinator.getNodeId(),
|
|
3186
4393
|
ts: Date.now(),
|
|
3187
4394
|
payload: txCommitted,
|
|
3188
4395
|
}
|
|
3189
|
-
this.persistence.coordinator.publish(this.
|
|
4396
|
+
this.persistence.coordinator.publish(this.storageCollectionId, envelope)
|
|
3190
4397
|
}
|
|
3191
4398
|
|
|
3192
4399
|
private observeStreamPosition(
|
|
@@ -3256,7 +4463,7 @@ class PersistedCollectionRuntime<
|
|
|
3256
4463
|
// A follower routes demand to the elected owner even when its own source
|
|
3257
4464
|
// cannot own acquisitions. Only an elected node needs a local owner.
|
|
3258
4465
|
return (
|
|
3259
|
-
!this.persistence.coordinator.isLeader(this.
|
|
4466
|
+
!this.persistence.coordinator.isLeader(this.storageCollectionId) ||
|
|
3260
4467
|
this.remoteSubsetOwnerUnsubscribe !== null
|
|
3261
4468
|
)
|
|
3262
4469
|
}
|
|
@@ -3326,8 +4533,8 @@ class PersistedCollectionRuntime<
|
|
|
3326
4533
|
}
|
|
3327
4534
|
try {
|
|
3328
4535
|
await this.persistence.coordinator.requestEnsureRemoteSubset(
|
|
3329
|
-
this.
|
|
3330
|
-
options,
|
|
4536
|
+
this.storageCollectionId,
|
|
4537
|
+
this.rotatedAcquisitions.get(options) ?? options,
|
|
3331
4538
|
)
|
|
3332
4539
|
this.pendingRemoteSubsetEnsures.delete(subsetKey)
|
|
3333
4540
|
} catch (error) {
|
|
@@ -3351,11 +4558,12 @@ class PersistedCollectionRuntime<
|
|
|
3351
4558
|
}
|
|
3352
4559
|
|
|
3353
4560
|
this.persistence.coordinator.setAdapterForCollection?.(
|
|
3354
|
-
this.
|
|
4561
|
+
this.storageCollectionId,
|
|
3355
4562
|
this.persistence.adapter,
|
|
4563
|
+
this.cacheGenerationClaim?.claimId,
|
|
3356
4564
|
)
|
|
3357
4565
|
this.coordinatorUnsubscribe = this.persistence.coordinator.subscribe(
|
|
3358
|
-
this.
|
|
4566
|
+
this.storageCollectionId,
|
|
3359
4567
|
(message) => {
|
|
3360
4568
|
this.onCoordinatorMessage(message)
|
|
3361
4569
|
},
|
|
@@ -3363,34 +4571,43 @@ class PersistedCollectionRuntime<
|
|
|
3363
4571
|
}
|
|
3364
4572
|
|
|
3365
4573
|
private onCoordinatorMessage(message: ProtocolEnvelope<unknown>): void {
|
|
3366
|
-
if (message.collectionId !== this.
|
|
4574
|
+
if (message.collectionId !== this.storageCollectionId) {
|
|
3367
4575
|
return
|
|
3368
4576
|
}
|
|
3369
4577
|
if (this.getCurrentTerminalFailure()) {
|
|
3370
4578
|
return
|
|
3371
4579
|
}
|
|
3372
|
-
|
|
3373
4580
|
const { payload } = message
|
|
3374
4581
|
const lifecycleGeneration = this.lifecycleGeneration
|
|
3375
4582
|
const isSelf = message.senderId === this.persistence.coordinator.getNodeId()
|
|
4583
|
+
const claimExpired = this.hasExpiredCacheClaim()
|
|
4584
|
+
if (claimExpired && !isTxCommittedPayload(payload)) {
|
|
4585
|
+
this.scheduleCacheClaimRecovery(lifecycleGeneration)
|
|
4586
|
+
return
|
|
4587
|
+
}
|
|
3376
4588
|
|
|
3377
4589
|
// Allow tx:committed from self — the coordinator produces these on behalf
|
|
3378
4590
|
// of both local and remote mutations. The seq dedup in
|
|
3379
4591
|
// processCommittedTxUnsafe prevents double-processing of our own writes.
|
|
3380
4592
|
if (isTxCommittedPayload(payload)) {
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
4593
|
+
// Commits apply in arrival order through the queue. While the claim is
|
|
4594
|
+
// expired, keep them until renewal or rotation decides this generation;
|
|
4595
|
+
// a renewed generation still owes its rows. A hydration flushes the
|
|
4596
|
+
// queue when it ends.
|
|
4597
|
+
this.queuedTxCommitted.push({
|
|
4598
|
+
txCommitted: payload,
|
|
4599
|
+
lifecycleGeneration,
|
|
4600
|
+
storageCollectionId: message.collectionId,
|
|
4601
|
+
})
|
|
4602
|
+
if (claimExpired) {
|
|
4603
|
+
this.scheduleCacheClaimRecovery(lifecycleGeneration)
|
|
4604
|
+
} else if (!this.isHydratingNow()) {
|
|
4605
|
+
void this.applyMutex
|
|
4606
|
+
.run(() => this.flushQueuedTxCommittedUnsafe())
|
|
4607
|
+
.catch((error) => {
|
|
4608
|
+
this.markTerminalFailure(error, lifecycleGeneration)
|
|
4609
|
+
})
|
|
3387
4610
|
}
|
|
3388
|
-
|
|
3389
|
-
void this.applyMutex
|
|
3390
|
-
.run(() => this.processCommittedTxUnsafe(payload, lifecycleGeneration))
|
|
3391
|
-
.catch((error) => {
|
|
3392
|
-
this.markTerminalFailure(error, lifecycleGeneration)
|
|
3393
|
-
})
|
|
3394
4611
|
return
|
|
3395
4612
|
}
|
|
3396
4613
|
|
|
@@ -3402,9 +4619,20 @@ class PersistedCollectionRuntime<
|
|
|
3402
4619
|
if (isCollectionResetPayload(payload)) {
|
|
3403
4620
|
void this.applyMutex
|
|
3404
4621
|
.run(async () => {
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
4622
|
+
if (message.collectionId !== this.storageCollectionId) return
|
|
4623
|
+
await this.runInHydrationScope(async (adapter) => {
|
|
4624
|
+
await this.truncateAndReloadUnsafe(adapter, lifecycleGeneration)
|
|
4625
|
+
await this.rebaseDurableStreamFromSnapshotUnsafe(
|
|
4626
|
+
adapter,
|
|
4627
|
+
lifecycleGeneration,
|
|
4628
|
+
)
|
|
4629
|
+
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
4630
|
+
// Reloaded rows and the later position-only read are separate
|
|
4631
|
+
// cuts. Zero is the conservative public version in the new epoch.
|
|
4632
|
+
this.publicRowVersion = 0
|
|
4633
|
+
this.resetTermFence = this.latestTerm
|
|
4634
|
+
}
|
|
4635
|
+
})
|
|
3408
4636
|
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
3409
4637
|
await this.flushQueuedTxCommittedUnsafe()
|
|
3410
4638
|
}
|
|
@@ -3417,14 +4645,26 @@ class PersistedCollectionRuntime<
|
|
|
3417
4645
|
|
|
3418
4646
|
private async flushQueuedTxCommittedUnsafe(): Promise<void> {
|
|
3419
4647
|
while (this.queuedTxCommitted.length > 0) {
|
|
4648
|
+
// Claim recovery flushes the queue once the generation is decided.
|
|
4649
|
+
if (this.hasExpiredCacheClaim()) {
|
|
4650
|
+
this.scheduleCacheClaimRecovery(this.lifecycleGeneration)
|
|
4651
|
+
return
|
|
4652
|
+
}
|
|
3420
4653
|
const queued = this.queuedTxCommitted.shift()
|
|
3421
4654
|
if (!queued) {
|
|
3422
4655
|
continue
|
|
3423
4656
|
}
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
4657
|
+
if (queued.storageCollectionId !== this.storageCollectionId) continue
|
|
4658
|
+
// A queued commit has left the queue, so a failure must be terminal,
|
|
4659
|
+
// as it is for a commit applied when it arrives.
|
|
4660
|
+
try {
|
|
4661
|
+
await this.processCommittedTxUnsafe(
|
|
4662
|
+
queued.txCommitted,
|
|
4663
|
+
queued.lifecycleGeneration,
|
|
4664
|
+
)
|
|
4665
|
+
} catch (error) {
|
|
4666
|
+
throw this.markTerminalFailure(error, queued.lifecycleGeneration)
|
|
4667
|
+
}
|
|
3428
4668
|
}
|
|
3429
4669
|
}
|
|
3430
4670
|
|
|
@@ -3433,34 +4673,58 @@ class PersistedCollectionRuntime<
|
|
|
3433
4673
|
lifecycleGeneration = this.lifecycleGeneration,
|
|
3434
4674
|
): Promise<void> {
|
|
3435
4675
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3436
|
-
if (
|
|
3437
|
-
|
|
4676
|
+
if (this.resetTermFence !== undefined) {
|
|
4677
|
+
if (txCommitted.term <= this.resetTermFence) {
|
|
4678
|
+
// This payload could have been committed before the schema reset.
|
|
4679
|
+
// The current durable snapshot is the only safe row authority.
|
|
4680
|
+
await this.runInHydrationScope((adapter) =>
|
|
4681
|
+
this.reloadActiveSubsetsUnsafe(adapter),
|
|
4682
|
+
)
|
|
4683
|
+
return
|
|
4684
|
+
}
|
|
3438
4685
|
}
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
4686
|
+
// An exact-ID reconciliation can discover a commit whose original
|
|
4687
|
+
// notification was lost. A peer may already be at this stream position
|
|
4688
|
+
// without having loaded that commit's rows.
|
|
4689
|
+
if (txCommitted.requiresFullReload && txCommitted.reconciliationReload) {
|
|
4690
|
+
await this.invalidateFromCommittedTxUnsafe(
|
|
4691
|
+
txCommitted,
|
|
4692
|
+
this.persistence.adapter,
|
|
4693
|
+
)
|
|
4694
|
+
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
4695
|
+
this.observeStreamPosition(
|
|
4696
|
+
txCommitted.term,
|
|
4697
|
+
txCommitted.seq,
|
|
4698
|
+
txCommitted.latestRowVersion,
|
|
4699
|
+
)
|
|
4700
|
+
this.publicRowVersion = Math.max(
|
|
4701
|
+
this.publicRowVersion,
|
|
4702
|
+
txCommitted.latestRowVersion,
|
|
4703
|
+
)
|
|
4704
|
+
await this.flushQueuedTxCommittedUnsafe()
|
|
4705
|
+
}
|
|
3444
4706
|
return
|
|
3445
4707
|
}
|
|
3446
|
-
|
|
4708
|
+
if (txCommitted.latestRowVersion <= this.publicRowVersion) return
|
|
4709
|
+
const observedButUnpublished =
|
|
4710
|
+
txCommitted.term < this.latestTerm ||
|
|
4711
|
+
(txCommitted.term === this.latestTerm &&
|
|
4712
|
+
txCommitted.seq <= this.latestSeq)
|
|
3447
4713
|
const hasGapInCurrentTerm =
|
|
3448
4714
|
txCommitted.term === this.latestTerm &&
|
|
3449
4715
|
txCommitted.seq > this.latestSeq + 1
|
|
3450
4716
|
const hasGapAcrossTerms =
|
|
3451
4717
|
txCommitted.term > this.latestTerm && txCommitted.seq > 1
|
|
3452
|
-
const hasGap =
|
|
4718
|
+
const hasGap =
|
|
4719
|
+
observedButUnpublished ||
|
|
4720
|
+
txCommitted.latestRowVersion > this.publicRowVersion + 1 ||
|
|
4721
|
+
hasGapInCurrentTerm ||
|
|
4722
|
+
hasGapAcrossTerms
|
|
3453
4723
|
|
|
3454
4724
|
if (hasGap) {
|
|
3455
|
-
await this.recoverFromSeqGapUnsafe(lifecycleGeneration)
|
|
4725
|
+
if (!(await this.recoverFromSeqGapUnsafe(lifecycleGeneration))) return
|
|
3456
4726
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3457
|
-
if (
|
|
3458
|
-
txCommitted.term < this.latestTerm ||
|
|
3459
|
-
(txCommitted.term === this.latestTerm &&
|
|
3460
|
-
txCommitted.seq <= this.latestSeq)
|
|
3461
|
-
) {
|
|
3462
|
-
return
|
|
3463
|
-
}
|
|
4727
|
+
if (txCommitted.latestRowVersion <= this.publicRowVersion) return
|
|
3464
4728
|
}
|
|
3465
4729
|
|
|
3466
4730
|
this.observeStreamPosition(
|
|
@@ -3474,27 +4738,66 @@ class PersistedCollectionRuntime<
|
|
|
3474
4738
|
this.persistence.adapter,
|
|
3475
4739
|
)
|
|
3476
4740
|
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
4741
|
+
this.publicRowVersion = Math.max(
|
|
4742
|
+
this.publicRowVersion,
|
|
4743
|
+
txCommitted.latestRowVersion,
|
|
4744
|
+
)
|
|
3477
4745
|
await this.flushQueuedTxCommittedUnsafe()
|
|
3478
4746
|
}
|
|
3479
4747
|
}
|
|
3480
4748
|
|
|
3481
|
-
private async
|
|
4749
|
+
private async rebaseDurableStreamFromSnapshotUnsafe(
|
|
4750
|
+
adapter: HydrationPersistenceAdapter,
|
|
3482
4751
|
lifecycleGeneration: number,
|
|
3483
4752
|
): Promise<void> {
|
|
3484
4753
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
4754
|
+
const snapshot = await adapter.loadResumeSnapshot(this.storageTargetFor(), {
|
|
4755
|
+
includeRows: false,
|
|
4756
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
4757
|
+
})
|
|
4758
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
4759
|
+
// The reset epoch can restart row versions at zero. Keep the term/seq
|
|
4760
|
+
// watermark to fence old notifications, but rebase version counters.
|
|
4761
|
+
this.latestRowVersion = snapshot.latestRowVersion
|
|
4762
|
+
this.localRowVersion = snapshot.latestRowVersion
|
|
4763
|
+
this.observeStreamPosition(
|
|
4764
|
+
snapshot.latestTerm,
|
|
4765
|
+
snapshot.latestSeq,
|
|
4766
|
+
snapshot.latestRowVersion,
|
|
4767
|
+
)
|
|
4768
|
+
}
|
|
4769
|
+
|
|
4770
|
+
private async recoverFromSeqGapUnsafe(
|
|
4771
|
+
lifecycleGeneration: number,
|
|
4772
|
+
): Promise<boolean> {
|
|
4773
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return false
|
|
3485
4774
|
if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) {
|
|
3486
4775
|
let pullResponse: PullSinceResponse | undefined
|
|
3487
4776
|
try {
|
|
3488
4777
|
pullResponse = await this.persistence.coordinator.pullSince(
|
|
3489
|
-
this.
|
|
3490
|
-
this.
|
|
4778
|
+
this.storageCollectionId,
|
|
4779
|
+
this.publicRowVersion,
|
|
4780
|
+
undefined,
|
|
4781
|
+
this.cacheGenerationClaim?.claimId,
|
|
3491
4782
|
)
|
|
3492
4783
|
} catch (error) {
|
|
3493
4784
|
console.warn(`Failed pullSince recovery attempt:`, error)
|
|
3494
4785
|
}
|
|
3495
4786
|
|
|
4787
|
+
// The replay read may have started under a live claim and returned
|
|
4788
|
+
// after expiry. Recovery needs the apply mutex held by this callback,
|
|
4789
|
+
// so schedule it after discarding the old response.
|
|
4790
|
+
if (this.scopedRecoveryPending) return false
|
|
4791
|
+
if (
|
|
4792
|
+
this.cacheGenerationClaim &&
|
|
4793
|
+
!(await this.renewCacheClaim(this.persistence.adapter))
|
|
4794
|
+
) {
|
|
4795
|
+
this.scheduleCacheClaimRecovery(lifecycleGeneration)
|
|
4796
|
+
return false
|
|
4797
|
+
}
|
|
4798
|
+
|
|
3496
4799
|
if (pullResponse) {
|
|
3497
|
-
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
4800
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return false
|
|
3498
4801
|
|
|
3499
4802
|
if (pullResponse.ok) {
|
|
3500
4803
|
this.observeStreamPosition(
|
|
@@ -3506,14 +4809,22 @@ class PersistedCollectionRuntime<
|
|
|
3506
4809
|
await this.runInHydrationScope((adapter) =>
|
|
3507
4810
|
this.reloadActiveSubsetsUnsafe(adapter),
|
|
3508
4811
|
)
|
|
3509
|
-
|
|
4812
|
+
this.publicRowVersion = Math.max(
|
|
4813
|
+
this.publicRowVersion,
|
|
4814
|
+
pullResponse.latestRowVersion,
|
|
4815
|
+
)
|
|
4816
|
+
return true
|
|
3510
4817
|
}
|
|
3511
4818
|
const deltas = pullResponse.deltas
|
|
3512
4819
|
if (!deltas) {
|
|
3513
4820
|
await this.runInHydrationScope((adapter) =>
|
|
3514
4821
|
this.reloadActiveSubsetsUnsafe(adapter),
|
|
3515
4822
|
)
|
|
3516
|
-
|
|
4823
|
+
this.publicRowVersion = Math.max(
|
|
4824
|
+
this.publicRowVersion,
|
|
4825
|
+
pullResponse.latestRowVersion,
|
|
4826
|
+
)
|
|
4827
|
+
return true
|
|
3517
4828
|
}
|
|
3518
4829
|
|
|
3519
4830
|
await this.runInHydrationScope(async (adapter) => {
|
|
@@ -3538,14 +4849,24 @@ class PersistedCollectionRuntime<
|
|
|
3538
4849
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3539
4850
|
}
|
|
3540
4851
|
})
|
|
3541
|
-
|
|
4852
|
+
this.publicRowVersion = Math.max(
|
|
4853
|
+
this.publicRowVersion,
|
|
4854
|
+
pullResponse.latestRowVersion,
|
|
4855
|
+
)
|
|
4856
|
+
return true
|
|
3542
4857
|
}
|
|
3543
4858
|
}
|
|
3544
4859
|
}
|
|
3545
4860
|
|
|
3546
|
-
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
4861
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return false
|
|
4862
|
+
// A gap is not a schema reset. Replace the durable snapshot in one
|
|
4863
|
+
// publication so readers never see an empty intermediate Collection.
|
|
3547
4864
|
await this.runInHydrationScope((adapter) =>
|
|
3548
|
-
this.
|
|
4865
|
+
this.reloadActiveSubsetsUnsafe(adapter),
|
|
4866
|
+
)
|
|
4867
|
+
this.publicRowVersion = Math.max(
|
|
4868
|
+
this.publicRowVersion,
|
|
4869
|
+
this.latestRowVersion,
|
|
3549
4870
|
)
|
|
3550
4871
|
|
|
3551
4872
|
if (this.mode === `sync-present`) {
|
|
@@ -3553,6 +4874,53 @@ class PersistedCollectionRuntime<
|
|
|
3553
4874
|
this.queueRemoteSubsetEnsure(options)
|
|
3554
4875
|
}
|
|
3555
4876
|
}
|
|
4877
|
+
return true
|
|
4878
|
+
}
|
|
4879
|
+
|
|
4880
|
+
private refreshScopedSubsets(): void {
|
|
4881
|
+
if (!this.remoteSubsetOwner) return
|
|
4882
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
4883
|
+
const storageId = this.storageCollectionId
|
|
4884
|
+
for (const [subsetKey, options] of this.activeSubsets) {
|
|
4885
|
+
if (options.signal?.aborted) continue
|
|
4886
|
+
this.hydratedDemands.delete(getLoadSubsetDemandKey(options))
|
|
4887
|
+
// A scoped cache cannot answer the invalidation. This one-shot
|
|
4888
|
+
// acquisition requests source evidence while the original demand keeps
|
|
4889
|
+
// its lease; releasing it after settlement preserves exact ownership.
|
|
4890
|
+
const refreshOptions: LoadSubsetOptions = { ...options, refetch: true }
|
|
4891
|
+
this.scopedRefreshes.set(refreshOptions, storageId)
|
|
4892
|
+
void Promise.resolve()
|
|
4893
|
+
.then(() => {
|
|
4894
|
+
if (
|
|
4895
|
+
lifecycleGeneration !== this.lifecycleGeneration ||
|
|
4896
|
+
this.activeSubsets.get(subsetKey) !== options
|
|
4897
|
+
) {
|
|
4898
|
+
return
|
|
4899
|
+
}
|
|
4900
|
+
return this.persistence.coordinator.requestEnsureRemoteSubset(
|
|
4901
|
+
storageId,
|
|
4902
|
+
refreshOptions,
|
|
4903
|
+
)
|
|
4904
|
+
})
|
|
4905
|
+
.catch((error: unknown) => {
|
|
4906
|
+
if (
|
|
4907
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
4908
|
+
!options.signal?.aborted
|
|
4909
|
+
) {
|
|
4910
|
+
this.reportSyncError(error)
|
|
4911
|
+
}
|
|
4912
|
+
})
|
|
4913
|
+
.finally(() => {
|
|
4914
|
+
if (!this.scopedRefreshes.delete(refreshOptions)) return
|
|
4915
|
+
void this.persistence.coordinator
|
|
4916
|
+
.requestReleaseRemoteSubset(storageId, refreshOptions)
|
|
4917
|
+
.catch((error: unknown) => {
|
|
4918
|
+
if (lifecycleGeneration === this.lifecycleGeneration) {
|
|
4919
|
+
this.reportSyncError(error)
|
|
4920
|
+
}
|
|
4921
|
+
})
|
|
4922
|
+
})
|
|
4923
|
+
}
|
|
3556
4924
|
}
|
|
3557
4925
|
|
|
3558
4926
|
private async truncateAndReloadUnsafe(
|
|
@@ -3560,10 +4928,21 @@ class PersistedCollectionRuntime<
|
|
|
3560
4928
|
lifecycleGeneration = this.lifecycleGeneration,
|
|
3561
4929
|
): Promise<void> {
|
|
3562
4930
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
4931
|
+
if (this.scopedRecovery) {
|
|
4932
|
+
this.refreshScopedSubsets()
|
|
4933
|
+
return
|
|
4934
|
+
}
|
|
3563
4935
|
// A subscriber may reacquire reentrantly from the truncate commit.
|
|
3564
4936
|
this.hydratedDemands.clear()
|
|
3565
4937
|
this.resetSequence++
|
|
3566
|
-
|
|
4938
|
+
// On-demand reset keeps previously public rows and hydration seeds until
|
|
4939
|
+
// the new durable snapshot can replace them in one publication. Active
|
|
4940
|
+
// demand may add rows; retired demand may only validate visible keys.
|
|
4941
|
+
if (
|
|
4942
|
+
this.syncMode === `eager` &&
|
|
4943
|
+
this.syncControls.begin &&
|
|
4944
|
+
this.syncControls.commit
|
|
4945
|
+
) {
|
|
3567
4946
|
const applied = this.withInternalApply(() => {
|
|
3568
4947
|
this.syncControls.begin?.()
|
|
3569
4948
|
this.syncControls.truncate?.()
|
|
@@ -3573,21 +4952,33 @@ class PersistedCollectionRuntime<
|
|
|
3573
4952
|
}
|
|
3574
4953
|
|
|
3575
4954
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3576
|
-
|
|
4955
|
+
// A reset replays demand: the owner's reset rows return only when the
|
|
4956
|
+
// follower requests its demand again.
|
|
4957
|
+
await this.reloadActiveSubsetsUnsafe(
|
|
4958
|
+
adapter,
|
|
4959
|
+
false,
|
|
4960
|
+
this.syncMode === `eager`,
|
|
4961
|
+
true,
|
|
4962
|
+
)
|
|
3577
4963
|
}
|
|
3578
4964
|
|
|
3579
4965
|
private async invalidateFromCommittedTxUnsafe(
|
|
3580
4966
|
txCommitted: TxCommitted,
|
|
3581
4967
|
adapter: HydrationPersistenceAdapter,
|
|
3582
4968
|
): Promise<void> {
|
|
3583
|
-
|
|
4969
|
+
if (this.scopedRecovery) {
|
|
4970
|
+
this.refreshScopedSubsets()
|
|
4971
|
+
return
|
|
4972
|
+
}
|
|
4973
|
+
const reloadActiveSubsets = (skipUnchangedReplace = false) =>
|
|
3584
4974
|
this.runInHydrationScope(
|
|
3585
|
-
(scopedAdapter) =>
|
|
4975
|
+
(scopedAdapter) =>
|
|
4976
|
+
this.reloadActiveSubsetsUnsafe(scopedAdapter, skipUnchangedReplace),
|
|
3586
4977
|
adapter,
|
|
3587
4978
|
)
|
|
3588
4979
|
|
|
3589
4980
|
if (txCommitted.requiresFullReload) {
|
|
3590
|
-
await reloadActiveSubsets()
|
|
4981
|
+
await reloadActiveSubsets(txCommitted.reconciliationReload === true)
|
|
3591
4982
|
return
|
|
3592
4983
|
}
|
|
3593
4984
|
|
|
@@ -3602,6 +4993,38 @@ class PersistedCollectionRuntime<
|
|
|
3602
4993
|
(opt) => opt.limit != null || opt.offset != null || opt.cursor != null,
|
|
3603
4994
|
)
|
|
3604
4995
|
|
|
4996
|
+
// A targeted envelope separates changed rows from deleted keys and omits
|
|
4997
|
+
// insert/update identity. A same-key insert replaces the durable row,
|
|
4998
|
+
// while an update merges omitted fields. The public row cannot decide
|
|
4999
|
+
// which operation produced a changed value. A previously unseen key also
|
|
5000
|
+
// has no public baseline with which to complete a source patch. Read the
|
|
5001
|
+
// durable result whenever that ambiguity affects a public row or demand.
|
|
5002
|
+
const changedKeys = new Set(txCommitted.changedRows.map((row) => row.key))
|
|
5003
|
+
const hasRepeatedChange =
|
|
5004
|
+
changedKeys.size !== txCommitted.changedRows.length
|
|
5005
|
+
const hasChangedAndDeletedKey = txCommitted.deletedKeys.some((key) =>
|
|
5006
|
+
changedKeys.has(key),
|
|
5007
|
+
)
|
|
5008
|
+
const hasChangedPublicKey = txCommitted.changedRows.some((row) =>
|
|
5009
|
+
this.collection?.base.has(row.key as TKey),
|
|
5010
|
+
)
|
|
5011
|
+
const hasUnseenDemandedKey =
|
|
5012
|
+
this.activeSubsets.size > 0 &&
|
|
5013
|
+
txCommitted.changedRows.some(
|
|
5014
|
+
(row) =>
|
|
5015
|
+
!this.collection?.base.has(row.key as TKey) &&
|
|
5016
|
+
!this.collection?._hasHydratedKey(row.key as TKey),
|
|
5017
|
+
)
|
|
5018
|
+
if (
|
|
5019
|
+
hasRepeatedChange ||
|
|
5020
|
+
hasChangedAndDeletedKey ||
|
|
5021
|
+
hasChangedPublicKey ||
|
|
5022
|
+
(this.syncMode === `on-demand` && hasUnseenDemandedKey)
|
|
5023
|
+
) {
|
|
5024
|
+
await reloadActiveSubsets()
|
|
5025
|
+
return
|
|
5026
|
+
}
|
|
5027
|
+
|
|
3605
5028
|
if (!hasPaginatedSubset || changedKeyCount === 0) {
|
|
3606
5029
|
await this.applyTargetedInvalidationUnsafe(txCommitted)
|
|
3607
5030
|
return
|
|
@@ -3626,23 +5049,40 @@ class PersistedCollectionRuntime<
|
|
|
3626
5049
|
key: changedKey,
|
|
3627
5050
|
value: newValue,
|
|
3628
5051
|
} of txCommitted.changedRows) {
|
|
3629
|
-
const
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
}
|
|
3637
|
-
|
|
5052
|
+
const key = changedKey as TKey
|
|
5053
|
+
if (this.collection?._hasHydratedKey(key)) continue
|
|
5054
|
+
const previous = this.collection?.base.get(key)
|
|
5055
|
+
if (previous === undefined && this.activeSubsets.size === 0) continue
|
|
5056
|
+
const completeValue = {
|
|
5057
|
+
...(previous as object | undefined),
|
|
5058
|
+
...newValue,
|
|
5059
|
+
} as T
|
|
5060
|
+
const currentlyDemanded = matchesAnySubset(
|
|
5061
|
+
completeValue,
|
|
5062
|
+
subsetEvaluators,
|
|
5063
|
+
)
|
|
5064
|
+
const previouslyOutsideDemand =
|
|
5065
|
+
previous !== undefined &&
|
|
5066
|
+
!matchesAnySubset(previous, subsetEvaluators)
|
|
5067
|
+
|
|
5068
|
+
if (currentlyDemanded || previouslyOutsideDemand) {
|
|
5069
|
+
this.syncControls.write?.({ type: `update`, value: completeValue })
|
|
5070
|
+
} else if (previous !== undefined) {
|
|
5071
|
+
this.syncControls.write?.({ type: `delete`, key })
|
|
3638
5072
|
}
|
|
3639
5073
|
}
|
|
3640
5074
|
|
|
3641
5075
|
for (const deletedKey of txCommitted.deletedKeys) {
|
|
3642
|
-
|
|
5076
|
+
const key = deletedKey as TKey
|
|
5077
|
+
if (!this.collection?._hasHydratedKey(key)) {
|
|
5078
|
+
this.syncControls.write?.({ type: `delete`, key })
|
|
5079
|
+
}
|
|
3643
5080
|
}
|
|
3644
5081
|
|
|
3645
5082
|
txCommitted.rowMetadataMutations?.forEach((mutation) => {
|
|
5083
|
+
// A DbClient hydration seed owns both its row and row metadata until
|
|
5084
|
+
// an accepted source transaction confirms the key.
|
|
5085
|
+
if (this.collection?._hasHydratedKey(mutation.key as TKey)) return
|
|
3646
5086
|
if (mutation.type === `delete`) {
|
|
3647
5087
|
this.syncControls.metadata?.row.delete(mutation.key as TKey)
|
|
3648
5088
|
} else {
|
|
@@ -3671,13 +5111,40 @@ class PersistedCollectionRuntime<
|
|
|
3671
5111
|
|
|
3672
5112
|
private async reloadActiveSubsetsUnsafe(
|
|
3673
5113
|
adapter: HydrationPersistenceAdapter,
|
|
5114
|
+
skipUnchangedReplace = false,
|
|
5115
|
+
restoreResetBaseline = false,
|
|
5116
|
+
replayDemand = false,
|
|
3674
5117
|
): Promise<void> {
|
|
5118
|
+
if (this.scopedRecovery) {
|
|
5119
|
+
this.refreshScopedSubsets()
|
|
5120
|
+
return
|
|
5121
|
+
}
|
|
3675
5122
|
const lifecycleGeneration = this.lifecycleGeneration
|
|
3676
5123
|
const truncateGeneration = this.sourceTruncateGeneration
|
|
3677
|
-
const activeSubsetOptions =
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
5124
|
+
const activeSubsetOptions = Array.from(this.activeSubsets.values())
|
|
5125
|
+
if (restoreResetBaseline && activeSubsetOptions.length === 0) {
|
|
5126
|
+
activeSubsetOptions.push({})
|
|
5127
|
+
}
|
|
5128
|
+
const evaluators = activeSubsetOptions.map((options) =>
|
|
5129
|
+
options.where ? compileSingleRowExpression(options.where) : null,
|
|
5130
|
+
)
|
|
5131
|
+
// A filtered read has no authority over public rows that were already
|
|
5132
|
+
// outside every active predicate. Validate those keys against the complete
|
|
5133
|
+
// durable snapshot; only active demand may admit previously unseen keys.
|
|
5134
|
+
const outsideDemandKeys = activeSubsetOptions.some(
|
|
5135
|
+
(options) =>
|
|
5136
|
+
options.limit != null ||
|
|
5137
|
+
options.offset != null ||
|
|
5138
|
+
options.cursor != null,
|
|
5139
|
+
)
|
|
5140
|
+
? []
|
|
5141
|
+
: Array.from(this.collection?.base ?? [])
|
|
5142
|
+
.filter(
|
|
5143
|
+
([key, row]) =>
|
|
5144
|
+
!this.collection?._hasHydratedKey(key) &&
|
|
5145
|
+
!matchesAnySubset(row, evaluators),
|
|
5146
|
+
)
|
|
5147
|
+
.map(([key]) => key)
|
|
3681
5148
|
|
|
3682
5149
|
this.hydratedDemands.clear()
|
|
3683
5150
|
this.hydrationSequence++
|
|
@@ -3686,33 +5153,124 @@ class PersistedCollectionRuntime<
|
|
|
3686
5153
|
this.hydratingGeneration = lifecycleGeneration
|
|
3687
5154
|
try {
|
|
3688
5155
|
const mergedRows = new Map<TKey, { value: T; metadata?: unknown }>()
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
5156
|
+
let validationSnapshot:
|
|
5157
|
+
Awaited<ReturnType<typeof adapter.loadResumeSnapshot>> | undefined
|
|
5158
|
+
let collectionMetadata: Array<{ key: string; value: unknown }> = []
|
|
5159
|
+
try {
|
|
5160
|
+
validationSnapshot =
|
|
5161
|
+
(activeSubsetOptions.length === 0 || outsideDemandKeys.length > 0) &&
|
|
5162
|
+
this.collection?.base.size
|
|
5163
|
+
? await adapter.loadResumeSnapshot(this.storageTargetFor(), {
|
|
5164
|
+
requiredIndexSignatures: this.getRequiredIndexSignatures(),
|
|
5165
|
+
includeRows: true,
|
|
5166
|
+
cacheGenerationClaimId: this.cacheGenerationClaim?.claimId,
|
|
5167
|
+
})
|
|
5168
|
+
: undefined
|
|
5169
|
+
collectionMetadata = validationSnapshot
|
|
5170
|
+
? validationSnapshot.collectionMetadata
|
|
5171
|
+
: await this.loadCollectionMetadataSnapshot(adapter)
|
|
3694
5172
|
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
3695
|
-
for (const
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
5173
|
+
for (const options of activeSubsetOptions) {
|
|
5174
|
+
const subsetRows = await this.loadSubsetRowsUnsafe(options, adapter)
|
|
5175
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
5176
|
+
for (const row of subsetRows) {
|
|
5177
|
+
mergedRows.set(row.key, {
|
|
5178
|
+
value: row.value,
|
|
5179
|
+
metadata: row.metadata,
|
|
5180
|
+
})
|
|
5181
|
+
}
|
|
5182
|
+
}
|
|
5183
|
+
if (outsideDemandKeys.length > 0 && validationSnapshot) {
|
|
5184
|
+
const durableRows = new Map(
|
|
5185
|
+
validationSnapshot.rows.map((row) => [row.key, row]),
|
|
5186
|
+
)
|
|
5187
|
+
for (const key of outsideDemandKeys) {
|
|
5188
|
+
if (mergedRows.has(key)) continue
|
|
5189
|
+
const row = durableRows.get(key)
|
|
5190
|
+
if (row) {
|
|
5191
|
+
mergedRows.set(key, {
|
|
5192
|
+
value: row.value as T,
|
|
5193
|
+
metadata: row.metadata,
|
|
5194
|
+
})
|
|
5195
|
+
}
|
|
5196
|
+
}
|
|
5197
|
+
}
|
|
5198
|
+
} catch (error) {
|
|
5199
|
+
// Recovery may have started while the adapter read was awaited.
|
|
5200
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
5201
|
+
if (this.scopedRecovery) return
|
|
5202
|
+
// SQLite may reject its second read-admission check after a claim
|
|
5203
|
+
// expires. A read from that cache has no authority to fail the
|
|
5204
|
+
// Collection; recovery will reload its active demands.
|
|
5205
|
+
if (!this.cacheGenerationClaim || !this.hasExpiredCacheClaim()) {
|
|
5206
|
+
throw error
|
|
3700
5207
|
}
|
|
5208
|
+
this.scheduleCacheClaimRecovery(lifecycleGeneration)
|
|
5209
|
+
return
|
|
5210
|
+
}
|
|
5211
|
+
|
|
5212
|
+
// A peer reload can hold its persisted read past claim expiry. Its
|
|
5213
|
+
// rows and metadata cannot publish under the old generation. Recovery
|
|
5214
|
+
// needs this apply mutex, so start it only after this callback returns.
|
|
5215
|
+
if (this.isScopedRecoveryPending()) return
|
|
5216
|
+
if (this.cacheGenerationClaim && !(await this.renewCacheClaim(adapter))) {
|
|
5217
|
+
this.scheduleCacheClaimRecovery(lifecycleGeneration)
|
|
5218
|
+
return
|
|
3701
5219
|
}
|
|
5220
|
+
if (
|
|
5221
|
+
this.isScopedRecoveryPending() ||
|
|
5222
|
+
lifecycleGeneration !== this.lifecycleGeneration
|
|
5223
|
+
)
|
|
5224
|
+
return
|
|
3702
5225
|
|
|
3703
5226
|
for (const key of mergedRows.keys()) {
|
|
3704
5227
|
hydrationContext.suppliedRowKeys.add(key)
|
|
3705
5228
|
}
|
|
3706
5229
|
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
5230
|
+
if (activeSubsetOptions.length === 0) {
|
|
5231
|
+
// Retired demand does not hide already public rows. Validate only
|
|
5232
|
+
// those keys; do not publish unrelated durable rows without demand.
|
|
5233
|
+
await whenSyncAccepted(
|
|
5234
|
+
this.applyRowsToCollection(
|
|
5235
|
+
[],
|
|
5236
|
+
(validationSnapshot?.rows ?? []) as Array<{
|
|
5237
|
+
key: TKey
|
|
5238
|
+
value: T
|
|
5239
|
+
metadata?: unknown
|
|
5240
|
+
}>,
|
|
5241
|
+
collectionMetadata,
|
|
5242
|
+
),
|
|
5243
|
+
)
|
|
5244
|
+
} else if (
|
|
5245
|
+
skipUnchangedReplace &&
|
|
5246
|
+
this.matchesCollectionSnapshot(mergedRows, collectionMetadata)
|
|
5247
|
+
) {
|
|
5248
|
+
// A full replacement would also clear metadata for missing rows.
|
|
5249
|
+
// Clear only those entries so equal public rows do not republish.
|
|
5250
|
+
const orphanKeys =
|
|
5251
|
+
this.collection?._syncedRowMetadataKeysOutsideBase() ?? []
|
|
5252
|
+
if (orphanKeys.length > 0) {
|
|
5253
|
+
const applied = this.withInternalApply(() => {
|
|
5254
|
+
this.syncControls.begin?.()
|
|
5255
|
+
for (const key of orphanKeys) {
|
|
5256
|
+
this.syncControls.metadata?.row.delete(key)
|
|
5257
|
+
}
|
|
5258
|
+
return this.syncControls.commit?.() ?? true
|
|
5259
|
+
})
|
|
5260
|
+
await whenSyncAccepted(applied)
|
|
5261
|
+
}
|
|
5262
|
+
} else {
|
|
5263
|
+
const applied = this.replaceCollectionSnapshot(
|
|
5264
|
+
Array.from(mergedRows.entries()).map(([key, row]) => ({
|
|
5265
|
+
key,
|
|
5266
|
+
value: row.value,
|
|
5267
|
+
metadata: row.metadata,
|
|
5268
|
+
})),
|
|
5269
|
+
collectionMetadata,
|
|
5270
|
+
replayDemand,
|
|
5271
|
+
)
|
|
5272
|
+
await whenSyncAccepted(applied)
|
|
5273
|
+
}
|
|
3716
5274
|
} finally {
|
|
3717
5275
|
if (
|
|
3718
5276
|
this.activeHydrationContext === hydrationContext &&
|
|
@@ -3752,7 +5310,12 @@ class PersistedCollectionRuntime<
|
|
|
3752
5310
|
}
|
|
3753
5311
|
|
|
3754
5312
|
this.indexAddedUnsubscribe = this.collection.on(`index:added`, (event) => {
|
|
3755
|
-
void this.ensurePersistedIndex(event.index)
|
|
5313
|
+
void this.ensurePersistedIndex(event.index).catch((error) => {
|
|
5314
|
+
console.warn(
|
|
5315
|
+
`Failed to ensure persisted index after index:added:`,
|
|
5316
|
+
error,
|
|
5317
|
+
)
|
|
5318
|
+
})
|
|
3756
5319
|
})
|
|
3757
5320
|
this.indexRemovedUnsubscribe = this.collection.on(
|
|
3758
5321
|
`index:removed`,
|
|
@@ -3774,8 +5337,22 @@ class PersistedCollectionRuntime<
|
|
|
3774
5337
|
const indexMetadata =
|
|
3775
5338
|
indexMetadataSnapshot ?? collection?.getIndexMetadata() ?? []
|
|
3776
5339
|
const completedLocalIndexSignatures = new Set<string>()
|
|
5340
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
3777
5341
|
for (const metadata of indexMetadata) {
|
|
3778
|
-
|
|
5342
|
+
const storageCollectionId = this.storageCollectionId
|
|
5343
|
+
const claimId = this.cacheGenerationClaim?.claimId
|
|
5344
|
+
const ensured = await this.ensureLocalPersistedIndex(metadata, adapter)
|
|
5345
|
+
if (
|
|
5346
|
+
ensured &&
|
|
5347
|
+
(await this.reconcilePersistedIndex(
|
|
5348
|
+
metadata,
|
|
5349
|
+
adapter,
|
|
5350
|
+
storageCollectionId,
|
|
5351
|
+
claimId,
|
|
5352
|
+
lifecycleGeneration,
|
|
5353
|
+
true,
|
|
5354
|
+
))
|
|
5355
|
+
) {
|
|
3779
5356
|
completedLocalIndexSignatures.add(metadata.signature)
|
|
3780
5357
|
}
|
|
3781
5358
|
}
|
|
@@ -3786,12 +5363,61 @@ class PersistedCollectionRuntime<
|
|
|
3786
5363
|
indexMetadata: Array<CollectionIndexMetadata>,
|
|
3787
5364
|
completedLocalIndexSignatures: ReadonlySet<string>,
|
|
3788
5365
|
): Promise<void> {
|
|
5366
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
3789
5367
|
for (const metadata of indexMetadata) {
|
|
5368
|
+
if (!this.hasCurrentIndex(metadata)) continue
|
|
5369
|
+
const storageCollectionId = this.storageCollectionId
|
|
5370
|
+
const claimId = this.cacheGenerationClaim?.claimId
|
|
3790
5371
|
await this.requestCoordinatorPersistedIndex(
|
|
3791
5372
|
metadata,
|
|
3792
5373
|
completedLocalIndexSignatures.has(metadata.signature),
|
|
3793
5374
|
)
|
|
5375
|
+
await this.reconcilePersistedIndex(
|
|
5376
|
+
metadata,
|
|
5377
|
+
this.persistence.adapter,
|
|
5378
|
+
storageCollectionId,
|
|
5379
|
+
claimId,
|
|
5380
|
+
lifecycleGeneration,
|
|
5381
|
+
true,
|
|
5382
|
+
)
|
|
5383
|
+
}
|
|
5384
|
+
}
|
|
5385
|
+
|
|
5386
|
+
/** Settle a late index operation against the Collection's current declaration. */
|
|
5387
|
+
private async reconcilePersistedIndex(
|
|
5388
|
+
indexMetadata: CollectionIndexMetadata,
|
|
5389
|
+
adapter: HydrationPersistenceAdapter,
|
|
5390
|
+
storageCollectionId: string,
|
|
5391
|
+
claimId: string | undefined,
|
|
5392
|
+
lifecycleGeneration: number,
|
|
5393
|
+
indexPresent: boolean,
|
|
5394
|
+
): Promise<boolean> {
|
|
5395
|
+
while (
|
|
5396
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
5397
|
+
storageCollectionId === this.storageCollectionId
|
|
5398
|
+
) {
|
|
5399
|
+
const declared = this.hasCurrentIndex(indexMetadata)
|
|
5400
|
+
if (declared === indexPresent) return declared
|
|
5401
|
+
if (declared) {
|
|
5402
|
+
if (!(await this.ensureLocalPersistedIndex(indexMetadata, adapter))) {
|
|
5403
|
+
return false
|
|
5404
|
+
}
|
|
5405
|
+
} else {
|
|
5406
|
+
if (!adapter.markIndexRemoved) return false
|
|
5407
|
+
try {
|
|
5408
|
+
await adapter.markIndexRemoved(
|
|
5409
|
+
this.storageTargetFor(storageCollectionId, claimId),
|
|
5410
|
+
indexMetadata.signature,
|
|
5411
|
+
{ cacheGenerationClaimId: claimId },
|
|
5412
|
+
)
|
|
5413
|
+
} catch (error) {
|
|
5414
|
+
console.warn(`Failed to mark persisted index removed:`, error)
|
|
5415
|
+
return false
|
|
5416
|
+
}
|
|
5417
|
+
}
|
|
5418
|
+
indexPresent = declared
|
|
3794
5419
|
}
|
|
5420
|
+
return false
|
|
3795
5421
|
}
|
|
3796
5422
|
|
|
3797
5423
|
private buildPersistedIndexSpec(
|
|
@@ -3811,11 +5437,39 @@ class PersistedCollectionRuntime<
|
|
|
3811
5437
|
indexMetadata: CollectionIndexMetadata,
|
|
3812
5438
|
adapter: HydrationPersistenceAdapter = this.persistence.adapter,
|
|
3813
5439
|
): Promise<void> {
|
|
5440
|
+
if (this.scopedRecoveryPending) return
|
|
5441
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
5442
|
+
await this.ensureCacheClaimActive()
|
|
5443
|
+
if (
|
|
5444
|
+
lifecycleGeneration !== this.lifecycleGeneration ||
|
|
5445
|
+
!this.hasCurrentIndex(indexMetadata)
|
|
5446
|
+
)
|
|
5447
|
+
return
|
|
3814
5448
|
const completedLocally = await this.ensureLocalPersistedIndex(
|
|
3815
5449
|
indexMetadata,
|
|
3816
5450
|
adapter,
|
|
3817
5451
|
)
|
|
5452
|
+
if (lifecycleGeneration !== this.lifecycleGeneration) return
|
|
5453
|
+
if (!this.hasCurrentIndex(indexMetadata)) {
|
|
5454
|
+
await this.markIndexRemoved(indexMetadata)
|
|
5455
|
+
return
|
|
5456
|
+
}
|
|
3818
5457
|
await this.requestCoordinatorPersistedIndex(indexMetadata, completedLocally)
|
|
5458
|
+
if (
|
|
5459
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
5460
|
+
!this.hasCurrentIndex(indexMetadata)
|
|
5461
|
+
) {
|
|
5462
|
+
await this.markIndexRemoved(indexMetadata)
|
|
5463
|
+
}
|
|
5464
|
+
}
|
|
5465
|
+
|
|
5466
|
+
private hasCurrentIndex(indexMetadata: CollectionIndexMetadata): boolean {
|
|
5467
|
+
return (
|
|
5468
|
+
this.collection
|
|
5469
|
+
?.getIndexMetadata()
|
|
5470
|
+
.some((current) => current.signature === indexMetadata.signature) ??
|
|
5471
|
+
false
|
|
5472
|
+
)
|
|
3819
5473
|
}
|
|
3820
5474
|
|
|
3821
5475
|
private async ensureLocalPersistedIndex(
|
|
@@ -3825,9 +5479,10 @@ class PersistedCollectionRuntime<
|
|
|
3825
5479
|
try {
|
|
3826
5480
|
const spec = this.buildPersistedIndexSpec(indexMetadata)
|
|
3827
5481
|
await adapter.ensureIndex(
|
|
3828
|
-
this.
|
|
5482
|
+
this.storageTargetFor(),
|
|
3829
5483
|
indexMetadata.signature,
|
|
3830
5484
|
spec,
|
|
5485
|
+
{ cacheGenerationClaimId: this.cacheGenerationClaim?.claimId },
|
|
3831
5486
|
)
|
|
3832
5487
|
return true
|
|
3833
5488
|
} catch (error) {
|
|
@@ -3843,11 +5498,12 @@ class PersistedCollectionRuntime<
|
|
|
3843
5498
|
try {
|
|
3844
5499
|
const spec = this.buildPersistedIndexSpec(indexMetadata)
|
|
3845
5500
|
await this.persistence.coordinator.requestEnsurePersistedIndex(
|
|
3846
|
-
this.
|
|
5501
|
+
this.storageCollectionId,
|
|
3847
5502
|
indexMetadata.signature,
|
|
3848
5503
|
spec,
|
|
3849
5504
|
completedLocally ? this.persistence.adapter : undefined,
|
|
3850
5505
|
completedLocally,
|
|
5506
|
+
this.cacheGenerationClaim?.claimId,
|
|
3851
5507
|
)
|
|
3852
5508
|
} catch (error) {
|
|
3853
5509
|
console.warn(
|
|
@@ -3865,10 +5521,25 @@ class PersistedCollectionRuntime<
|
|
|
3865
5521
|
}
|
|
3866
5522
|
|
|
3867
5523
|
try {
|
|
5524
|
+
if (this.scopedRecoveryPending) return
|
|
5525
|
+
const lifecycleGeneration = this.lifecycleGeneration
|
|
5526
|
+
await this.ensureCacheClaimActive()
|
|
5527
|
+
if (
|
|
5528
|
+
lifecycleGeneration !== this.lifecycleGeneration ||
|
|
5529
|
+
this.hasCurrentIndex(indexMetadata)
|
|
5530
|
+
)
|
|
5531
|
+
return
|
|
3868
5532
|
await this.persistence.adapter.markIndexRemoved(
|
|
3869
|
-
this.
|
|
5533
|
+
this.storageTargetFor(),
|
|
3870
5534
|
indexMetadata.signature,
|
|
5535
|
+
{ cacheGenerationClaimId: this.cacheGenerationClaim?.claimId },
|
|
3871
5536
|
)
|
|
5537
|
+
if (
|
|
5538
|
+
lifecycleGeneration === this.lifecycleGeneration &&
|
|
5539
|
+
this.hasCurrentIndex(indexMetadata)
|
|
5540
|
+
) {
|
|
5541
|
+
await this.ensurePersistedIndex(indexMetadata)
|
|
5542
|
+
}
|
|
3872
5543
|
} catch (error) {
|
|
3873
5544
|
console.warn(`Failed to mark persisted index removed:`, error)
|
|
3874
5545
|
}
|
|
@@ -4023,6 +5694,7 @@ function createWrappedSyncConfig<
|
|
|
4023
5694
|
const isCleanedUp = () => startupState.cleanedUp
|
|
4024
5695
|
type SubsetAcquisition = {
|
|
4025
5696
|
forwarded: boolean
|
|
5697
|
+
source?: SyncConfigRes
|
|
4026
5698
|
cancelAdmissionWait?: () => void
|
|
4027
5699
|
}
|
|
4028
5700
|
const acquisitions = new Map<LoadSubsetOptions, SubsetAcquisition>()
|
|
@@ -4140,21 +5812,27 @@ function createWrappedSyncConfig<
|
|
|
4140
5812
|
throw error
|
|
4141
5813
|
}
|
|
4142
5814
|
}
|
|
4143
|
-
runtime.setSyncControls({
|
|
4144
|
-
begin: params.begin,
|
|
4145
|
-
write: params.write as SyncControlFns<T, TKey>[`write`],
|
|
4146
|
-
commit: params.commit,
|
|
4147
|
-
truncate: params.truncate,
|
|
4148
|
-
metadata: params.metadata ?? null,
|
|
4149
|
-
markError: params.markError,
|
|
4150
|
-
})
|
|
4151
5815
|
runtime.setCollection(
|
|
4152
5816
|
params.collection as Collection<T, TKey, PersistedCollectionUtils>,
|
|
5817
|
+
{
|
|
5818
|
+
begin: params.begin,
|
|
5819
|
+
write: params.write as SyncControlFns<T, TKey>[`write`],
|
|
5820
|
+
commit: params.commit,
|
|
5821
|
+
truncate: params.truncate,
|
|
5822
|
+
metadata: params.metadata ?? null,
|
|
5823
|
+
markError: params.markError,
|
|
5824
|
+
},
|
|
4153
5825
|
)
|
|
4154
5826
|
|
|
4155
5827
|
const persistenceCapability: SyncPersistenceCapabilityV1<TKey> = {
|
|
4156
5828
|
protocol: SYNC_PERSISTENCE_PROTOCOL,
|
|
4157
5829
|
version: SYNC_PERSISTENCE_VERSION,
|
|
5830
|
+
get managedCacheGeneration() {
|
|
5831
|
+
return runtime.hasManagedCacheGeneration()
|
|
5832
|
+
},
|
|
5833
|
+
get startupCacheGenerationRotated() {
|
|
5834
|
+
return runtime.hasRotatedStartupCacheGeneration()
|
|
5835
|
+
},
|
|
4158
5836
|
reserveCommitTurn: () => {
|
|
4159
5837
|
const openTransaction = getOpenTransaction()
|
|
4160
5838
|
if (openTransaction && !openTransaction.internal)
|
|
@@ -4168,6 +5846,10 @@ function createWrappedSyncConfig<
|
|
|
4168
5846
|
throw runtime.reportSyncError(error)
|
|
4169
5847
|
}
|
|
4170
5848
|
},
|
|
5849
|
+
startScopedRecovery: (resetMetadata) =>
|
|
5850
|
+
startupState.cleanedUp
|
|
5851
|
+
? Promise.resolve()
|
|
5852
|
+
: runtime.startScopedRecovery(resetMetadata),
|
|
4171
5853
|
scanPersistedRows: (options) =>
|
|
4172
5854
|
startupState.cleanedUp
|
|
4173
5855
|
? Promise.resolve([])
|
|
@@ -4197,6 +5879,7 @@ function createWrappedSyncConfig<
|
|
|
4197
5879
|
},
|
|
4198
5880
|
}
|
|
4199
5881
|
|
|
5882
|
+
let sourceSession = 0
|
|
4200
5883
|
const wrappedParams = {
|
|
4201
5884
|
...params,
|
|
4202
5885
|
markError: reportUpstreamError,
|
|
@@ -4526,6 +6209,12 @@ function createWrappedSyncConfig<
|
|
|
4526
6209
|
settlePublicationAdmissionWaiters(openTransaction, error)
|
|
4527
6210
|
return createHandledRejection(error)
|
|
4528
6211
|
}
|
|
6212
|
+
if (runtime.isScopedRecoveryPending()) {
|
|
6213
|
+
settlePendingTransaction(openTransaction)
|
|
6214
|
+
const error = new SyncTransactionAbortedError()
|
|
6215
|
+
settlePublicationAdmissionWaiters(openTransaction, error)
|
|
6216
|
+
return createHandledRejection(error)
|
|
6217
|
+
}
|
|
4529
6218
|
try {
|
|
4530
6219
|
bindToCurrentHydration(openTransaction)
|
|
4531
6220
|
assertHydrationSequenceCurrent(openTransaction)
|
|
@@ -4618,9 +6307,183 @@ function createWrappedSyncConfig<
|
|
|
4618
6307
|
},
|
|
4619
6308
|
}
|
|
4620
6309
|
|
|
6310
|
+
// Each invocation of a custom source owns its callbacks. Retiring that
|
|
6311
|
+
// invocation before rotation makes late work harmless even when the
|
|
6312
|
+
// source does not provide its own restart hook.
|
|
6313
|
+
const sourceParamsFor = (session: number) => {
|
|
6314
|
+
const active = () =>
|
|
6315
|
+
!startupState.cleanedUp && session === sourceSession
|
|
6316
|
+
const metadata = wrappedParams.metadata
|
|
6317
|
+
return {
|
|
6318
|
+
...wrappedParams,
|
|
6319
|
+
markReady: () => {
|
|
6320
|
+
if (active()) wrappedParams.markReady()
|
|
6321
|
+
},
|
|
6322
|
+
markError: (error?: unknown) => {
|
|
6323
|
+
if (active()) wrappedParams.markError(error)
|
|
6324
|
+
},
|
|
6325
|
+
begin: () => {
|
|
6326
|
+
if (active()) wrappedParams.begin()
|
|
6327
|
+
},
|
|
6328
|
+
write: (message: ChangeMessageOrDeleteKeyMessage<T, TKey>) => {
|
|
6329
|
+
if (active()) wrappedParams.write(message)
|
|
6330
|
+
},
|
|
6331
|
+
truncate: () => {
|
|
6332
|
+
if (active()) wrappedParams.truncate()
|
|
6333
|
+
},
|
|
6334
|
+
commit: (signal?: AbortSignal) =>
|
|
6335
|
+
active() ? wrappedParams.commit(signal) : true,
|
|
6336
|
+
metadata: metadata && {
|
|
6337
|
+
...metadata,
|
|
6338
|
+
row: {
|
|
6339
|
+
get: (key: TKey) =>
|
|
6340
|
+
active() ? metadata.row.get(key) : undefined,
|
|
6341
|
+
set: (key: TKey, value: unknown) => {
|
|
6342
|
+
if (active()) metadata.row.set(key, value)
|
|
6343
|
+
},
|
|
6344
|
+
delete: (key: TKey) => {
|
|
6345
|
+
if (active()) metadata.row.delete(key)
|
|
6346
|
+
},
|
|
6347
|
+
},
|
|
6348
|
+
collection: {
|
|
6349
|
+
get: (key: string) =>
|
|
6350
|
+
active() ? metadata.collection.get(key) : undefined,
|
|
6351
|
+
set: (key: string, value: unknown) => {
|
|
6352
|
+
if (active()) metadata.collection.set(key, value)
|
|
6353
|
+
},
|
|
6354
|
+
delete: (key: string) => {
|
|
6355
|
+
if (active()) metadata.collection.delete(key)
|
|
6356
|
+
},
|
|
6357
|
+
list: (prefix?: string) =>
|
|
6358
|
+
active() ? metadata.collection.list(prefix) : [],
|
|
6359
|
+
},
|
|
6360
|
+
persistence: {
|
|
6361
|
+
...metadata.persistence,
|
|
6362
|
+
reserveCommitTurn: () => {
|
|
6363
|
+
if (active()) metadata.persistence.reserveCommitTurn()
|
|
6364
|
+
},
|
|
6365
|
+
hydrateBaseline: () =>
|
|
6366
|
+
active()
|
|
6367
|
+
? metadata.persistence.hydrateBaseline()
|
|
6368
|
+
: Promise.resolve(),
|
|
6369
|
+
startScopedRecovery: (resetMetadata?: {
|
|
6370
|
+
key: string
|
|
6371
|
+
value: unknown
|
|
6372
|
+
}) =>
|
|
6373
|
+
active()
|
|
6374
|
+
? (metadata.persistence.startScopedRecovery?.(
|
|
6375
|
+
resetMetadata,
|
|
6376
|
+
) ?? Promise.resolve())
|
|
6377
|
+
: Promise.resolve(),
|
|
6378
|
+
scanPersistedRows: (
|
|
6379
|
+
options?: Parameters<
|
|
6380
|
+
typeof metadata.persistence.scanPersistedRows
|
|
6381
|
+
>[0],
|
|
6382
|
+
) =>
|
|
6383
|
+
active()
|
|
6384
|
+
? metadata.persistence.scanPersistedRows(options)
|
|
6385
|
+
: Promise.resolve([]),
|
|
6386
|
+
resumeSnapshot: {
|
|
6387
|
+
certify: () =>
|
|
6388
|
+
active()
|
|
6389
|
+
? metadata.persistence.resumeSnapshot.certify()
|
|
6390
|
+
: Promise.resolve(),
|
|
6391
|
+
getKeySetEvidence: () =>
|
|
6392
|
+
active()
|
|
6393
|
+
? metadata.persistence.resumeSnapshot.getKeySetEvidence()
|
|
6394
|
+
: undefined,
|
|
6395
|
+
expectCurrentCommit: () => {
|
|
6396
|
+
if (active())
|
|
6397
|
+
metadata.persistence.resumeSnapshot.expectCurrentCommit()
|
|
6398
|
+
},
|
|
6399
|
+
},
|
|
6400
|
+
},
|
|
6401
|
+
},
|
|
6402
|
+
}
|
|
6403
|
+
}
|
|
6404
|
+
|
|
4621
6405
|
let sourceResult: SyncConfigRes = {}
|
|
4622
6406
|
let sourceResultSettled = false
|
|
4623
6407
|
let activeSourceSyncEntry: Promise<void> | undefined
|
|
6408
|
+
let sourceCleanupPromise: Promise<void> | undefined
|
|
6409
|
+
let customRestartSequence = 0
|
|
6410
|
+
const enterSourceSession = (): void => {
|
|
6411
|
+
const session = ++sourceSession
|
|
6412
|
+
let settleSourceSyncEntry!: () => void
|
|
6413
|
+
activeSourceSyncEntry = new Promise<void>((resolve) => {
|
|
6414
|
+
settleSourceSyncEntry = resolve
|
|
6415
|
+
})
|
|
6416
|
+
try {
|
|
6417
|
+
sourceResult = normalizeSyncFnResult(
|
|
6418
|
+
sourceSyncConfig.sync(
|
|
6419
|
+
runtime.hasManagedCacheGeneration()
|
|
6420
|
+
? sourceParamsFor(session)
|
|
6421
|
+
: wrappedParams,
|
|
6422
|
+
),
|
|
6423
|
+
)
|
|
6424
|
+
// A coordinator acquisition owns the source session that accepted
|
|
6425
|
+
// it. A late release must not unload the replacement session.
|
|
6426
|
+
if (!startupState.cleanedUp && sourceResult.loadSubset) {
|
|
6427
|
+
const ownerSource = sourceResult
|
|
6428
|
+
const loadSubset = async (
|
|
6429
|
+
options: TransportedLoadSubsetOptions,
|
|
6430
|
+
) => {
|
|
6431
|
+
if (startupState.cleanedUp || session !== sourceSession) {
|
|
6432
|
+
throw new Error(`persisted sync source is no longer active`)
|
|
6433
|
+
}
|
|
6434
|
+
await ownerSource.loadSubset?.(
|
|
6435
|
+
options as unknown as LoadSubsetOptions,
|
|
6436
|
+
)
|
|
6437
|
+
}
|
|
6438
|
+
runtime.registerRemoteSubsetOwner(
|
|
6439
|
+
Object.assign(loadSubset, {
|
|
6440
|
+
unloadSubset: (options: TransportedLoadSubsetOptions) =>
|
|
6441
|
+
ownerSource.unloadSubset?.(
|
|
6442
|
+
options as unknown as LoadSubsetOptions,
|
|
6443
|
+
),
|
|
6444
|
+
onError: (error: unknown) => {
|
|
6445
|
+
if (session === sourceSession) reportUpstreamError(error)
|
|
6446
|
+
},
|
|
6447
|
+
}),
|
|
6448
|
+
)
|
|
6449
|
+
}
|
|
6450
|
+
} finally {
|
|
6451
|
+
activeSourceSyncEntry = undefined
|
|
6452
|
+
settleSourceSyncEntry()
|
|
6453
|
+
}
|
|
6454
|
+
}
|
|
6455
|
+
const restartCustomSource = (cacheRotated: Promise<void>) => {
|
|
6456
|
+
const restartSequence = ++customRestartSequence
|
|
6457
|
+
// Synchronous retirement is required by the recovery boundary. Any
|
|
6458
|
+
// open old-source transaction is abandoned before the new cache binds.
|
|
6459
|
+
if (!sourceCleanupPromise) {
|
|
6460
|
+
sourceSession++
|
|
6461
|
+
const abortError = new SyncTransactionAbortedError()
|
|
6462
|
+
for (const transaction of transactionStack) {
|
|
6463
|
+
settlePublicationAdmissionWaiters(transaction, abortError)
|
|
6464
|
+
}
|
|
6465
|
+
transactionStack.length = 0
|
|
6466
|
+
const retiredResult = sourceResult
|
|
6467
|
+
sourceCleanupPromise = Promise.resolve().then(() =>
|
|
6468
|
+
retiredResult.cleanup?.(),
|
|
6469
|
+
)
|
|
6470
|
+
}
|
|
6471
|
+
return Promise.all([sourceCleanupPromise, cacheRotated]).then(() => {
|
|
6472
|
+
if (
|
|
6473
|
+
startupState.cleanedUp ||
|
|
6474
|
+
restartSequence !== customRestartSequence
|
|
6475
|
+
)
|
|
6476
|
+
return
|
|
6477
|
+
sourceCleanupPromise = undefined
|
|
6478
|
+
enterSourceSession()
|
|
6479
|
+
runtime.setSourceRestartAfterScopedRecovery(
|
|
6480
|
+
sourceResult.restartAfterScopedRecovery ??
|
|
6481
|
+
(runtime.hasManagedCacheGeneration()
|
|
6482
|
+
? restartCustomSource
|
|
6483
|
+
: undefined),
|
|
6484
|
+
)
|
|
6485
|
+
})
|
|
6486
|
+
}
|
|
4624
6487
|
fullStartPromise = runtime.ensureStarted()
|
|
4625
6488
|
// Startup can fail before the source reaches markReady or loadSubset,
|
|
4626
6489
|
// which are the two eventual consumers of this outer adopting promise.
|
|
@@ -4636,40 +6499,14 @@ function createWrappedSyncConfig<
|
|
|
4636
6499
|
return sourceResult
|
|
4637
6500
|
}
|
|
4638
6501
|
|
|
4639
|
-
|
|
4640
|
-
activeSourceSyncEntry = new Promise<void>((resolve) => {
|
|
4641
|
-
settleSourceSyncEntry = resolve
|
|
4642
|
-
})
|
|
4643
|
-
try {
|
|
4644
|
-
sourceResult = normalizeSyncFnResult(
|
|
4645
|
-
sourceSyncConfig.sync(wrappedParams),
|
|
4646
|
-
)
|
|
4647
|
-
} finally {
|
|
4648
|
-
activeSourceSyncEntry = undefined
|
|
4649
|
-
settleSourceSyncEntry()
|
|
4650
|
-
}
|
|
4651
|
-
// Cleanup can reenter while the source sync function is still active.
|
|
4652
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
4653
|
-
if (!startupState.cleanedUp && sourceResult.loadSubset) {
|
|
4654
|
-
const loadSubset = async (options: TransportedLoadSubsetOptions) => {
|
|
4655
|
-
if (startupState.cleanedUp) {
|
|
4656
|
-
throw new Error(`persisted sync source is no longer active`)
|
|
4657
|
-
}
|
|
4658
|
-
await sourceResult.loadSubset?.(
|
|
4659
|
-
options as unknown as LoadSubsetOptions,
|
|
4660
|
-
)
|
|
4661
|
-
}
|
|
4662
|
-
runtime.registerRemoteSubsetOwner(
|
|
4663
|
-
Object.assign(loadSubset, {
|
|
4664
|
-
unloadSubset: (options: TransportedLoadSubsetOptions) =>
|
|
4665
|
-
sourceResult.unloadSubset?.(
|
|
4666
|
-
options as unknown as LoadSubsetOptions,
|
|
4667
|
-
),
|
|
4668
|
-
onError: reportUpstreamError,
|
|
4669
|
-
}),
|
|
4670
|
-
)
|
|
4671
|
-
}
|
|
6502
|
+
enterSourceSession()
|
|
4672
6503
|
sourceResultSettled = true
|
|
6504
|
+
runtime.setSourceRestartAfterScopedRecovery(
|
|
6505
|
+
sourceResult.restartAfterScopedRecovery ??
|
|
6506
|
+
(runtime.hasManagedCacheGeneration()
|
|
6507
|
+
? restartCustomSource
|
|
6508
|
+
: undefined),
|
|
6509
|
+
)
|
|
4673
6510
|
return sourceResult
|
|
4674
6511
|
})()
|
|
4675
6512
|
resolveSourceResultAssigned()
|
|
@@ -4680,6 +6517,7 @@ function createWrappedSyncConfig<
|
|
|
4680
6517
|
return {
|
|
4681
6518
|
cleanup: async () => {
|
|
4682
6519
|
startupState.cleanedUp = true
|
|
6520
|
+
runtime.clearSourceRestartAfterScopedRecovery()
|
|
4683
6521
|
// The optional source starts behind persistence metadata. If it
|
|
4684
6522
|
// reenters cleanup before returning, wait for that exact synchronous
|
|
4685
6523
|
// entry to publish its cleanup callback before sampling the result.
|
|
@@ -4696,9 +6534,11 @@ function createWrappedSyncConfig<
|
|
|
4696
6534
|
transactionStack.length = 0
|
|
4697
6535
|
let sourceCleanup: Promise<void>
|
|
4698
6536
|
try {
|
|
4699
|
-
sourceCleanup =
|
|
4700
|
-
|
|
4701
|
-
|
|
6537
|
+
sourceCleanup =
|
|
6538
|
+
sourceCleanupPromise ??
|
|
6539
|
+
(sourceSyncEntry
|
|
6540
|
+
? sourceSyncEntry.then(() => sourceResult.cleanup?.())
|
|
6541
|
+
: Promise.resolve(sourceResult.cleanup?.()))
|
|
4702
6542
|
} catch (error) {
|
|
4703
6543
|
sourceCleanup = Promise.reject(error)
|
|
4704
6544
|
}
|
|
@@ -4726,10 +6566,9 @@ function createWrappedSyncConfig<
|
|
|
4726
6566
|
if (runtimeOutcome.status === `rejected`) throw runtimeOutcome.reason
|
|
4727
6567
|
},
|
|
4728
6568
|
loadSubset: (options: LoadSubsetOptions): true | Promise<void> => {
|
|
4729
|
-
const acquisition = { forwarded: false }
|
|
6569
|
+
const acquisition: SubsetAcquisition = { forwarded: false }
|
|
4730
6570
|
acquisitions.set(options, acquisition)
|
|
4731
6571
|
const forwardUpstream = (
|
|
4732
|
-
resolvedSourceResult: SyncConfigRes,
|
|
4733
6572
|
loadOptions: LoadSubsetOptions,
|
|
4734
6573
|
): true | Promise<void> => {
|
|
4735
6574
|
// Hydration is another async boundary. A release before this
|
|
@@ -4740,14 +6579,17 @@ function createWrappedSyncConfig<
|
|
|
4740
6579
|
) {
|
|
4741
6580
|
return true
|
|
4742
6581
|
}
|
|
4743
|
-
|
|
6582
|
+
const currentSource = sourceResult
|
|
6583
|
+
if (!currentSource.loadSubset) return true
|
|
4744
6584
|
acquisition.forwarded = true
|
|
6585
|
+
acquisition.source = currentSource
|
|
4745
6586
|
try {
|
|
4746
6587
|
// Returning a promise transfers its lease even if it rejects.
|
|
4747
6588
|
// Only a synchronous throw leaves no upstream lease to release.
|
|
4748
|
-
return
|
|
6589
|
+
return currentSource.loadSubset(loadOptions)
|
|
4749
6590
|
} catch (error) {
|
|
4750
6591
|
acquisition.forwarded = false
|
|
6592
|
+
acquisition.source = undefined
|
|
4751
6593
|
throw error
|
|
4752
6594
|
}
|
|
4753
6595
|
}
|
|
@@ -4755,19 +6597,60 @@ function createWrappedSyncConfig<
|
|
|
4755
6597
|
const openTransaction = getOpenTransaction()
|
|
4756
6598
|
const needsPublicationAdmission =
|
|
4757
6599
|
openTransaction?.reservesCommitTurn === true
|
|
6600
|
+
let hydrationStarted = false
|
|
6601
|
+
const raceRetainedLoad = (
|
|
6602
|
+
work: Promise<void>,
|
|
6603
|
+
canAbortPromptly = () => true,
|
|
6604
|
+
): Promise<void> => {
|
|
6605
|
+
const signal = options.signal
|
|
6606
|
+
const abortable = !signal
|
|
6607
|
+
? work
|
|
6608
|
+
: new Promise<void>((resolve, reject) => {
|
|
6609
|
+
const abortError = () =>
|
|
6610
|
+
signal.reason ?? new SyncTransactionAbortedError()
|
|
6611
|
+
const onAbort = () => {
|
|
6612
|
+
// Once an uncancelable baseline read begins, its returned
|
|
6613
|
+
// load keeps core's overlapping replay private until the
|
|
6614
|
+
// resulting rows and receipts become visible.
|
|
6615
|
+
if (canAbortPromptly()) reject(abortError())
|
|
6616
|
+
}
|
|
6617
|
+
signal.addEventListener(`abort`, onAbort, { once: true })
|
|
6618
|
+
if (signal.aborted) onAbort()
|
|
6619
|
+
void work.then(
|
|
6620
|
+
() => {
|
|
6621
|
+
signal.removeEventListener(`abort`, onAbort)
|
|
6622
|
+
if (signal.aborted) reject(abortError())
|
|
6623
|
+
else resolve()
|
|
6624
|
+
},
|
|
6625
|
+
(error: unknown) => {
|
|
6626
|
+
signal.removeEventListener(`abort`, onAbort)
|
|
6627
|
+
reject(signal.aborted ? abortError() : error)
|
|
6628
|
+
},
|
|
6629
|
+
)
|
|
6630
|
+
})
|
|
6631
|
+
return abortable.catch((error: unknown) => {
|
|
6632
|
+
if (
|
|
6633
|
+
startupState.cleanedUp ||
|
|
6634
|
+
acquisitions.get(options) !== acquisition
|
|
6635
|
+
) {
|
|
6636
|
+
return
|
|
6637
|
+
}
|
|
6638
|
+
throw error
|
|
6639
|
+
})
|
|
6640
|
+
}
|
|
4758
6641
|
const hydrated =
|
|
4759
6642
|
sourceResultSettled && !needsPublicationAdmission
|
|
4760
6643
|
? runtime.loadHydratedSubset(options, (loadOptions) =>
|
|
4761
|
-
forwardUpstream(
|
|
6644
|
+
forwardUpstream(loadOptions),
|
|
4762
6645
|
)
|
|
4763
6646
|
: undefined
|
|
4764
6647
|
if (hydrated !== undefined) {
|
|
4765
|
-
return hydrated
|
|
6648
|
+
return hydrated === true ? true : raceRetainedLoad(hydrated)
|
|
4766
6649
|
}
|
|
4767
6650
|
|
|
4768
|
-
|
|
6651
|
+
const work = (async () => {
|
|
4769
6652
|
await fullStartPromise
|
|
4770
|
-
|
|
6653
|
+
await sourceResultPromise
|
|
4771
6654
|
if (
|
|
4772
6655
|
startupState.cleanedUp ||
|
|
4773
6656
|
acquisitions.get(options) !== acquisition
|
|
@@ -4790,10 +6673,18 @@ function createWrappedSyncConfig<
|
|
|
4790
6673
|
) {
|
|
4791
6674
|
return
|
|
4792
6675
|
}
|
|
4793
|
-
return runtime.loadSubset(
|
|
4794
|
-
|
|
6676
|
+
return runtime.loadSubset(
|
|
6677
|
+
options,
|
|
6678
|
+
(loadOptions) => forwardUpstream(loadOptions),
|
|
6679
|
+
() => {
|
|
6680
|
+
hydrationStarted = true
|
|
6681
|
+
},
|
|
6682
|
+
() =>
|
|
6683
|
+
!startupState.cleanedUp &&
|
|
6684
|
+
acquisitions.get(options) === acquisition,
|
|
4795
6685
|
)
|
|
4796
6686
|
})()
|
|
6687
|
+
return raceRetainedLoad(work, () => !hydrationStarted)
|
|
4797
6688
|
},
|
|
4798
6689
|
unloadSubset: (options: LoadSubsetOptions) => {
|
|
4799
6690
|
const acquisition = acquisitions.get(options)
|
|
@@ -4801,7 +6692,9 @@ function createWrappedSyncConfig<
|
|
|
4801
6692
|
acquisitions.delete(options)
|
|
4802
6693
|
runtime.unloadSubset(
|
|
4803
6694
|
options,
|
|
4804
|
-
acquisition?.forwarded
|
|
6695
|
+
acquisition?.forwarded
|
|
6696
|
+
? acquisition.source?.unloadSubset
|
|
6697
|
+
: undefined,
|
|
4805
6698
|
)
|
|
4806
6699
|
},
|
|
4807
6700
|
}
|
|
@@ -4815,16 +6708,16 @@ function createLoopbackSyncConfig<
|
|
|
4815
6708
|
>(runtime: PersistedCollectionRuntime<T, TKey>): SyncConfig<T, TKey> {
|
|
4816
6709
|
return {
|
|
4817
6710
|
sync: (params) => {
|
|
4818
|
-
runtime.setSyncControls({
|
|
4819
|
-
begin: params.begin,
|
|
4820
|
-
write: params.write as SyncControlFns<T, TKey>[`write`],
|
|
4821
|
-
commit: params.commit,
|
|
4822
|
-
truncate: params.truncate,
|
|
4823
|
-
metadata: params.metadata ?? null,
|
|
4824
|
-
markError: params.markError,
|
|
4825
|
-
})
|
|
4826
6711
|
runtime.setCollection(
|
|
4827
6712
|
params.collection as Collection<T, TKey, PersistedCollectionUtils>,
|
|
6713
|
+
{
|
|
6714
|
+
begin: params.begin,
|
|
6715
|
+
write: params.write as SyncControlFns<T, TKey>[`write`],
|
|
6716
|
+
commit: params.commit,
|
|
6717
|
+
truncate: params.truncate,
|
|
6718
|
+
metadata: params.metadata ?? null,
|
|
6719
|
+
markError: params.markError,
|
|
6720
|
+
},
|
|
4828
6721
|
)
|
|
4829
6722
|
|
|
4830
6723
|
void runtime
|
|
@@ -4944,6 +6837,20 @@ export function persistedCollectionOptions<
|
|
|
4944
6837
|
? new PersistedReadinessTracker(networkTimeoutMs)
|
|
4945
6838
|
: undefined
|
|
4946
6839
|
|
|
6840
|
+
let claimed = false
|
|
6841
|
+
let claimedRuntime: PersistedCollectionRuntime<T, TKey> | undefined
|
|
6842
|
+
const claimOptions = () => {
|
|
6843
|
+
if (claimed) {
|
|
6844
|
+
throw new InvalidPersistedCollectionConfigError(
|
|
6845
|
+
`Persisted options can create only one Collection. Create fresh options for each Collection.`,
|
|
6846
|
+
)
|
|
6847
|
+
}
|
|
6848
|
+
claimed = true
|
|
6849
|
+
return () => {
|
|
6850
|
+
if (!claimedRuntime?.hasBoundCollection()) claimed = false
|
|
6851
|
+
}
|
|
6852
|
+
}
|
|
6853
|
+
|
|
4947
6854
|
if (hasOwnSyncKey(options)) {
|
|
4948
6855
|
if (!isValidSyncConfig(options.sync)) {
|
|
4949
6856
|
throw new InvalidSyncConfigError(
|
|
@@ -4974,7 +6881,9 @@ export function persistedCollectionOptions<
|
|
|
4974
6881
|
syncOptions.syncMode ?? `eager`,
|
|
4975
6882
|
collectionId,
|
|
4976
6883
|
persistedReadinessTracker,
|
|
6884
|
+
claimOptions,
|
|
4977
6885
|
)
|
|
6886
|
+
claimedRuntime = runtime
|
|
4978
6887
|
|
|
4979
6888
|
const result = {
|
|
4980
6889
|
...syncOptions,
|
|
@@ -4997,6 +6906,7 @@ export function persistedCollectionOptions<
|
|
|
4997
6906
|
...options,
|
|
4998
6907
|
id: collectionId,
|
|
4999
6908
|
} as never) as typeof result,
|
|
6909
|
+
claimOptions,
|
|
5000
6910
|
)
|
|
5001
6911
|
}
|
|
5002
6912
|
|
|
@@ -5022,7 +6932,9 @@ export function persistedCollectionOptions<
|
|
|
5022
6932
|
localOnlyOptions.syncMode ?? `eager`,
|
|
5023
6933
|
localOnlyOptions.id ?? DEFAULT_DB_NAME,
|
|
5024
6934
|
persistedReadinessTracker,
|
|
6935
|
+
claimOptions,
|
|
5025
6936
|
)
|
|
6937
|
+
claimedRuntime = runtime
|
|
5026
6938
|
|
|
5027
6939
|
const wrappedOnInsert = async (
|
|
5028
6940
|
params: InsertMutationFnParams<T, TKey, TUtils & PersistedCollectionUtils>,
|
|
@@ -5119,6 +7031,7 @@ export function persistedCollectionOptions<
|
|
|
5119
7031
|
...options,
|
|
5120
7032
|
id: collectionId,
|
|
5121
7033
|
} as never) as typeof result,
|
|
7034
|
+
claimOptions,
|
|
5122
7035
|
)
|
|
5123
7036
|
}
|
|
5124
7037
|
|