@peerbit/shared-log 16.0.1 → 16.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/coordinate-persistence.d.ts +18 -1
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +61 -4
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/index.d.ts +17 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +301 -201
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/simple.d.ts +1 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +1 -1
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +11 -11
- package/src/coordinate-persistence.ts +66 -5
- package/src/index.ts +391 -257
- package/src/sync/simple.ts +1 -1
package/src/index.ts
CHANGED
|
@@ -290,6 +290,7 @@ import {
|
|
|
290
290
|
} from "./sync/profile.js";
|
|
291
291
|
import {
|
|
292
292
|
ConfirmEntriesMessage,
|
|
293
|
+
RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
|
|
293
294
|
SYNC_MESSAGE_PRIORITY,
|
|
294
295
|
SimpleSyncronizer,
|
|
295
296
|
} from "./sync/simple.js";
|
|
@@ -636,6 +637,13 @@ export type NativeBackboneCoordinateRollback<R extends "u32" | "u64"> = {
|
|
|
636
637
|
hashes: Set<string>;
|
|
637
638
|
entries: Map<string, ResidentCoordinateEntry<R>>;
|
|
638
639
|
generations: Map<string, number>;
|
|
640
|
+
/**
|
|
641
|
+
* Set by `settleResidentCoordinateSnapshot` once the token's holds on the
|
|
642
|
+
* mutation-generation map have been released. Load-bearing: it makes the
|
|
643
|
+
* settle idempotent, so a second settle of this token cannot consume a
|
|
644
|
+
* different token's hold on a shared hash.
|
|
645
|
+
*/
|
|
646
|
+
settled?: boolean;
|
|
639
647
|
};
|
|
640
648
|
|
|
641
649
|
export type RepairDispatchEntry<R extends "u32" | "u64"> =
|
|
@@ -1439,6 +1447,16 @@ const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
|
|
|
1439
1447
|
];
|
|
1440
1448
|
const APPEND_BACKFILL_RETRY_SCHEDULE_MS = [0, 1_000, 3_000, 7_000];
|
|
1441
1449
|
const RECENT_KNOWN_REPAIR_SUPPRESSION_MS = 30_000;
|
|
1450
|
+
// `_entryKnownPeerObservedAt` is read ONLY through isEntryRecentlyKnownByPeer,
|
|
1451
|
+
// which treats an over-age row and an absent row identically (both false). So
|
|
1452
|
+
// rows older than the longest horizon any caller asks about are dead weight,
|
|
1453
|
+
// and dropping them is behaviour-identical rather than merely safe. Derived
|
|
1454
|
+
// from the horizons themselves -- never hardcode it -- so a future caller with
|
|
1455
|
+
// a longer window cannot silently outlive the retention that serves it.
|
|
1456
|
+
const ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS = Math.max(
|
|
1457
|
+
RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
|
|
1458
|
+
RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
|
|
1459
|
+
);
|
|
1442
1460
|
const JOIN_AUTHORITATIVE_REPAIR_DELAY_MS = 2_000;
|
|
1443
1461
|
const JOIN_AUTHORITATIVE_REPAIR_SWEEP_DELAYS_MS = [
|
|
1444
1462
|
JOIN_AUTHORITATIVE_REPAIR_DELAY_MS,
|
|
@@ -3103,9 +3121,23 @@ export class SharedLog<
|
|
|
3103
3121
|
};
|
|
3104
3122
|
for (const coordinate of intent.coordinates) {
|
|
3105
3123
|
rollback.hashes.add(coordinate.hash);
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3124
|
+
// Same hold-counted row shape the coordinator's own
|
|
3125
|
+
// snapshot writes: one hold per hash, released by the
|
|
3126
|
+
// settle after the replay consumes the token below.
|
|
3127
|
+
// RELIES ON `intent.coordinates` HOLDING UNIQUE HASHES —
|
|
3128
|
+
// it is built from the token's `hashes` Set (see
|
|
3129
|
+
// setNativeStrictDurableTransactionOperation). Holds are
|
|
3130
|
+
// taken per element here but released per unique hash by
|
|
3131
|
+
// the settle, so a duplicate would take two and release
|
|
3132
|
+
// one and retain that row forever. That is the safe
|
|
3133
|
+
// direction (a retained row, never a fail-open rollback),
|
|
3134
|
+
// but keep the source a Set.
|
|
3135
|
+
const row = mutationGenerations.get(coordinate.hash);
|
|
3136
|
+
const generation = (row?.generation ?? 0) + 1;
|
|
3137
|
+
mutationGenerations.set(coordinate.hash, {
|
|
3138
|
+
generation,
|
|
3139
|
+
holds: (row?.holds ?? 0) + 1,
|
|
3140
|
+
});
|
|
3109
3141
|
rollback.generations.set(coordinate.hash, generation);
|
|
3110
3142
|
if (coordinate.value) {
|
|
3111
3143
|
const number = (value: string) =>
|
|
@@ -3131,6 +3163,9 @@ export class SharedLog<
|
|
|
3131
3163
|
"",
|
|
3132
3164
|
rollback,
|
|
3133
3165
|
);
|
|
3166
|
+
// The replay fabricated these generations itself one turn
|
|
3167
|
+
// earlier and has now consumed them, so the rows are dead.
|
|
3168
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollback);
|
|
3134
3169
|
}
|
|
3135
3170
|
for (const document of intent.documents) {
|
|
3136
3171
|
this.restoreNativeBackboneDocument({
|
|
@@ -3294,6 +3329,7 @@ export class SharedLog<
|
|
|
3294
3329
|
private _repairSweepOptimisticGidsByPeer!: Map<string, Set<string>>;
|
|
3295
3330
|
private _entryKnownPeers!: Map<string, Set<string>>;
|
|
3296
3331
|
private _entryKnownPeerObservedAt!: Map<string, Map<string, number>>;
|
|
3332
|
+
private _entryKnownPeerObservedAtSweptAt = 0;
|
|
3297
3333
|
private _joinAuthoritativeRepairTimersByDelay!: Map<
|
|
3298
3334
|
number,
|
|
3299
3335
|
ReturnType<typeof setTimeout>
|
|
@@ -3644,6 +3680,7 @@ export class SharedLog<
|
|
|
3644
3680
|
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
3645
3681
|
this._entryKnownPeers = new Map();
|
|
3646
3682
|
this._entryKnownPeerObservedAt = new Map();
|
|
3683
|
+
this._entryKnownPeerObservedAtSweptAt = 0;
|
|
3647
3684
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
3648
3685
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
3649
3686
|
this._appendBackfillPendingByTarget = new Map();
|
|
@@ -7649,6 +7686,16 @@ export class SharedLog<
|
|
|
7649
7686
|
this._nativeSharedLogState?.markEntriesKnownByPeer(hashArray, peer);
|
|
7650
7687
|
this._nativeBackbone?.markEntriesKnownByPeer(hashArray, peer);
|
|
7651
7688
|
const now = Date.now();
|
|
7689
|
+
// Growth is driven by writes, so the sweep rides the write path rather
|
|
7690
|
+
// than a timer or the rebalance pass: cost stays proportional to the
|
|
7691
|
+
// traffic that creates rows. Rate-limited to one pass per retention
|
|
7692
|
+
// window, over a map that after the first pass holds one window of marks.
|
|
7693
|
+
if (
|
|
7694
|
+
now - this._entryKnownPeerObservedAtSweptAt >=
|
|
7695
|
+
ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS
|
|
7696
|
+
) {
|
|
7697
|
+
this.sweepEntryKnownPeerObservedAt(now);
|
|
7698
|
+
}
|
|
7652
7699
|
for (const hash of hashArray) {
|
|
7653
7700
|
let peers = this._entryKnownPeers.get(hash);
|
|
7654
7701
|
if (!peers) {
|
|
@@ -7718,6 +7765,28 @@ export class SharedLog<
|
|
|
7718
7765
|
return observedAt != null && Date.now() - observedAt <= maxAgeMs;
|
|
7719
7766
|
}
|
|
7720
7767
|
|
|
7768
|
+
/** Drop recency marks no reader can still act on.
|
|
7769
|
+
*
|
|
7770
|
+
* Touches ONLY `_entryKnownPeerObservedAt`. `_entryKnownPeers` carries
|
|
7771
|
+
* membership, not recency, and its rows stay until the peer dimension
|
|
7772
|
+
* clears them; the native mirrors have no recency dimension at all
|
|
7773
|
+
* (mark/remove/removePeer only), so this must not call into them or the
|
|
7774
|
+
* two sides would disagree.
|
|
7775
|
+
*/
|
|
7776
|
+
private sweepEntryKnownPeerObservedAt(now: number) {
|
|
7777
|
+
for (const [hash, observedAt] of this._entryKnownPeerObservedAt) {
|
|
7778
|
+
for (const [peer, timestamp] of observedAt) {
|
|
7779
|
+
if (now - timestamp > ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS) {
|
|
7780
|
+
observedAt.delete(peer);
|
|
7781
|
+
}
|
|
7782
|
+
}
|
|
7783
|
+
if (observedAt.size === 0) {
|
|
7784
|
+
this._entryKnownPeerObservedAt.delete(hash);
|
|
7785
|
+
}
|
|
7786
|
+
}
|
|
7787
|
+
this._entryKnownPeerObservedAtSweptAt = now;
|
|
7788
|
+
}
|
|
7789
|
+
|
|
7721
7790
|
private markRepairSweepOptimisticPeer(
|
|
7722
7791
|
gid: string,
|
|
7723
7792
|
peer: string,
|
|
@@ -10934,12 +11003,21 @@ export class SharedLog<
|
|
|
10934
11003
|
return rollbackLowerPublication(error);
|
|
10935
11004
|
}
|
|
10936
11005
|
if (!result) {
|
|
11006
|
+
// Abandon arm: the token was minted but nothing downstream can
|
|
11007
|
+
// roll it back from here.
|
|
11008
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11009
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
11010
|
+
);
|
|
10937
11011
|
return this.completeNativeStrictDurableTransaction(
|
|
10938
11012
|
nativeStrictTransaction,
|
|
10939
11013
|
).then(() => undefined);
|
|
10940
11014
|
}
|
|
10941
11015
|
return mapMaybePromise(result, async (prepared) => {
|
|
10942
11016
|
if (!prepared) {
|
|
11017
|
+
// Abandon arm: same shape as the `!result` arm above.
|
|
11018
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11019
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
11020
|
+
);
|
|
10943
11021
|
await this.completeNativeStrictDurableTransaction(
|
|
10944
11022
|
nativeStrictTransaction,
|
|
10945
11023
|
);
|
|
@@ -10973,6 +11051,11 @@ export class SharedLog<
|
|
|
10973
11051
|
prepared.appendFacts.hash,
|
|
10974
11052
|
lowerPublicationRollback?.coordinateEntries,
|
|
10975
11053
|
);
|
|
11054
|
+
// Terminal: this is the last rollback consumer for the
|
|
11055
|
+
// token. A throw above leaves the row, which is safe.
|
|
11056
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11057
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
11058
|
+
);
|
|
10976
11059
|
for (const document of lowerPublicationRollback?.documents ?? []) {
|
|
10977
11060
|
this.restoreNativeBackboneDocument(document);
|
|
10978
11061
|
}
|
|
@@ -11043,6 +11126,14 @@ export class SharedLog<
|
|
|
11043
11126
|
} catch (error) {
|
|
11044
11127
|
return rollback(error);
|
|
11045
11128
|
}
|
|
11129
|
+
// Success seam. The last await inside the protected try is the
|
|
11130
|
+
// finalizer acknowledge; `finish()` is synchronous, so no async
|
|
11131
|
+
// boundary separates the catch above from this statement and
|
|
11132
|
+
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
11133
|
+
// (the retire below only warns), so the token is terminal here.
|
|
11134
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11135
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
11136
|
+
);
|
|
11046
11137
|
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
11047
11138
|
prepared.appendFacts,
|
|
11048
11139
|
prepared.removed,
|
|
@@ -11499,6 +11590,12 @@ export class SharedLog<
|
|
|
11499
11590
|
}
|
|
11500
11591
|
return mapMaybePromise(result, async (prepared) => {
|
|
11501
11592
|
if (!prepared || !backboneAppend) {
|
|
11593
|
+
// Abandon arm: the token was minted inside
|
|
11594
|
+
// `prepareBackboneAppend` and nothing downstream of
|
|
11595
|
+
// this return can roll it back.
|
|
11596
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11597
|
+
nativeCoordinateRollback,
|
|
11598
|
+
);
|
|
11502
11599
|
await this.completeNativeStrictDurableTransaction(
|
|
11503
11600
|
nativeStrictTransaction,
|
|
11504
11601
|
);
|
|
@@ -11590,6 +11687,10 @@ export class SharedLog<
|
|
|
11590
11687
|
prepared.appendFacts.hash,
|
|
11591
11688
|
rollbackCoordinateEntries,
|
|
11592
11689
|
);
|
|
11690
|
+
// Terminal: last rollback consumer for the token.
|
|
11691
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11692
|
+
rollbackCoordinateEntries,
|
|
11693
|
+
);
|
|
11593
11694
|
} catch (rollbackError) {
|
|
11594
11695
|
rollbackFailures.push(rollbackError);
|
|
11595
11696
|
}
|
|
@@ -11666,6 +11767,12 @@ export class SharedLog<
|
|
|
11666
11767
|
} catch (error) {
|
|
11667
11768
|
return rollback(error);
|
|
11668
11769
|
}
|
|
11770
|
+
// Success seam: the finalizer acknowledge above is the last
|
|
11771
|
+
// await inside the protected try, so `rollback` can no
|
|
11772
|
+
// longer fire and the retire below only warns.
|
|
11773
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
11774
|
+
rollbackCoordinateEntries,
|
|
11775
|
+
);
|
|
11669
11776
|
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
11670
11777
|
prepared.appendFacts,
|
|
11671
11778
|
prepared.removed,
|
|
@@ -12585,6 +12692,10 @@ export class SharedLog<
|
|
|
12585
12692
|
throw error;
|
|
12586
12693
|
}
|
|
12587
12694
|
if (!appended || !backboneAppends) {
|
|
12695
|
+
// Abandon arm: no consumer downstream of this return.
|
|
12696
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
12697
|
+
batchCoordinateRollback,
|
|
12698
|
+
);
|
|
12588
12699
|
await this.completeNativeStrictDurableTransaction(
|
|
12589
12700
|
nativeStrictTransaction,
|
|
12590
12701
|
);
|
|
@@ -12622,6 +12733,10 @@ export class SharedLog<
|
|
|
12622
12733
|
appended.appendFacts[0]?.hash ?? "",
|
|
12623
12734
|
batchCoordinateRollback,
|
|
12624
12735
|
);
|
|
12736
|
+
// Terminal: last rollback consumer for the batch token.
|
|
12737
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
12738
|
+
batchCoordinateRollback,
|
|
12739
|
+
);
|
|
12625
12740
|
} catch (rollbackError) {
|
|
12626
12741
|
rollbackFailures.push(rollbackError);
|
|
12627
12742
|
}
|
|
@@ -12753,6 +12868,11 @@ export class SharedLog<
|
|
|
12753
12868
|
} catch (error) {
|
|
12754
12869
|
return rollbackBatch(error);
|
|
12755
12870
|
}
|
|
12871
|
+
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
12872
|
+
// above), and everything from here on escapes without any rollback.
|
|
12873
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
12874
|
+
batchCoordinateRollback,
|
|
12875
|
+
);
|
|
12756
12876
|
|
|
12757
12877
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
12758
12878
|
ownershipLifecycleController,
|
|
@@ -14213,6 +14333,7 @@ export class SharedLog<
|
|
|
14213
14333
|
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
14214
14334
|
this._entryKnownPeers = new Map();
|
|
14215
14335
|
this._entryKnownPeerObservedAt = new Map();
|
|
14336
|
+
this._entryKnownPeerObservedAtSweptAt = 0;
|
|
14216
14337
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
14217
14338
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
14218
14339
|
this._assumeSyncedRepairSuppressedUntil = 0;
|
|
@@ -18861,292 +18982,305 @@ export class SharedLog<
|
|
|
18861
18982
|
reusableCoordinatePersistItems,
|
|
18862
18983
|
)
|
|
18863
18984
|
: undefined;
|
|
18864
|
-
|
|
18865
|
-
|
|
18866
|
-
|
|
18867
|
-
|
|
18868
|
-
|
|
18869
|
-
|
|
18870
|
-
|
|
18871
|
-
|
|
18872
|
-
|
|
18873
|
-
(committedHashes) => {
|
|
18874
|
-
nativePreparedCommittedHashes = new Set(committedHashes);
|
|
18875
|
-
},
|
|
18876
|
-
)
|
|
18877
|
-
: undefined;
|
|
18878
|
-
const finishNativePreparedCoordinates = async (properties: {
|
|
18879
|
-
nativePreparedCommitted: boolean;
|
|
18880
|
-
}) => {
|
|
18881
|
-
if (
|
|
18882
|
-
!properties.nativePreparedCommitted ||
|
|
18883
|
-
!nativePreparedCoordinateBatch
|
|
18884
|
-
) {
|
|
18885
|
-
return;
|
|
18886
|
-
}
|
|
18887
|
-
try {
|
|
18888
|
-
nativeBackboneOnlyPersistedHashes =
|
|
18889
|
-
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
|
|
18890
|
-
nativePreparedCoordinateBatch,
|
|
18985
|
+
try {
|
|
18986
|
+
const nativePreparedJoinCommit = canUsePreparedAppendFacts
|
|
18987
|
+
? this._coordinates.createNativeBackbonePreparedJoinCommit(
|
|
18988
|
+
nativeReceiveCoordinateBatch,
|
|
18989
|
+
(batch) => {
|
|
18990
|
+
nativePreparedCoordinateBatch = batch;
|
|
18991
|
+
},
|
|
18992
|
+
nativeCommitVerifyHashes,
|
|
18993
|
+
nativeCommitVerifyAllHashes,
|
|
18891
18994
|
syncProfile,
|
|
18995
|
+
(committedHashes) => {
|
|
18996
|
+
nativePreparedCommittedHashes = new Set(committedHashes);
|
|
18997
|
+
},
|
|
18998
|
+
)
|
|
18999
|
+
: undefined;
|
|
19000
|
+
const finishNativePreparedCoordinates = async (properties: {
|
|
19001
|
+
nativePreparedCommitted: boolean;
|
|
19002
|
+
}) => {
|
|
19003
|
+
if (
|
|
19004
|
+
!properties.nativePreparedCommitted ||
|
|
19005
|
+
!nativePreparedCoordinateBatch
|
|
19006
|
+
) {
|
|
19007
|
+
return;
|
|
19008
|
+
}
|
|
19009
|
+
try {
|
|
19010
|
+
nativeBackboneOnlyPersistedHashes =
|
|
19011
|
+
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
|
|
19012
|
+
nativePreparedCoordinateBatch,
|
|
19013
|
+
syncProfile,
|
|
19014
|
+
);
|
|
19015
|
+
nativePreparedCoordinatesFinished = true;
|
|
19016
|
+
} catch (error) {
|
|
19017
|
+
this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
|
|
19018
|
+
nativePreparedCoordinateBatch,
|
|
18892
19019
|
);
|
|
18893
|
-
|
|
18894
|
-
|
|
18895
|
-
|
|
18896
|
-
|
|
18897
|
-
|
|
18898
|
-
|
|
19020
|
+
throw error;
|
|
19021
|
+
}
|
|
19022
|
+
};
|
|
19023
|
+
const preparedAppendCanValidateAppend =
|
|
19024
|
+
canAppendAlreadyValidated ||
|
|
19025
|
+
(nativeCommitCanValidateAppend && !!nativePreparedJoinCommit);
|
|
19026
|
+
if (!preparedAppendCanValidateAppend) {
|
|
19027
|
+
canUsePreparedAppendFacts = false;
|
|
18899
19028
|
}
|
|
18900
|
-
|
|
18901
|
-
|
|
18902
|
-
|
|
18903
|
-
|
|
18904
|
-
|
|
18905
|
-
|
|
18906
|
-
|
|
18907
|
-
|
|
18908
|
-
|
|
18909
|
-
|
|
18910
|
-
? nativeCommitVerifyAllHashes
|
|
18911
|
-
? !!this._nativeBackbone?.graph
|
|
18912
|
-
.commitVerifiedAllPreparedRawReceiveJoinBatch ||
|
|
18913
|
-
!!this._nativeBackbone?.graph
|
|
18914
|
-
.commitVerifiedPreparedRawReceiveJoinBatch
|
|
19029
|
+
const nativePreparedJoinCommitValidatesPlan =
|
|
19030
|
+
!!nativePreparedJoinCommit &&
|
|
19031
|
+
(nativeCommitVerifyHashes && nativeCommitVerifyHashes.length > 0
|
|
19032
|
+
? nativeCommitVerifyAllHashes
|
|
19033
|
+
? !!this._nativeBackbone?.graph
|
|
19034
|
+
.commitVerifiedAllPreparedRawReceiveJoinBatch ||
|
|
19035
|
+
!!this._nativeBackbone?.graph
|
|
19036
|
+
.commitVerifiedPreparedRawReceiveJoinBatch
|
|
19037
|
+
: !!this._nativeBackbone?.graph
|
|
19038
|
+
.commitVerifiedPreparedRawReceiveJoinBatch
|
|
18915
19039
|
: !!this._nativeBackbone?.graph
|
|
18916
|
-
.
|
|
18917
|
-
|
|
18918
|
-
|
|
18919
|
-
|
|
18920
|
-
|
|
18921
|
-
|
|
18922
|
-
|
|
18923
|
-
|
|
18924
|
-
|
|
18925
|
-
|
|
18926
|
-
|
|
18927
|
-
|
|
18928
|
-
|
|
18929
|
-
|
|
18930
|
-
|
|
18931
|
-
|
|
19040
|
+
.commitPreparedRawReceiveJoinBatch);
|
|
19041
|
+
const trustedLowerLog = this.log as unknown as TrustedLowerLog<T>;
|
|
19042
|
+
// With a program-level onChange consumer the hash-only
|
|
19043
|
+
// sink is not used: the lower-log join dispatches the
|
|
19044
|
+
// change event (lazy entry views over the prepared raw
|
|
19045
|
+
// facts) so per-entry consumers observe every commit.
|
|
19046
|
+
const joinOnAppendHashes = programOnChange
|
|
19047
|
+
? undefined
|
|
19048
|
+
: onAppendHashes;
|
|
19049
|
+
const joinedPreparedFacts =
|
|
19050
|
+
canUsePreparedAppendFacts &&
|
|
19051
|
+
(await trustedLowerLog.joinPreparedAppendFactsBatch(
|
|
19052
|
+
preparedAppendFacts,
|
|
19053
|
+
{
|
|
19054
|
+
__peerbitEntriesAlreadyMissing: true,
|
|
19055
|
+
__peerbitCanAppendAlreadyValidated: true,
|
|
19056
|
+
__peerbitDeferIndexWrite: true,
|
|
19057
|
+
__peerbitOnAppendHashes: joinOnAppendHashes,
|
|
19058
|
+
__peerbitProfile: syncProfile,
|
|
19059
|
+
__peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
|
|
19060
|
+
__peerbitNativePreparedJoinCommitValidatesPlan:
|
|
19061
|
+
nativePreparedJoinCommitValidatesPlan,
|
|
19062
|
+
__peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
|
|
19063
|
+
? finishNativePreparedCoordinates
|
|
19064
|
+
: undefined,
|
|
19065
|
+
},
|
|
19066
|
+
));
|
|
19067
|
+
if (!joinedPreparedFacts) {
|
|
19068
|
+
await trustedLowerLog.join(materializeAllToMergeEntries(), {
|
|
19069
|
+
__peerbitBatchIndependent: true,
|
|
18932
19070
|
__peerbitEntriesAlreadyMissing: true,
|
|
18933
|
-
__peerbitCanAppendAlreadyValidated:
|
|
19071
|
+
__peerbitCanAppendAlreadyValidated:
|
|
19072
|
+
fallbackCanAppendAlreadyValidated,
|
|
18934
19073
|
__peerbitDeferIndexWrite: true,
|
|
18935
19074
|
__peerbitOnAppendHashes: joinOnAppendHashes,
|
|
18936
19075
|
__peerbitProfile: syncProfile,
|
|
18937
|
-
|
|
18938
|
-
|
|
18939
|
-
|
|
18940
|
-
|
|
18941
|
-
|
|
18942
|
-
|
|
18943
|
-
|
|
18944
|
-
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
|
|
18948
|
-
|
|
18949
|
-
|
|
18950
|
-
|
|
18951
|
-
|
|
18952
|
-
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
18956
|
-
|
|
18957
|
-
|
|
18958
|
-
|
|
18959
|
-
|
|
18960
|
-
|
|
18961
|
-
|
|
18962
|
-
const admittedHashes = joinedPreparedFacts
|
|
18963
|
-
? new Set(allToMergeHashes)
|
|
18964
|
-
: await this.log.hasMany(allToMergeHashes);
|
|
18965
|
-
admittedMergeHashes = admittedHashes;
|
|
18966
|
-
const admittedShallowEntries =
|
|
18967
|
-
admittedHashes.size === allToMergeShallowEntries.length
|
|
18968
|
-
? allToMergeShallowEntries
|
|
18969
|
-
: allToMergeShallowEntries.filter((entry) =>
|
|
19076
|
+
});
|
|
19077
|
+
}
|
|
19078
|
+
// A recursive lower-log join can resolve successfully while declining
|
|
19079
|
+
// an individual top-level entry (for example, when one of its parents
|
|
19080
|
+
// is temporarily unavailable). The public Log.join() API intentionally
|
|
19081
|
+
// does not expose that per-entry result, so make local index presence the
|
|
19082
|
+
// authority before publishing any SharedLog-side effects. A successful
|
|
19083
|
+
// prepared-facts batch is atomic and already proves every input hash.
|
|
19084
|
+
const admittedHashes = joinedPreparedFacts
|
|
19085
|
+
? new Set(allToMergeHashes)
|
|
19086
|
+
: await this.log.hasMany(allToMergeHashes);
|
|
19087
|
+
admittedMergeHashes = admittedHashes;
|
|
19088
|
+
const admittedShallowEntries =
|
|
19089
|
+
admittedHashes.size === allToMergeShallowEntries.length
|
|
19090
|
+
? allToMergeShallowEntries
|
|
19091
|
+
: allToMergeShallowEntries.filter((entry) =>
|
|
19092
|
+
admittedHashes.has(entry.hash),
|
|
19093
|
+
);
|
|
19094
|
+
if (!joinedPreparedFacts) {
|
|
19095
|
+
reusableCoordinatePersistItems =
|
|
19096
|
+
reusableCoordinatePersistItems.filter((item) =>
|
|
19097
|
+
admittedHashes.has(item.entry.hash),
|
|
19098
|
+
);
|
|
19099
|
+
coordinatePersistFallbackEntries =
|
|
19100
|
+
coordinatePersistFallbackEntries.filter((entry) =>
|
|
18970
19101
|
admittedHashes.has(entry.hash),
|
|
18971
19102
|
);
|
|
18972
|
-
|
|
18973
|
-
|
|
18974
|
-
reusableCoordinatePersistItems.
|
|
18975
|
-
|
|
18976
|
-
|
|
18977
|
-
|
|
18978
|
-
|
|
18979
|
-
|
|
18980
|
-
|
|
18981
|
-
|
|
18982
|
-
|
|
18983
|
-
|
|
18984
|
-
|
|
18985
|
-
|
|
18986
|
-
|
|
18987
|
-
|
|
18988
|
-
|
|
18989
|
-
|
|
18990
|
-
|
|
18991
|
-
|
|
18992
|
-
|
|
18993
|
-
|
|
18994
|
-
|
|
18995
|
-
|
|
18996
|
-
|
|
18997
|
-
|
|
18998
|
-
|
|
18999
|
-
|
|
19000
|
-
|
|
19001
|
-
|
|
19002
|
-
|
|
19003
|
-
|
|
19004
|
-
} else if (nativePreparedCoordinateBatch) {
|
|
19005
|
-
try {
|
|
19006
|
-
nativeBackboneOnlyPersistedHashes =
|
|
19007
|
-
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
|
|
19103
|
+
}
|
|
19104
|
+
const reusableCoordinatePersistItemCount =
|
|
19105
|
+
reusableCoordinatePersistItems.length;
|
|
19106
|
+
if (syncProfile) {
|
|
19107
|
+
emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
|
|
19108
|
+
name: "sharedLog.receive.lowerLogJoin",
|
|
19109
|
+
component: "shared-log",
|
|
19110
|
+
entries: allToMerge.length,
|
|
19111
|
+
messages: 1,
|
|
19112
|
+
details: {
|
|
19113
|
+
hashOnlyEntryAdded,
|
|
19114
|
+
batchHashOnlyEntryAdded,
|
|
19115
|
+
programOnChange,
|
|
19116
|
+
joinedPreparedFacts,
|
|
19117
|
+
admittedEntries: admittedHashes.size,
|
|
19118
|
+
nativePreparedCoordinatesFinished,
|
|
19119
|
+
},
|
|
19120
|
+
});
|
|
19121
|
+
}
|
|
19122
|
+
const coordinatePersistStartedAt = syncProfileStart(syncProfile);
|
|
19123
|
+
if (nativePreparedCoordinatesFinished) {
|
|
19124
|
+
// The lower-log prepared receive transaction already finished
|
|
19125
|
+
// the native coordinate mirror/journal after entry-index commit.
|
|
19126
|
+
} else if (nativePreparedCoordinateBatch) {
|
|
19127
|
+
try {
|
|
19128
|
+
nativeBackboneOnlyPersistedHashes =
|
|
19129
|
+
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
|
|
19130
|
+
nativePreparedCoordinateBatch,
|
|
19131
|
+
syncProfile,
|
|
19132
|
+
);
|
|
19133
|
+
} catch (error) {
|
|
19134
|
+
this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
|
|
19008
19135
|
nativePreparedCoordinateBatch,
|
|
19009
|
-
syncProfile,
|
|
19010
19136
|
);
|
|
19011
|
-
|
|
19012
|
-
|
|
19013
|
-
|
|
19014
|
-
|
|
19015
|
-
|
|
19137
|
+
throw error;
|
|
19138
|
+
}
|
|
19139
|
+
} else {
|
|
19140
|
+
nativeBackboneOnlyPersistedHashes =
|
|
19141
|
+
await this._coordinates.persistBackboneOnlyReceiveCoordinateBatch(
|
|
19142
|
+
reusableCoordinatePersistItems,
|
|
19143
|
+
);
|
|
19016
19144
|
}
|
|
19017
|
-
|
|
19018
|
-
|
|
19019
|
-
|
|
19020
|
-
reusableCoordinatePersistItems,
|
|
19021
|
-
);
|
|
19022
|
-
}
|
|
19023
|
-
if (
|
|
19024
|
-
nativeBackboneOnlyPersistedHashes &&
|
|
19025
|
-
nativeBackboneOnlyPersistedHashes.size > 0
|
|
19026
|
-
) {
|
|
19027
|
-
for (
|
|
19028
|
-
let i = reusableCoordinatePersistItems.length - 1;
|
|
19029
|
-
i >= 0;
|
|
19030
|
-
i--
|
|
19145
|
+
if (
|
|
19146
|
+
nativeBackboneOnlyPersistedHashes &&
|
|
19147
|
+
nativeBackboneOnlyPersistedHashes.size > 0
|
|
19031
19148
|
) {
|
|
19032
|
-
|
|
19033
|
-
|
|
19034
|
-
|
|
19035
|
-
|
|
19149
|
+
for (
|
|
19150
|
+
let i = reusableCoordinatePersistItems.length - 1;
|
|
19151
|
+
i >= 0;
|
|
19152
|
+
i--
|
|
19036
19153
|
) {
|
|
19037
|
-
|
|
19154
|
+
if (
|
|
19155
|
+
nativeBackboneOnlyPersistedHashes.has(
|
|
19156
|
+
reusableCoordinatePersistItems[i]!.entry.hash,
|
|
19157
|
+
)
|
|
19158
|
+
) {
|
|
19159
|
+
reusableCoordinatePersistItems.splice(i, 1);
|
|
19160
|
+
}
|
|
19038
19161
|
}
|
|
19039
19162
|
}
|
|
19040
|
-
|
|
19041
|
-
|
|
19042
|
-
|
|
19043
|
-
|
|
19044
|
-
|
|
19045
|
-
|
|
19046
|
-
|
|
19047
|
-
|
|
19048
|
-
|
|
19049
|
-
|
|
19050
|
-
|
|
19051
|
-
|
|
19052
|
-
|
|
19053
|
-
|
|
19054
|
-
|
|
19055
|
-
|
|
19056
|
-
|
|
19057
|
-
|
|
19058
|
-
|
|
19059
|
-
|
|
19060
|
-
|
|
19061
|
-
|
|
19062
|
-
|
|
19063
|
-
|
|
19064
|
-
|
|
19065
|
-
|
|
19066
|
-
|
|
19067
|
-
}
|
|
19068
|
-
}
|
|
19069
|
-
|
|
19070
|
-
|
|
19071
|
-
|
|
19072
|
-
|
|
19073
|
-
|
|
19074
|
-
|
|
19075
|
-
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
|
|
19079
|
-
|
|
19080
|
-
|
|
19081
|
-
|
|
19082
|
-
|
|
19163
|
+
if (reusableCoordinatePersistItems.length > 0) {
|
|
19164
|
+
await this._coordinates.persistCoordinatesBatch(
|
|
19165
|
+
reusableCoordinatePersistItems,
|
|
19166
|
+
);
|
|
19167
|
+
}
|
|
19168
|
+
if (coordinatePersistFallbackEntries.length > 0) {
|
|
19169
|
+
await this.planEntryLeaderBatch(
|
|
19170
|
+
coordinatePersistFallbackEntries.map((entry) => ({
|
|
19171
|
+
entry,
|
|
19172
|
+
replicas:
|
|
19173
|
+
receiveReplicaCounts.get(entry.hash) ??
|
|
19174
|
+
decodeReplicas(entry).getValue(this),
|
|
19175
|
+
options: { roleAge: 0, persist: {} },
|
|
19176
|
+
})),
|
|
19177
|
+
);
|
|
19178
|
+
}
|
|
19179
|
+
if (syncProfile) {
|
|
19180
|
+
emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
|
|
19181
|
+
name: "sharedLog.receive.coordinatePersist",
|
|
19182
|
+
component: "shared-log",
|
|
19183
|
+
entries: entriesToPersist.length,
|
|
19184
|
+
messages: 1,
|
|
19185
|
+
details: {
|
|
19186
|
+
reusedLeaderPlans: reusableCoordinatePersistItemCount,
|
|
19187
|
+
nativeBackboneOnly:
|
|
19188
|
+
nativeBackboneOnlyPersistedHashes?.size ?? 0,
|
|
19189
|
+
},
|
|
19190
|
+
});
|
|
19191
|
+
}
|
|
19192
|
+
for (const hash of admittedHashes) {
|
|
19193
|
+
confirmedHashes.add(hash);
|
|
19194
|
+
}
|
|
19195
|
+
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
19196
|
+
const ownershipChangedDuringReceive =
|
|
19197
|
+
!this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
|
|
19198
|
+
if (ownershipChangedDuringReceive) {
|
|
19199
|
+
const freshAuditRevision =
|
|
19200
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
19201
|
+
const armFreshAuditRetry = () => {
|
|
19202
|
+
for (const entry of admittedShallowEntries) {
|
|
19203
|
+
this.scheduleCheckedPruneRetry(
|
|
19204
|
+
{ entry, leaders: new Map() },
|
|
19205
|
+
receiveOwnershipLifecycleController,
|
|
19206
|
+
);
|
|
19207
|
+
}
|
|
19208
|
+
};
|
|
19209
|
+
try {
|
|
19210
|
+
await this.pruneJoinedEntriesNoLongerLed(
|
|
19211
|
+
admittedShallowEntries,
|
|
19212
|
+
{
|
|
19213
|
+
decodedReplicaCounts: receiveReplicaCounts,
|
|
19214
|
+
freshReceiveOwnerAudit: true,
|
|
19215
|
+
preserveExistingPruneOnLocalResult: true,
|
|
19216
|
+
profile: syncProfile,
|
|
19217
|
+
},
|
|
19083
19218
|
receiveOwnershipLifecycleController,
|
|
19084
19219
|
);
|
|
19220
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
19221
|
+
receiveOwnershipLifecycleController,
|
|
19222
|
+
);
|
|
19223
|
+
if (
|
|
19224
|
+
!this.isReceiveOwnershipSnapshotStable(freshAuditRevision)
|
|
19225
|
+
) {
|
|
19226
|
+
armFreshAuditRetry();
|
|
19227
|
+
}
|
|
19228
|
+
} catch {
|
|
19229
|
+
// The lower-log and coordinate commits are already durable. A
|
|
19230
|
+
// sender retry will filter these hashes as present, so retain a
|
|
19231
|
+
// bounded local obligation instead of failing the admitted receive.
|
|
19232
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
19233
|
+
receiveOwnershipLifecycleController,
|
|
19234
|
+
);
|
|
19235
|
+
armFreshAuditRetry();
|
|
19085
19236
|
}
|
|
19086
|
-
}
|
|
19087
|
-
try {
|
|
19237
|
+
} else {
|
|
19088
19238
|
await this.pruneJoinedEntriesNoLongerLed(
|
|
19089
19239
|
admittedShallowEntries,
|
|
19090
19240
|
{
|
|
19091
19241
|
decodedReplicaCounts: receiveReplicaCounts,
|
|
19092
|
-
freshReceiveOwnerAudit: true,
|
|
19093
19242
|
preserveExistingPruneOnLocalResult: true,
|
|
19243
|
+
reusableLeaderPlans: reusableCoordinatePlans,
|
|
19094
19244
|
profile: syncProfile,
|
|
19095
19245
|
},
|
|
19096
19246
|
receiveOwnershipLifecycleController,
|
|
19097
19247
|
);
|
|
19098
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
19099
|
-
receiveOwnershipLifecycleController,
|
|
19100
|
-
);
|
|
19101
|
-
if (
|
|
19102
|
-
!this.isReceiveOwnershipSnapshotStable(freshAuditRevision)
|
|
19103
|
-
) {
|
|
19104
|
-
armFreshAuditRetry();
|
|
19105
|
-
}
|
|
19106
|
-
} catch {
|
|
19107
|
-
// The lower-log and coordinate commits are already durable. A
|
|
19108
|
-
// sender retry will filter these hashes as present, so retain a
|
|
19109
|
-
// bounded local obligation instead of failing the admitted receive.
|
|
19110
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
19111
|
-
receiveOwnershipLifecycleController,
|
|
19112
|
-
);
|
|
19113
|
-
armFreshAuditRetry();
|
|
19114
19248
|
}
|
|
19115
|
-
|
|
19116
|
-
|
|
19117
|
-
|
|
19118
|
-
|
|
19119
|
-
|
|
19120
|
-
|
|
19121
|
-
|
|
19122
|
-
|
|
19123
|
-
},
|
|
19124
|
-
receiveOwnershipLifecycleController,
|
|
19125
|
-
);
|
|
19126
|
-
}
|
|
19127
|
-
if (syncProfile) {
|
|
19128
|
-
emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
|
|
19129
|
-
name: "sharedLog.receive.checkedPrune",
|
|
19130
|
-
component: "shared-log",
|
|
19131
|
-
entries: allToMerge.length,
|
|
19132
|
-
messages: 1,
|
|
19133
|
-
});
|
|
19134
|
-
}
|
|
19249
|
+
if (syncProfile) {
|
|
19250
|
+
emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
|
|
19251
|
+
name: "sharedLog.receive.checkedPrune",
|
|
19252
|
+
component: "shared-log",
|
|
19253
|
+
entries: allToMerge.length,
|
|
19254
|
+
messages: 1,
|
|
19255
|
+
});
|
|
19256
|
+
}
|
|
19135
19257
|
|
|
19136
|
-
|
|
19137
|
-
|
|
19138
|
-
|
|
19139
|
-
|
|
19140
|
-
|
|
19141
|
-
|
|
19142
|
-
|
|
19143
|
-
|
|
19144
|
-
|
|
19145
|
-
|
|
19146
|
-
|
|
19147
|
-
|
|
19258
|
+
for (const plan of joinPlans) {
|
|
19259
|
+
plan.toDelete
|
|
19260
|
+
?.filter((entry) => admittedMergeHashes.has(entry.hash))
|
|
19261
|
+
.map((entry) =>
|
|
19262
|
+
this.pruneDebouncedFnAddIfNotKeeping({
|
|
19263
|
+
key: entry.hash,
|
|
19264
|
+
value: {
|
|
19265
|
+
entry,
|
|
19266
|
+
leaders: plan.leaders as Map<string, any>,
|
|
19267
|
+
},
|
|
19268
|
+
}),
|
|
19269
|
+
);
|
|
19270
|
+
}
|
|
19271
|
+
this.rebalanceParticipationDebounced?.call();
|
|
19272
|
+
} finally {
|
|
19273
|
+
// Settle seam for the receive token. Every consumer that can
|
|
19274
|
+
// roll it back runs inline before control leaves this block: the
|
|
19275
|
+
// prepared-join callback resolves during the join await, and the
|
|
19276
|
+
// late finish/rollback arm runs above. This `finally` is what
|
|
19277
|
+
// closes the abandon arms (no prepared-join commit, a declined
|
|
19278
|
+
// native commit, a downgrade to the plain join) without having
|
|
19279
|
+
// to enumerate them.
|
|
19280
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
19281
|
+
nativeReceiveCoordinateBatch?.rollbackCoordinateEntries,
|
|
19282
|
+
);
|
|
19148
19283
|
}
|
|
19149
|
-
this.rebalanceParticipationDebounced?.call();
|
|
19150
19284
|
}
|
|
19151
19285
|
|
|
19152
19286
|
for (const plan of joinPlans) {
|