@peerbit/shared-log 16.0.1 → 16.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/coordinate-persistence.d.ts +18 -1
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +61 -4
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/index.d.ts +17 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +301 -201
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/simple.d.ts +1 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +1 -1
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +11 -11
- package/src/coordinate-persistence.ts +66 -5
- package/src/index.ts +391 -257
- package/src/sync/simple.ts +1 -1
package/dist/src/index.js
CHANGED
|
@@ -76,7 +76,7 @@ import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmen
|
|
|
76
76
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
77
77
|
import { createSyncronizer } from "./sync/factory.js";
|
|
78
78
|
import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./sync/profile.js";
|
|
79
|
-
import { ConfirmEntriesMessage, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
79
|
+
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
80
80
|
import { groupByGid, tryGroupByGidSync } from "./utils.js";
|
|
81
81
|
const getSharedLogFanoutService = (services) => services.fanout;
|
|
82
82
|
const createOneShotPeerReceiveLease = (releaseFn) => {
|
|
@@ -449,6 +449,13 @@ const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
|
|
|
449
449
|
];
|
|
450
450
|
const APPEND_BACKFILL_RETRY_SCHEDULE_MS = [0, 1_000, 3_000, 7_000];
|
|
451
451
|
const RECENT_KNOWN_REPAIR_SUPPRESSION_MS = 30_000;
|
|
452
|
+
// `_entryKnownPeerObservedAt` is read ONLY through isEntryRecentlyKnownByPeer,
|
|
453
|
+
// which treats an over-age row and an absent row identically (both false). So
|
|
454
|
+
// rows older than the longest horizon any caller asks about are dead weight,
|
|
455
|
+
// and dropping them is behaviour-identical rather than merely safe. Derived
|
|
456
|
+
// from the horizons themselves -- never hardcode it -- so a future caller with
|
|
457
|
+
// a longer window cannot silently outlive the retention that serves it.
|
|
458
|
+
const ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS = Math.max(RECENT_KNOWN_REPAIR_SUPPRESSION_MS, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS);
|
|
452
459
|
const JOIN_AUTHORITATIVE_REPAIR_DELAY_MS = 2_000;
|
|
453
460
|
const JOIN_AUTHORITATIVE_REPAIR_SWEEP_DELAYS_MS = [
|
|
454
461
|
JOIN_AUTHORITATIVE_REPAIR_DELAY_MS,
|
|
@@ -1511,8 +1518,23 @@ let SharedLog = (() => {
|
|
|
1511
1518
|
};
|
|
1512
1519
|
for (const coordinate of intent.coordinates) {
|
|
1513
1520
|
rollback.hashes.add(coordinate.hash);
|
|
1514
|
-
|
|
1515
|
-
|
|
1521
|
+
// Same hold-counted row shape the coordinator's own
|
|
1522
|
+
// snapshot writes: one hold per hash, released by the
|
|
1523
|
+
// settle after the replay consumes the token below.
|
|
1524
|
+
// RELIES ON `intent.coordinates` HOLDING UNIQUE HASHES —
|
|
1525
|
+
// it is built from the token's `hashes` Set (see
|
|
1526
|
+
// setNativeStrictDurableTransactionOperation). Holds are
|
|
1527
|
+
// taken per element here but released per unique hash by
|
|
1528
|
+
// the settle, so a duplicate would take two and release
|
|
1529
|
+
// one and retain that row forever. That is the safe
|
|
1530
|
+
// direction (a retained row, never a fail-open rollback),
|
|
1531
|
+
// but keep the source a Set.
|
|
1532
|
+
const row = mutationGenerations.get(coordinate.hash);
|
|
1533
|
+
const generation = (row?.generation ?? 0) + 1;
|
|
1534
|
+
mutationGenerations.set(coordinate.hash, {
|
|
1535
|
+
generation,
|
|
1536
|
+
holds: (row?.holds ?? 0) + 1,
|
|
1537
|
+
});
|
|
1516
1538
|
rollback.generations.set(coordinate.hash, generation);
|
|
1517
1539
|
if (coordinate.value) {
|
|
1518
1540
|
const number = (value) => (this.domain.resolution === "u32"
|
|
@@ -1530,6 +1552,9 @@ let SharedLog = (() => {
|
|
|
1530
1552
|
}
|
|
1531
1553
|
}
|
|
1532
1554
|
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably("", rollback);
|
|
1555
|
+
// The replay fabricated these generations itself one turn
|
|
1556
|
+
// earlier and has now consumed them, so the rows are dead.
|
|
1557
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollback);
|
|
1533
1558
|
}
|
|
1534
1559
|
for (const document of intent.documents) {
|
|
1535
1560
|
this.restoreNativeBackboneDocument({
|
|
@@ -1633,6 +1658,7 @@ let SharedLog = (() => {
|
|
|
1633
1658
|
_repairSweepOptimisticGidsByPeer;
|
|
1634
1659
|
_entryKnownPeers;
|
|
1635
1660
|
_entryKnownPeerObservedAt;
|
|
1661
|
+
_entryKnownPeerObservedAtSweptAt = 0;
|
|
1636
1662
|
_joinAuthoritativeRepairTimersByDelay;
|
|
1637
1663
|
_joinAuthoritativeRepairPeersByDelay;
|
|
1638
1664
|
_assumeSyncedRepairSuppressedUntil;
|
|
@@ -1885,6 +1911,7 @@ let SharedLog = (() => {
|
|
|
1885
1911
|
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
1886
1912
|
this._entryKnownPeers = new Map();
|
|
1887
1913
|
this._entryKnownPeerObservedAt = new Map();
|
|
1914
|
+
this._entryKnownPeerObservedAtSweptAt = 0;
|
|
1888
1915
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
1889
1916
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
1890
1917
|
this._appendBackfillPendingByTarget = new Map();
|
|
@@ -4830,6 +4857,14 @@ let SharedLog = (() => {
|
|
|
4830
4857
|
this._nativeSharedLogState?.markEntriesKnownByPeer(hashArray, peer);
|
|
4831
4858
|
this._nativeBackbone?.markEntriesKnownByPeer(hashArray, peer);
|
|
4832
4859
|
const now = Date.now();
|
|
4860
|
+
// Growth is driven by writes, so the sweep rides the write path rather
|
|
4861
|
+
// than a timer or the rebalance pass: cost stays proportional to the
|
|
4862
|
+
// traffic that creates rows. Rate-limited to one pass per retention
|
|
4863
|
+
// window, over a map that after the first pass holds one window of marks.
|
|
4864
|
+
if (now - this._entryKnownPeerObservedAtSweptAt >=
|
|
4865
|
+
ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS) {
|
|
4866
|
+
this.sweepEntryKnownPeerObservedAt(now);
|
|
4867
|
+
}
|
|
4833
4868
|
for (const hash of hashArray) {
|
|
4834
4869
|
let peers = this._entryKnownPeers.get(hash);
|
|
4835
4870
|
if (!peers) {
|
|
@@ -4889,6 +4924,27 @@ let SharedLog = (() => {
|
|
|
4889
4924
|
const observedAt = this._entryKnownPeerObservedAt.get(hash)?.get(peer);
|
|
4890
4925
|
return observedAt != null && Date.now() - observedAt <= maxAgeMs;
|
|
4891
4926
|
}
|
|
4927
|
+
/** Drop recency marks no reader can still act on.
|
|
4928
|
+
*
|
|
4929
|
+
* Touches ONLY `_entryKnownPeerObservedAt`. `_entryKnownPeers` carries
|
|
4930
|
+
* membership, not recency, and its rows stay until the peer dimension
|
|
4931
|
+
* clears them; the native mirrors have no recency dimension at all
|
|
4932
|
+
* (mark/remove/removePeer only), so this must not call into them or the
|
|
4933
|
+
* two sides would disagree.
|
|
4934
|
+
*/
|
|
4935
|
+
sweepEntryKnownPeerObservedAt(now) {
|
|
4936
|
+
for (const [hash, observedAt] of this._entryKnownPeerObservedAt) {
|
|
4937
|
+
for (const [peer, timestamp] of observedAt) {
|
|
4938
|
+
if (now - timestamp > ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS) {
|
|
4939
|
+
observedAt.delete(peer);
|
|
4940
|
+
}
|
|
4941
|
+
}
|
|
4942
|
+
if (observedAt.size === 0) {
|
|
4943
|
+
this._entryKnownPeerObservedAt.delete(hash);
|
|
4944
|
+
}
|
|
4945
|
+
}
|
|
4946
|
+
this._entryKnownPeerObservedAtSweptAt = now;
|
|
4947
|
+
}
|
|
4892
4948
|
markRepairSweepOptimisticPeer(gid, peer, session) {
|
|
4893
4949
|
let peers = this._repairSweepOptimisticGidPeersPending.get(gid);
|
|
4894
4950
|
if (!peers) {
|
|
@@ -7043,10 +7099,15 @@ let SharedLog = (() => {
|
|
|
7043
7099
|
return rollbackLowerPublication(error);
|
|
7044
7100
|
}
|
|
7045
7101
|
if (!result) {
|
|
7102
|
+
// Abandon arm: the token was minted but nothing downstream can
|
|
7103
|
+
// roll it back from here.
|
|
7104
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
7046
7105
|
return this.completeNativeStrictDurableTransaction(nativeStrictTransaction).then(() => undefined);
|
|
7047
7106
|
}
|
|
7048
7107
|
return mapMaybePromise(result, async (prepared) => {
|
|
7049
7108
|
if (!prepared) {
|
|
7109
|
+
// Abandon arm: same shape as the `!result` arm above.
|
|
7110
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
7050
7111
|
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7051
7112
|
return undefined;
|
|
7052
7113
|
}
|
|
@@ -7067,6 +7128,9 @@ let SharedLog = (() => {
|
|
|
7067
7128
|
}
|
|
7068
7129
|
try {
|
|
7069
7130
|
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, lowerPublicationRollback?.coordinateEntries);
|
|
7131
|
+
// Terminal: this is the last rollback consumer for the
|
|
7132
|
+
// token. A throw above leaves the row, which is safe.
|
|
7133
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
7070
7134
|
for (const document of lowerPublicationRollback?.documents ?? []) {
|
|
7071
7135
|
this.restoreNativeBackboneDocument(document);
|
|
7072
7136
|
}
|
|
@@ -7126,6 +7190,12 @@ let SharedLog = (() => {
|
|
|
7126
7190
|
catch (error) {
|
|
7127
7191
|
return rollback(error);
|
|
7128
7192
|
}
|
|
7193
|
+
// Success seam. The last await inside the protected try is the
|
|
7194
|
+
// finalizer acknowledge; `finish()` is synchronous, so no async
|
|
7195
|
+
// boundary separates the catch above from this statement and
|
|
7196
|
+
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
7197
|
+
// (the retire below only warns), so the token is terminal here.
|
|
7198
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
7129
7199
|
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, { removedHashes: prepared.removedHashes });
|
|
7130
7200
|
try {
|
|
7131
7201
|
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
@@ -7428,6 +7498,10 @@ let SharedLog = (() => {
|
|
|
7428
7498
|
}
|
|
7429
7499
|
return mapMaybePromise(result, async (prepared) => {
|
|
7430
7500
|
if (!prepared || !backboneAppend) {
|
|
7501
|
+
// Abandon arm: the token was minted inside
|
|
7502
|
+
// `prepareBackboneAppend` and nothing downstream of
|
|
7503
|
+
// this return can roll it back.
|
|
7504
|
+
this._coordinates.settleResidentCoordinateSnapshot(nativeCoordinateRollback);
|
|
7431
7505
|
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7432
7506
|
return undefined;
|
|
7433
7507
|
}
|
|
@@ -7495,6 +7569,8 @@ let SharedLog = (() => {
|
|
|
7495
7569
|
}
|
|
7496
7570
|
try {
|
|
7497
7571
|
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(prepared.appendFacts.hash, rollbackCoordinateEntries);
|
|
7572
|
+
// Terminal: last rollback consumer for the token.
|
|
7573
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollbackCoordinateEntries);
|
|
7498
7574
|
}
|
|
7499
7575
|
catch (rollbackError) {
|
|
7500
7576
|
rollbackFailures.push(rollbackError);
|
|
@@ -7556,6 +7632,10 @@ let SharedLog = (() => {
|
|
|
7556
7632
|
catch (error) {
|
|
7557
7633
|
return rollback(error);
|
|
7558
7634
|
}
|
|
7635
|
+
// Success seam: the finalizer acknowledge above is the last
|
|
7636
|
+
// await inside the protected try, so `rollback` can no
|
|
7637
|
+
// longer fire and the retire below only warns.
|
|
7638
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollbackCoordinateEntries);
|
|
7559
7639
|
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
7560
7640
|
forgetNativeCoordinates: false,
|
|
7561
7641
|
removedHashes: prepared.removedHashes,
|
|
@@ -8110,6 +8190,8 @@ let SharedLog = (() => {
|
|
|
8110
8190
|
throw error;
|
|
8111
8191
|
}
|
|
8112
8192
|
if (!appended || !backboneAppends) {
|
|
8193
|
+
// Abandon arm: no consumer downstream of this return.
|
|
8194
|
+
this._coordinates.settleResidentCoordinateSnapshot(batchCoordinateRollback);
|
|
8113
8195
|
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
8114
8196
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8115
8197
|
return undefined;
|
|
@@ -8132,6 +8214,8 @@ let SharedLog = (() => {
|
|
|
8132
8214
|
}
|
|
8133
8215
|
try {
|
|
8134
8216
|
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(appended.appendFacts[0]?.hash ?? "", batchCoordinateRollback);
|
|
8217
|
+
// Terminal: last rollback consumer for the batch token.
|
|
8218
|
+
this._coordinates.settleResidentCoordinateSnapshot(batchCoordinateRollback);
|
|
8135
8219
|
}
|
|
8136
8220
|
catch (rollbackError) {
|
|
8137
8221
|
rollbackFailures.push(rollbackError);
|
|
@@ -8222,6 +8306,9 @@ let SharedLog = (() => {
|
|
|
8222
8306
|
catch (error) {
|
|
8223
8307
|
return rollbackBatch(error);
|
|
8224
8308
|
}
|
|
8309
|
+
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
8310
|
+
// above), and everything from here on escapes without any rollback.
|
|
8311
|
+
this._coordinates.settleResidentCoordinateSnapshot(batchCoordinateRollback);
|
|
8225
8312
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8226
8313
|
const appendCommits = [];
|
|
8227
8314
|
for (let i = 0; i < coordinateRows.length; i++) {
|
|
@@ -9160,6 +9247,7 @@ let SharedLog = (() => {
|
|
|
9160
9247
|
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
9161
9248
|
this._entryKnownPeers = new Map();
|
|
9162
9249
|
this._entryKnownPeerObservedAt = new Map();
|
|
9250
|
+
this._entryKnownPeerObservedAtSweptAt = 0;
|
|
9163
9251
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
9164
9252
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
9165
9253
|
this._assumeSyncedRepairSuppressedUntil = 0;
|
|
@@ -12886,221 +12974,233 @@ let SharedLog = (() => {
|
|
|
12886
12974
|
const nativeReceiveCoordinateBatch = canUsePreparedAppendFacts
|
|
12887
12975
|
? this._coordinates.createBackboneOnlyReceiveCoordinateBatch(reusableCoordinatePersistItems)
|
|
12888
12976
|
: undefined;
|
|
12889
|
-
|
|
12890
|
-
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
|
|
12897
|
-
|
|
12898
|
-
!
|
|
12899
|
-
|
|
12900
|
-
|
|
12901
|
-
|
|
12902
|
-
|
|
12903
|
-
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
12908
|
-
|
|
12977
|
+
try {
|
|
12978
|
+
const nativePreparedJoinCommit = canUsePreparedAppendFacts
|
|
12979
|
+
? this._coordinates.createNativeBackbonePreparedJoinCommit(nativeReceiveCoordinateBatch, (batch) => {
|
|
12980
|
+
nativePreparedCoordinateBatch = batch;
|
|
12981
|
+
}, nativeCommitVerifyHashes, nativeCommitVerifyAllHashes, syncProfile, (committedHashes) => {
|
|
12982
|
+
nativePreparedCommittedHashes = new Set(committedHashes);
|
|
12983
|
+
})
|
|
12984
|
+
: undefined;
|
|
12985
|
+
const finishNativePreparedCoordinates = async (properties) => {
|
|
12986
|
+
if (!properties.nativePreparedCommitted ||
|
|
12987
|
+
!nativePreparedCoordinateBatch) {
|
|
12988
|
+
return;
|
|
12989
|
+
}
|
|
12990
|
+
try {
|
|
12991
|
+
nativeBackboneOnlyPersistedHashes =
|
|
12992
|
+
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(nativePreparedCoordinateBatch, syncProfile);
|
|
12993
|
+
nativePreparedCoordinatesFinished = true;
|
|
12994
|
+
}
|
|
12995
|
+
catch (error) {
|
|
12996
|
+
this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(nativePreparedCoordinateBatch);
|
|
12997
|
+
throw error;
|
|
12998
|
+
}
|
|
12999
|
+
};
|
|
13000
|
+
const preparedAppendCanValidateAppend = canAppendAlreadyValidated ||
|
|
13001
|
+
(nativeCommitCanValidateAppend && !!nativePreparedJoinCommit);
|
|
13002
|
+
if (!preparedAppendCanValidateAppend) {
|
|
13003
|
+
canUsePreparedAppendFacts = false;
|
|
12909
13004
|
}
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
|
|
12916
|
-
|
|
12917
|
-
|
|
12918
|
-
? nativeCommitVerifyAllHashes
|
|
12919
|
-
? !!this._nativeBackbone?.graph
|
|
12920
|
-
.commitVerifiedAllPreparedRawReceiveJoinBatch ||
|
|
12921
|
-
!!this._nativeBackbone?.graph
|
|
13005
|
+
const nativePreparedJoinCommitValidatesPlan = !!nativePreparedJoinCommit &&
|
|
13006
|
+
(nativeCommitVerifyHashes && nativeCommitVerifyHashes.length > 0
|
|
13007
|
+
? nativeCommitVerifyAllHashes
|
|
13008
|
+
? !!this._nativeBackbone?.graph
|
|
13009
|
+
.commitVerifiedAllPreparedRawReceiveJoinBatch ||
|
|
13010
|
+
!!this._nativeBackbone?.graph
|
|
13011
|
+
.commitVerifiedPreparedRawReceiveJoinBatch
|
|
13012
|
+
: !!this._nativeBackbone?.graph
|
|
12922
13013
|
.commitVerifiedPreparedRawReceiveJoinBatch
|
|
12923
13014
|
: !!this._nativeBackbone?.graph
|
|
12924
|
-
.
|
|
12925
|
-
|
|
12926
|
-
|
|
12927
|
-
|
|
12928
|
-
|
|
12929
|
-
|
|
12930
|
-
|
|
12931
|
-
|
|
12932
|
-
|
|
12933
|
-
|
|
12934
|
-
|
|
12935
|
-
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
|
|
12939
|
-
|
|
12940
|
-
|
|
12941
|
-
|
|
12942
|
-
|
|
12943
|
-
|
|
12944
|
-
|
|
12945
|
-
|
|
12946
|
-
|
|
12947
|
-
|
|
12948
|
-
|
|
12949
|
-
|
|
12950
|
-
|
|
12951
|
-
|
|
12952
|
-
|
|
12953
|
-
|
|
12954
|
-
|
|
12955
|
-
__peerbitProfile: syncProfile,
|
|
12956
|
-
});
|
|
12957
|
-
}
|
|
12958
|
-
// A recursive lower-log join can resolve successfully while declining
|
|
12959
|
-
// an individual top-level entry (for example, when one of its parents
|
|
12960
|
-
// is temporarily unavailable). The public Log.join() API intentionally
|
|
12961
|
-
// does not expose that per-entry result, so make local index presence the
|
|
12962
|
-
// authority before publishing any SharedLog-side effects. A successful
|
|
12963
|
-
// prepared-facts batch is atomic and already proves every input hash.
|
|
12964
|
-
const admittedHashes = joinedPreparedFacts
|
|
12965
|
-
? new Set(allToMergeHashes)
|
|
12966
|
-
: await this.log.hasMany(allToMergeHashes);
|
|
12967
|
-
admittedMergeHashes = admittedHashes;
|
|
12968
|
-
const admittedShallowEntries = admittedHashes.size === allToMergeShallowEntries.length
|
|
12969
|
-
? allToMergeShallowEntries
|
|
12970
|
-
: allToMergeShallowEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
12971
|
-
if (!joinedPreparedFacts) {
|
|
12972
|
-
reusableCoordinatePersistItems =
|
|
12973
|
-
reusableCoordinatePersistItems.filter((item) => admittedHashes.has(item.entry.hash));
|
|
12974
|
-
coordinatePersistFallbackEntries =
|
|
12975
|
-
coordinatePersistFallbackEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
12976
|
-
}
|
|
12977
|
-
const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
|
|
12978
|
-
if (syncProfile) {
|
|
12979
|
-
emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
|
|
12980
|
-
name: "sharedLog.receive.lowerLogJoin",
|
|
12981
|
-
component: "shared-log",
|
|
12982
|
-
entries: allToMerge.length,
|
|
12983
|
-
messages: 1,
|
|
12984
|
-
details: {
|
|
12985
|
-
hashOnlyEntryAdded,
|
|
12986
|
-
batchHashOnlyEntryAdded,
|
|
12987
|
-
programOnChange,
|
|
12988
|
-
joinedPreparedFacts,
|
|
12989
|
-
admittedEntries: admittedHashes.size,
|
|
12990
|
-
nativePreparedCoordinatesFinished,
|
|
12991
|
-
},
|
|
12992
|
-
});
|
|
12993
|
-
}
|
|
12994
|
-
const coordinatePersistStartedAt = syncProfileStart(syncProfile);
|
|
12995
|
-
if (nativePreparedCoordinatesFinished) {
|
|
12996
|
-
// The lower-log prepared receive transaction already finished
|
|
12997
|
-
// the native coordinate mirror/journal after entry-index commit.
|
|
12998
|
-
}
|
|
12999
|
-
else if (nativePreparedCoordinateBatch) {
|
|
13000
|
-
try {
|
|
13001
|
-
nativeBackboneOnlyPersistedHashes =
|
|
13002
|
-
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(nativePreparedCoordinateBatch, syncProfile);
|
|
13015
|
+
.commitPreparedRawReceiveJoinBatch);
|
|
13016
|
+
const trustedLowerLog = this.log;
|
|
13017
|
+
// With a program-level onChange consumer the hash-only
|
|
13018
|
+
// sink is not used: the lower-log join dispatches the
|
|
13019
|
+
// change event (lazy entry views over the prepared raw
|
|
13020
|
+
// facts) so per-entry consumers observe every commit.
|
|
13021
|
+
const joinOnAppendHashes = programOnChange
|
|
13022
|
+
? undefined
|
|
13023
|
+
: onAppendHashes;
|
|
13024
|
+
const joinedPreparedFacts = canUsePreparedAppendFacts &&
|
|
13025
|
+
(await trustedLowerLog.joinPreparedAppendFactsBatch(preparedAppendFacts, {
|
|
13026
|
+
__peerbitEntriesAlreadyMissing: true,
|
|
13027
|
+
__peerbitCanAppendAlreadyValidated: true,
|
|
13028
|
+
__peerbitDeferIndexWrite: true,
|
|
13029
|
+
__peerbitOnAppendHashes: joinOnAppendHashes,
|
|
13030
|
+
__peerbitProfile: syncProfile,
|
|
13031
|
+
__peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
|
|
13032
|
+
__peerbitNativePreparedJoinCommitValidatesPlan: nativePreparedJoinCommitValidatesPlan,
|
|
13033
|
+
__peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
|
|
13034
|
+
? finishNativePreparedCoordinates
|
|
13035
|
+
: undefined,
|
|
13036
|
+
}));
|
|
13037
|
+
if (!joinedPreparedFacts) {
|
|
13038
|
+
await trustedLowerLog.join(materializeAllToMergeEntries(), {
|
|
13039
|
+
__peerbitBatchIndependent: true,
|
|
13040
|
+
__peerbitEntriesAlreadyMissing: true,
|
|
13041
|
+
__peerbitCanAppendAlreadyValidated: fallbackCanAppendAlreadyValidated,
|
|
13042
|
+
__peerbitDeferIndexWrite: true,
|
|
13043
|
+
__peerbitOnAppendHashes: joinOnAppendHashes,
|
|
13044
|
+
__peerbitProfile: syncProfile,
|
|
13045
|
+
});
|
|
13003
13046
|
}
|
|
13004
|
-
|
|
13005
|
-
|
|
13006
|
-
|
|
13047
|
+
// A recursive lower-log join can resolve successfully while declining
|
|
13048
|
+
// an individual top-level entry (for example, when one of its parents
|
|
13049
|
+
// is temporarily unavailable). The public Log.join() API intentionally
|
|
13050
|
+
// does not expose that per-entry result, so make local index presence the
|
|
13051
|
+
// authority before publishing any SharedLog-side effects. A successful
|
|
13052
|
+
// prepared-facts batch is atomic and already proves every input hash.
|
|
13053
|
+
const admittedHashes = joinedPreparedFacts
|
|
13054
|
+
? new Set(allToMergeHashes)
|
|
13055
|
+
: await this.log.hasMany(allToMergeHashes);
|
|
13056
|
+
admittedMergeHashes = admittedHashes;
|
|
13057
|
+
const admittedShallowEntries = admittedHashes.size === allToMergeShallowEntries.length
|
|
13058
|
+
? allToMergeShallowEntries
|
|
13059
|
+
: allToMergeShallowEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
13060
|
+
if (!joinedPreparedFacts) {
|
|
13061
|
+
reusableCoordinatePersistItems =
|
|
13062
|
+
reusableCoordinatePersistItems.filter((item) => admittedHashes.has(item.entry.hash));
|
|
13063
|
+
coordinatePersistFallbackEntries =
|
|
13064
|
+
coordinatePersistFallbackEntries.filter((entry) => admittedHashes.has(entry.hash));
|
|
13007
13065
|
}
|
|
13008
|
-
|
|
13009
|
-
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
|
|
13015
|
-
|
|
13016
|
-
|
|
13017
|
-
|
|
13066
|
+
const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
|
|
13067
|
+
if (syncProfile) {
|
|
13068
|
+
emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
|
|
13069
|
+
name: "sharedLog.receive.lowerLogJoin",
|
|
13070
|
+
component: "shared-log",
|
|
13071
|
+
entries: allToMerge.length,
|
|
13072
|
+
messages: 1,
|
|
13073
|
+
details: {
|
|
13074
|
+
hashOnlyEntryAdded,
|
|
13075
|
+
batchHashOnlyEntryAdded,
|
|
13076
|
+
programOnChange,
|
|
13077
|
+
joinedPreparedFacts,
|
|
13078
|
+
admittedEntries: admittedHashes.size,
|
|
13079
|
+
nativePreparedCoordinatesFinished,
|
|
13080
|
+
},
|
|
13081
|
+
});
|
|
13082
|
+
}
|
|
13083
|
+
const coordinatePersistStartedAt = syncProfileStart(syncProfile);
|
|
13084
|
+
if (nativePreparedCoordinatesFinished) {
|
|
13085
|
+
// The lower-log prepared receive transaction already finished
|
|
13086
|
+
// the native coordinate mirror/journal after entry-index commit.
|
|
13087
|
+
}
|
|
13088
|
+
else if (nativePreparedCoordinateBatch) {
|
|
13089
|
+
try {
|
|
13090
|
+
nativeBackboneOnlyPersistedHashes =
|
|
13091
|
+
await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(nativePreparedCoordinateBatch, syncProfile);
|
|
13092
|
+
}
|
|
13093
|
+
catch (error) {
|
|
13094
|
+
this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(nativePreparedCoordinateBatch);
|
|
13095
|
+
throw error;
|
|
13018
13096
|
}
|
|
13019
13097
|
}
|
|
13020
|
-
|
|
13021
|
-
|
|
13022
|
-
|
|
13023
|
-
|
|
13024
|
-
|
|
13025
|
-
|
|
13026
|
-
|
|
13027
|
-
|
|
13028
|
-
|
|
13029
|
-
|
|
13030
|
-
})));
|
|
13031
|
-
}
|
|
13032
|
-
if (syncProfile) {
|
|
13033
|
-
emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
|
|
13034
|
-
name: "sharedLog.receive.coordinatePersist",
|
|
13035
|
-
component: "shared-log",
|
|
13036
|
-
entries: entriesToPersist.length,
|
|
13037
|
-
messages: 1,
|
|
13038
|
-
details: {
|
|
13039
|
-
reusedLeaderPlans: reusableCoordinatePersistItemCount,
|
|
13040
|
-
nativeBackboneOnly: nativeBackboneOnlyPersistedHashes?.size ?? 0,
|
|
13041
|
-
},
|
|
13042
|
-
});
|
|
13043
|
-
}
|
|
13044
|
-
for (const hash of admittedHashes) {
|
|
13045
|
-
confirmedHashes.add(hash);
|
|
13046
|
-
}
|
|
13047
|
-
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
13048
|
-
const ownershipChangedDuringReceive = !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
|
|
13049
|
-
if (ownershipChangedDuringReceive) {
|
|
13050
|
-
const freshAuditRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
13051
|
-
const armFreshAuditRetry = () => {
|
|
13052
|
-
for (const entry of admittedShallowEntries) {
|
|
13053
|
-
this.scheduleCheckedPruneRetry({ entry, leaders: new Map() }, receiveOwnershipLifecycleController);
|
|
13098
|
+
else {
|
|
13099
|
+
nativeBackboneOnlyPersistedHashes =
|
|
13100
|
+
await this._coordinates.persistBackboneOnlyReceiveCoordinateBatch(reusableCoordinatePersistItems);
|
|
13101
|
+
}
|
|
13102
|
+
if (nativeBackboneOnlyPersistedHashes &&
|
|
13103
|
+
nativeBackboneOnlyPersistedHashes.size > 0) {
|
|
13104
|
+
for (let i = reusableCoordinatePersistItems.length - 1; i >= 0; i--) {
|
|
13105
|
+
if (nativeBackboneOnlyPersistedHashes.has(reusableCoordinatePersistItems[i].entry.hash)) {
|
|
13106
|
+
reusableCoordinatePersistItems.splice(i, 1);
|
|
13107
|
+
}
|
|
13054
13108
|
}
|
|
13055
|
-
}
|
|
13056
|
-
|
|
13109
|
+
}
|
|
13110
|
+
if (reusableCoordinatePersistItems.length > 0) {
|
|
13111
|
+
await this._coordinates.persistCoordinatesBatch(reusableCoordinatePersistItems);
|
|
13112
|
+
}
|
|
13113
|
+
if (coordinatePersistFallbackEntries.length > 0) {
|
|
13114
|
+
await this.planEntryLeaderBatch(coordinatePersistFallbackEntries.map((entry) => ({
|
|
13115
|
+
entry,
|
|
13116
|
+
replicas: receiveReplicaCounts.get(entry.hash) ??
|
|
13117
|
+
decodeReplicas(entry).getValue(this),
|
|
13118
|
+
options: { roleAge: 0, persist: {} },
|
|
13119
|
+
})));
|
|
13120
|
+
}
|
|
13121
|
+
if (syncProfile) {
|
|
13122
|
+
emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
|
|
13123
|
+
name: "sharedLog.receive.coordinatePersist",
|
|
13124
|
+
component: "shared-log",
|
|
13125
|
+
entries: entriesToPersist.length,
|
|
13126
|
+
messages: 1,
|
|
13127
|
+
details: {
|
|
13128
|
+
reusedLeaderPlans: reusableCoordinatePersistItemCount,
|
|
13129
|
+
nativeBackboneOnly: nativeBackboneOnlyPersistedHashes?.size ?? 0,
|
|
13130
|
+
},
|
|
13131
|
+
});
|
|
13132
|
+
}
|
|
13133
|
+
for (const hash of admittedHashes) {
|
|
13134
|
+
confirmedHashes.add(hash);
|
|
13135
|
+
}
|
|
13136
|
+
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
13137
|
+
const ownershipChangedDuringReceive = !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
|
|
13138
|
+
if (ownershipChangedDuringReceive) {
|
|
13139
|
+
const freshAuditRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
13140
|
+
const armFreshAuditRetry = () => {
|
|
13141
|
+
for (const entry of admittedShallowEntries) {
|
|
13142
|
+
this.scheduleCheckedPruneRetry({ entry, leaders: new Map() }, receiveOwnershipLifecycleController);
|
|
13143
|
+
}
|
|
13144
|
+
};
|
|
13145
|
+
try {
|
|
13146
|
+
await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
|
|
13147
|
+
decodedReplicaCounts: receiveReplicaCounts,
|
|
13148
|
+
freshReceiveOwnerAudit: true,
|
|
13149
|
+
preserveExistingPruneOnLocalResult: true,
|
|
13150
|
+
profile: syncProfile,
|
|
13151
|
+
}, receiveOwnershipLifecycleController);
|
|
13152
|
+
this.throwIfReplicationOwnershipLifecycleInactive(receiveOwnershipLifecycleController);
|
|
13153
|
+
if (!this.isReceiveOwnershipSnapshotStable(freshAuditRevision)) {
|
|
13154
|
+
armFreshAuditRetry();
|
|
13155
|
+
}
|
|
13156
|
+
}
|
|
13157
|
+
catch {
|
|
13158
|
+
// The lower-log and coordinate commits are already durable. A
|
|
13159
|
+
// sender retry will filter these hashes as present, so retain a
|
|
13160
|
+
// bounded local obligation instead of failing the admitted receive.
|
|
13161
|
+
this.throwIfReplicationOwnershipLifecycleInactive(receiveOwnershipLifecycleController);
|
|
13162
|
+
armFreshAuditRetry();
|
|
13163
|
+
}
|
|
13164
|
+
}
|
|
13165
|
+
else {
|
|
13057
13166
|
await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
|
|
13058
13167
|
decodedReplicaCounts: receiveReplicaCounts,
|
|
13059
|
-
freshReceiveOwnerAudit: true,
|
|
13060
13168
|
preserveExistingPruneOnLocalResult: true,
|
|
13169
|
+
reusableLeaderPlans: reusableCoordinatePlans,
|
|
13061
13170
|
profile: syncProfile,
|
|
13062
13171
|
}, receiveOwnershipLifecycleController);
|
|
13063
|
-
this.throwIfReplicationOwnershipLifecycleInactive(receiveOwnershipLifecycleController);
|
|
13064
|
-
if (!this.isReceiveOwnershipSnapshotStable(freshAuditRevision)) {
|
|
13065
|
-
armFreshAuditRetry();
|
|
13066
|
-
}
|
|
13067
13172
|
}
|
|
13068
|
-
|
|
13069
|
-
|
|
13070
|
-
|
|
13071
|
-
|
|
13072
|
-
|
|
13073
|
-
|
|
13173
|
+
if (syncProfile) {
|
|
13174
|
+
emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
|
|
13175
|
+
name: "sharedLog.receive.checkedPrune",
|
|
13176
|
+
component: "shared-log",
|
|
13177
|
+
entries: allToMerge.length,
|
|
13178
|
+
messages: 1,
|
|
13179
|
+
});
|
|
13074
13180
|
}
|
|
13181
|
+
for (const plan of joinPlans) {
|
|
13182
|
+
plan.toDelete
|
|
13183
|
+
?.filter((entry) => admittedMergeHashes.has(entry.hash))
|
|
13184
|
+
.map((entry) => this.pruneDebouncedFnAddIfNotKeeping({
|
|
13185
|
+
key: entry.hash,
|
|
13186
|
+
value: {
|
|
13187
|
+
entry,
|
|
13188
|
+
leaders: plan.leaders,
|
|
13189
|
+
},
|
|
13190
|
+
}));
|
|
13191
|
+
}
|
|
13192
|
+
this.rebalanceParticipationDebounced?.call();
|
|
13075
13193
|
}
|
|
13076
|
-
|
|
13077
|
-
|
|
13078
|
-
|
|
13079
|
-
|
|
13080
|
-
|
|
13081
|
-
|
|
13082
|
-
|
|
13083
|
-
|
|
13084
|
-
|
|
13085
|
-
emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
|
|
13086
|
-
name: "sharedLog.receive.checkedPrune",
|
|
13087
|
-
component: "shared-log",
|
|
13088
|
-
entries: allToMerge.length,
|
|
13089
|
-
messages: 1,
|
|
13090
|
-
});
|
|
13091
|
-
}
|
|
13092
|
-
for (const plan of joinPlans) {
|
|
13093
|
-
plan.toDelete
|
|
13094
|
-
?.filter((entry) => admittedMergeHashes.has(entry.hash))
|
|
13095
|
-
.map((entry) => this.pruneDebouncedFnAddIfNotKeeping({
|
|
13096
|
-
key: entry.hash,
|
|
13097
|
-
value: {
|
|
13098
|
-
entry,
|
|
13099
|
-
leaders: plan.leaders,
|
|
13100
|
-
},
|
|
13101
|
-
}));
|
|
13194
|
+
finally {
|
|
13195
|
+
// Settle seam for the receive token. Every consumer that can
|
|
13196
|
+
// roll it back runs inline before control leaves this block: the
|
|
13197
|
+
// prepared-join callback resolves during the join await, and the
|
|
13198
|
+
// late finish/rollback arm runs above. This `finally` is what
|
|
13199
|
+
// closes the abandon arms (no prepared-join commit, a declined
|
|
13200
|
+
// native commit, a downgrade to the plain join) without having
|
|
13201
|
+
// to enumerate them.
|
|
13202
|
+
this._coordinates.settleResidentCoordinateSnapshot(nativeReceiveCoordinateBatch?.rollbackCoordinateEntries);
|
|
13102
13203
|
}
|
|
13103
|
-
this.rebalanceParticipationDebounced?.call();
|
|
13104
13204
|
}
|
|
13105
13205
|
for (const plan of joinPlans) {
|
|
13106
13206
|
if (!plan.maybeDelete) {
|