@peerbit/shared-log 16.0.1 → 16.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -290,6 +290,7 @@ import {
290
290
  } from "./sync/profile.js";
291
291
  import {
292
292
  ConfirmEntriesMessage,
293
+ RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
293
294
  SYNC_MESSAGE_PRIORITY,
294
295
  SimpleSyncronizer,
295
296
  } from "./sync/simple.js";
@@ -636,6 +637,13 @@ export type NativeBackboneCoordinateRollback<R extends "u32" | "u64"> = {
636
637
  hashes: Set<string>;
637
638
  entries: Map<string, ResidentCoordinateEntry<R>>;
638
639
  generations: Map<string, number>;
640
+ /**
641
+ * Set by `settleResidentCoordinateSnapshot` once the token's holds on the
642
+ * mutation-generation map have been released. Load-bearing: it makes the
643
+ * settle idempotent, so a second settle of this token cannot consume a
644
+ * different token's hold on a shared hash.
645
+ */
646
+ settled?: boolean;
639
647
  };
640
648
 
641
649
  export type RepairDispatchEntry<R extends "u32" | "u64"> =
@@ -1439,6 +1447,16 @@ const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
1439
1447
  ];
1440
1448
  const APPEND_BACKFILL_RETRY_SCHEDULE_MS = [0, 1_000, 3_000, 7_000];
1441
1449
  const RECENT_KNOWN_REPAIR_SUPPRESSION_MS = 30_000;
1450
+ // `_entryKnownPeerObservedAt` is read ONLY through isEntryRecentlyKnownByPeer,
1451
+ // which treats an over-age row and an absent row identically (both false). So
1452
+ // rows older than the longest horizon any caller asks about are dead weight,
1453
+ // and dropping them is behaviour-identical rather than merely safe. Derived
1454
+ // from the horizons themselves -- never hardcode it -- so a future caller with
1455
+ // a longer window cannot silently outlive the retention that serves it.
1456
+ const ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS = Math.max(
1457
+ RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
1458
+ RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
1459
+ );
1442
1460
  const JOIN_AUTHORITATIVE_REPAIR_DELAY_MS = 2_000;
1443
1461
  const JOIN_AUTHORITATIVE_REPAIR_SWEEP_DELAYS_MS = [
1444
1462
  JOIN_AUTHORITATIVE_REPAIR_DELAY_MS,
@@ -3103,9 +3121,23 @@ export class SharedLog<
3103
3121
  };
3104
3122
  for (const coordinate of intent.coordinates) {
3105
3123
  rollback.hashes.add(coordinate.hash);
3106
- const generation =
3107
- (mutationGenerations.get(coordinate.hash) ?? 0) + 1;
3108
- mutationGenerations.set(coordinate.hash, generation);
3124
+ // Same hold-counted row shape the coordinator's own
3125
+ // snapshot writes: one hold per hash, released by the
3126
+ // settle after the replay consumes the token below.
3127
+ // RELIES ON `intent.coordinates` HOLDING UNIQUE HASHES —
3128
+ // it is built from the token's `hashes` Set (see
3129
+ // setNativeStrictDurableTransactionOperation). Holds are
3130
+ // taken per element here but released per unique hash by
3131
+ // the settle, so a duplicate would take two and release
3132
+ // one and retain that row forever. That is the safe
3133
+ // direction (a retained row, never a fail-open rollback),
3134
+ // but keep the source a Set.
3135
+ const row = mutationGenerations.get(coordinate.hash);
3136
+ const generation = (row?.generation ?? 0) + 1;
3137
+ mutationGenerations.set(coordinate.hash, {
3138
+ generation,
3139
+ holds: (row?.holds ?? 0) + 1,
3140
+ });
3109
3141
  rollback.generations.set(coordinate.hash, generation);
3110
3142
  if (coordinate.value) {
3111
3143
  const number = (value: string) =>
@@ -3131,6 +3163,9 @@ export class SharedLog<
3131
3163
  "",
3132
3164
  rollback,
3133
3165
  );
3166
+ // The replay fabricated these generations itself one turn
3167
+ // earlier and has now consumed them, so the rows are dead.
3168
+ this._coordinates.settleResidentCoordinateSnapshot(rollback);
3134
3169
  }
3135
3170
  for (const document of intent.documents) {
3136
3171
  this.restoreNativeBackboneDocument({
@@ -3294,6 +3329,7 @@ export class SharedLog<
3294
3329
  private _repairSweepOptimisticGidsByPeer!: Map<string, Set<string>>;
3295
3330
  private _entryKnownPeers!: Map<string, Set<string>>;
3296
3331
  private _entryKnownPeerObservedAt!: Map<string, Map<string, number>>;
3332
+ private _entryKnownPeerObservedAtSweptAt = 0;
3297
3333
  private _joinAuthoritativeRepairTimersByDelay!: Map<
3298
3334
  number,
3299
3335
  ReturnType<typeof setTimeout>
@@ -3644,6 +3680,7 @@ export class SharedLog<
3644
3680
  this._repairSweepOptimisticGidsByPeer = new Map();
3645
3681
  this._entryKnownPeers = new Map();
3646
3682
  this._entryKnownPeerObservedAt = new Map();
3683
+ this._entryKnownPeerObservedAtSweptAt = 0;
3647
3684
  this._joinAuthoritativeRepairTimersByDelay = new Map();
3648
3685
  this._joinAuthoritativeRepairPeersByDelay = new Map();
3649
3686
  this._appendBackfillPendingByTarget = new Map();
@@ -7649,6 +7686,16 @@ export class SharedLog<
7649
7686
  this._nativeSharedLogState?.markEntriesKnownByPeer(hashArray, peer);
7650
7687
  this._nativeBackbone?.markEntriesKnownByPeer(hashArray, peer);
7651
7688
  const now = Date.now();
7689
+ // Growth is driven by writes, so the sweep rides the write path rather
7690
+ // than a timer or the rebalance pass: cost stays proportional to the
7691
+ // traffic that creates rows. Rate-limited to one pass per retention
7692
+ // window, over a map that after the first pass holds one window of marks.
7693
+ if (
7694
+ now - this._entryKnownPeerObservedAtSweptAt >=
7695
+ ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS
7696
+ ) {
7697
+ this.sweepEntryKnownPeerObservedAt(now);
7698
+ }
7652
7699
  for (const hash of hashArray) {
7653
7700
  let peers = this._entryKnownPeers.get(hash);
7654
7701
  if (!peers) {
@@ -7718,6 +7765,28 @@ export class SharedLog<
7718
7765
  return observedAt != null && Date.now() - observedAt <= maxAgeMs;
7719
7766
  }
7720
7767
 
7768
+ /** Drop recency marks no reader can still act on.
7769
+ *
7770
+ * Touches ONLY `_entryKnownPeerObservedAt`. `_entryKnownPeers` carries
7771
+ * membership, not recency, and its rows stay until the peer dimension
7772
+ * clears them; the native mirrors have no recency dimension at all
7773
+ * (mark/remove/removePeer only), so this must not call into them or the
7774
+ * two sides would disagree.
7775
+ */
7776
+ private sweepEntryKnownPeerObservedAt(now: number) {
7777
+ for (const [hash, observedAt] of this._entryKnownPeerObservedAt) {
7778
+ for (const [peer, timestamp] of observedAt) {
7779
+ if (now - timestamp > ENTRY_KNOWN_PEER_OBSERVED_AT_RETENTION_MS) {
7780
+ observedAt.delete(peer);
7781
+ }
7782
+ }
7783
+ if (observedAt.size === 0) {
7784
+ this._entryKnownPeerObservedAt.delete(hash);
7785
+ }
7786
+ }
7787
+ this._entryKnownPeerObservedAtSweptAt = now;
7788
+ }
7789
+
7721
7790
  private markRepairSweepOptimisticPeer(
7722
7791
  gid: string,
7723
7792
  peer: string,
@@ -10934,12 +11003,21 @@ export class SharedLog<
10934
11003
  return rollbackLowerPublication(error);
10935
11004
  }
10936
11005
  if (!result) {
11006
+ // Abandon arm: the token was minted but nothing downstream can
11007
+ // roll it back from here.
11008
+ this._coordinates.settleResidentCoordinateSnapshot(
11009
+ lowerPublicationRollback?.coordinateEntries,
11010
+ );
10937
11011
  return this.completeNativeStrictDurableTransaction(
10938
11012
  nativeStrictTransaction,
10939
11013
  ).then(() => undefined);
10940
11014
  }
10941
11015
  return mapMaybePromise(result, async (prepared) => {
10942
11016
  if (!prepared) {
11017
+ // Abandon arm: same shape as the `!result` arm above.
11018
+ this._coordinates.settleResidentCoordinateSnapshot(
11019
+ lowerPublicationRollback?.coordinateEntries,
11020
+ );
10943
11021
  await this.completeNativeStrictDurableTransaction(
10944
11022
  nativeStrictTransaction,
10945
11023
  );
@@ -10973,6 +11051,11 @@ export class SharedLog<
10973
11051
  prepared.appendFacts.hash,
10974
11052
  lowerPublicationRollback?.coordinateEntries,
10975
11053
  );
11054
+ // Terminal: this is the last rollback consumer for the
11055
+ // token. A throw above leaves the row, which is safe.
11056
+ this._coordinates.settleResidentCoordinateSnapshot(
11057
+ lowerPublicationRollback?.coordinateEntries,
11058
+ );
10976
11059
  for (const document of lowerPublicationRollback?.documents ?? []) {
10977
11060
  this.restoreNativeBackboneDocument(document);
10978
11061
  }
@@ -11043,6 +11126,14 @@ export class SharedLog<
11043
11126
  } catch (error) {
11044
11127
  return rollback(error);
11045
11128
  }
11129
+ // Success seam. The last await inside the protected try is the
11130
+ // finalizer acknowledge; `finish()` is synchronous, so no async
11131
+ // boundary separates the catch above from this statement and
11132
+ // `rollback` can no longer fire. Nothing downstream rolls back
11133
+ // (the retire below only warns), so the token is terminal here.
11134
+ this._coordinates.settleResidentCoordinateSnapshot(
11135
+ lowerPublicationRollback?.coordinateEntries,
11136
+ );
11046
11137
  this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
11047
11138
  prepared.appendFacts,
11048
11139
  prepared.removed,
@@ -11499,6 +11590,12 @@ export class SharedLog<
11499
11590
  }
11500
11591
  return mapMaybePromise(result, async (prepared) => {
11501
11592
  if (!prepared || !backboneAppend) {
11593
+ // Abandon arm: the token was minted inside
11594
+ // `prepareBackboneAppend` and nothing downstream of
11595
+ // this return can roll it back.
11596
+ this._coordinates.settleResidentCoordinateSnapshot(
11597
+ nativeCoordinateRollback,
11598
+ );
11502
11599
  await this.completeNativeStrictDurableTransaction(
11503
11600
  nativeStrictTransaction,
11504
11601
  );
@@ -11590,6 +11687,10 @@ export class SharedLog<
11590
11687
  prepared.appendFacts.hash,
11591
11688
  rollbackCoordinateEntries,
11592
11689
  );
11690
+ // Terminal: last rollback consumer for the token.
11691
+ this._coordinates.settleResidentCoordinateSnapshot(
11692
+ rollbackCoordinateEntries,
11693
+ );
11593
11694
  } catch (rollbackError) {
11594
11695
  rollbackFailures.push(rollbackError);
11595
11696
  }
@@ -11666,6 +11767,12 @@ export class SharedLog<
11666
11767
  } catch (error) {
11667
11768
  return rollback(error);
11668
11769
  }
11770
+ // Success seam: the finalizer acknowledge above is the last
11771
+ // await inside the protected try, so `rollback` can no
11772
+ // longer fire and the retire below only warns.
11773
+ this._coordinates.settleResidentCoordinateSnapshot(
11774
+ rollbackCoordinateEntries,
11775
+ );
11669
11776
  this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
11670
11777
  prepared.appendFacts,
11671
11778
  prepared.removed,
@@ -12585,6 +12692,10 @@ export class SharedLog<
12585
12692
  throw error;
12586
12693
  }
12587
12694
  if (!appended || !backboneAppends) {
12695
+ // Abandon arm: no consumer downstream of this return.
12696
+ this._coordinates.settleResidentCoordinateSnapshot(
12697
+ batchCoordinateRollback,
12698
+ );
12588
12699
  await this.completeNativeStrictDurableTransaction(
12589
12700
  nativeStrictTransaction,
12590
12701
  );
@@ -12622,6 +12733,10 @@ export class SharedLog<
12622
12733
  appended.appendFacts[0]?.hash ?? "",
12623
12734
  batchCoordinateRollback,
12624
12735
  );
12736
+ // Terminal: last rollback consumer for the batch token.
12737
+ this._coordinates.settleResidentCoordinateSnapshot(
12738
+ batchCoordinateRollback,
12739
+ );
12625
12740
  } catch (rollbackError) {
12626
12741
  rollbackFailures.push(rollbackError);
12627
12742
  }
@@ -12753,6 +12868,11 @@ export class SharedLog<
12753
12868
  } catch (error) {
12754
12869
  return rollbackBatch(error);
12755
12870
  }
12871
+ // Success seam: `rollbackBatch` has exactly one call site (the catch
12872
+ // above), and everything from here on escapes without any rollback.
12873
+ this._coordinates.settleResidentCoordinateSnapshot(
12874
+ batchCoordinateRollback,
12875
+ );
12756
12876
 
12757
12877
  this.throwIfReplicationOwnershipLifecycleInactive(
12758
12878
  ownershipLifecycleController,
@@ -14213,6 +14333,7 @@ export class SharedLog<
14213
14333
  this._repairSweepOptimisticGidsByPeer = new Map();
14214
14334
  this._entryKnownPeers = new Map();
14215
14335
  this._entryKnownPeerObservedAt = new Map();
14336
+ this._entryKnownPeerObservedAtSweptAt = 0;
14216
14337
  this._joinAuthoritativeRepairTimersByDelay = new Map();
14217
14338
  this._joinAuthoritativeRepairPeersByDelay = new Map();
14218
14339
  this._assumeSyncedRepairSuppressedUntil = 0;
@@ -18861,292 +18982,305 @@ export class SharedLog<
18861
18982
  reusableCoordinatePersistItems,
18862
18983
  )
18863
18984
  : undefined;
18864
- const nativePreparedJoinCommit = canUsePreparedAppendFacts
18865
- ? this._coordinates.createNativeBackbonePreparedJoinCommit(
18866
- nativeReceiveCoordinateBatch,
18867
- (batch) => {
18868
- nativePreparedCoordinateBatch = batch;
18869
- },
18870
- nativeCommitVerifyHashes,
18871
- nativeCommitVerifyAllHashes,
18872
- syncProfile,
18873
- (committedHashes) => {
18874
- nativePreparedCommittedHashes = new Set(committedHashes);
18875
- },
18876
- )
18877
- : undefined;
18878
- const finishNativePreparedCoordinates = async (properties: {
18879
- nativePreparedCommitted: boolean;
18880
- }) => {
18881
- if (
18882
- !properties.nativePreparedCommitted ||
18883
- !nativePreparedCoordinateBatch
18884
- ) {
18885
- return;
18886
- }
18887
- try {
18888
- nativeBackboneOnlyPersistedHashes =
18889
- await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
18890
- nativePreparedCoordinateBatch,
18985
+ try {
18986
+ const nativePreparedJoinCommit = canUsePreparedAppendFacts
18987
+ ? this._coordinates.createNativeBackbonePreparedJoinCommit(
18988
+ nativeReceiveCoordinateBatch,
18989
+ (batch) => {
18990
+ nativePreparedCoordinateBatch = batch;
18991
+ },
18992
+ nativeCommitVerifyHashes,
18993
+ nativeCommitVerifyAllHashes,
18891
18994
  syncProfile,
18995
+ (committedHashes) => {
18996
+ nativePreparedCommittedHashes = new Set(committedHashes);
18997
+ },
18998
+ )
18999
+ : undefined;
19000
+ const finishNativePreparedCoordinates = async (properties: {
19001
+ nativePreparedCommitted: boolean;
19002
+ }) => {
19003
+ if (
19004
+ !properties.nativePreparedCommitted ||
19005
+ !nativePreparedCoordinateBatch
19006
+ ) {
19007
+ return;
19008
+ }
19009
+ try {
19010
+ nativeBackboneOnlyPersistedHashes =
19011
+ await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
19012
+ nativePreparedCoordinateBatch,
19013
+ syncProfile,
19014
+ );
19015
+ nativePreparedCoordinatesFinished = true;
19016
+ } catch (error) {
19017
+ this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
19018
+ nativePreparedCoordinateBatch,
18892
19019
  );
18893
- nativePreparedCoordinatesFinished = true;
18894
- } catch (error) {
18895
- this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
18896
- nativePreparedCoordinateBatch,
18897
- );
18898
- throw error;
19020
+ throw error;
19021
+ }
19022
+ };
19023
+ const preparedAppendCanValidateAppend =
19024
+ canAppendAlreadyValidated ||
19025
+ (nativeCommitCanValidateAppend && !!nativePreparedJoinCommit);
19026
+ if (!preparedAppendCanValidateAppend) {
19027
+ canUsePreparedAppendFacts = false;
18899
19028
  }
18900
- };
18901
- const preparedAppendCanValidateAppend =
18902
- canAppendAlreadyValidated ||
18903
- (nativeCommitCanValidateAppend && !!nativePreparedJoinCommit);
18904
- if (!preparedAppendCanValidateAppend) {
18905
- canUsePreparedAppendFacts = false;
18906
- }
18907
- const nativePreparedJoinCommitValidatesPlan =
18908
- !!nativePreparedJoinCommit &&
18909
- (nativeCommitVerifyHashes && nativeCommitVerifyHashes.length > 0
18910
- ? nativeCommitVerifyAllHashes
18911
- ? !!this._nativeBackbone?.graph
18912
- .commitVerifiedAllPreparedRawReceiveJoinBatch ||
18913
- !!this._nativeBackbone?.graph
18914
- .commitVerifiedPreparedRawReceiveJoinBatch
19029
+ const nativePreparedJoinCommitValidatesPlan =
19030
+ !!nativePreparedJoinCommit &&
19031
+ (nativeCommitVerifyHashes && nativeCommitVerifyHashes.length > 0
19032
+ ? nativeCommitVerifyAllHashes
19033
+ ? !!this._nativeBackbone?.graph
19034
+ .commitVerifiedAllPreparedRawReceiveJoinBatch ||
19035
+ !!this._nativeBackbone?.graph
19036
+ .commitVerifiedPreparedRawReceiveJoinBatch
19037
+ : !!this._nativeBackbone?.graph
19038
+ .commitVerifiedPreparedRawReceiveJoinBatch
18915
19039
  : !!this._nativeBackbone?.graph
18916
- .commitVerifiedPreparedRawReceiveJoinBatch
18917
- : !!this._nativeBackbone?.graph
18918
- .commitPreparedRawReceiveJoinBatch);
18919
- const trustedLowerLog = this.log as unknown as TrustedLowerLog<T>;
18920
- // With a program-level onChange consumer the hash-only
18921
- // sink is not used: the lower-log join dispatches the
18922
- // change event (lazy entry views over the prepared raw
18923
- // facts) so per-entry consumers observe every commit.
18924
- const joinOnAppendHashes = programOnChange
18925
- ? undefined
18926
- : onAppendHashes;
18927
- const joinedPreparedFacts =
18928
- canUsePreparedAppendFacts &&
18929
- (await trustedLowerLog.joinPreparedAppendFactsBatch(
18930
- preparedAppendFacts,
18931
- {
19040
+ .commitPreparedRawReceiveJoinBatch);
19041
+ const trustedLowerLog = this.log as unknown as TrustedLowerLog<T>;
19042
+ // With a program-level onChange consumer the hash-only
19043
+ // sink is not used: the lower-log join dispatches the
19044
+ // change event (lazy entry views over the prepared raw
19045
+ // facts) so per-entry consumers observe every commit.
19046
+ const joinOnAppendHashes = programOnChange
19047
+ ? undefined
19048
+ : onAppendHashes;
19049
+ const joinedPreparedFacts =
19050
+ canUsePreparedAppendFacts &&
19051
+ (await trustedLowerLog.joinPreparedAppendFactsBatch(
19052
+ preparedAppendFacts,
19053
+ {
19054
+ __peerbitEntriesAlreadyMissing: true,
19055
+ __peerbitCanAppendAlreadyValidated: true,
19056
+ __peerbitDeferIndexWrite: true,
19057
+ __peerbitOnAppendHashes: joinOnAppendHashes,
19058
+ __peerbitProfile: syncProfile,
19059
+ __peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
19060
+ __peerbitNativePreparedJoinCommitValidatesPlan:
19061
+ nativePreparedJoinCommitValidatesPlan,
19062
+ __peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
19063
+ ? finishNativePreparedCoordinates
19064
+ : undefined,
19065
+ },
19066
+ ));
19067
+ if (!joinedPreparedFacts) {
19068
+ await trustedLowerLog.join(materializeAllToMergeEntries(), {
19069
+ __peerbitBatchIndependent: true,
18932
19070
  __peerbitEntriesAlreadyMissing: true,
18933
- __peerbitCanAppendAlreadyValidated: true,
19071
+ __peerbitCanAppendAlreadyValidated:
19072
+ fallbackCanAppendAlreadyValidated,
18934
19073
  __peerbitDeferIndexWrite: true,
18935
19074
  __peerbitOnAppendHashes: joinOnAppendHashes,
18936
19075
  __peerbitProfile: syncProfile,
18937
- __peerbitNativePreparedJoinCommit: nativePreparedJoinCommit,
18938
- __peerbitNativePreparedJoinCommitValidatesPlan:
18939
- nativePreparedJoinCommitValidatesPlan,
18940
- __peerbitOnPreparedJoinCommitted: nativePreparedJoinCommit
18941
- ? finishNativePreparedCoordinates
18942
- : undefined,
18943
- },
18944
- ));
18945
- if (!joinedPreparedFacts) {
18946
- await trustedLowerLog.join(materializeAllToMergeEntries(), {
18947
- __peerbitBatchIndependent: true,
18948
- __peerbitEntriesAlreadyMissing: true,
18949
- __peerbitCanAppendAlreadyValidated:
18950
- fallbackCanAppendAlreadyValidated,
18951
- __peerbitDeferIndexWrite: true,
18952
- __peerbitOnAppendHashes: joinOnAppendHashes,
18953
- __peerbitProfile: syncProfile,
18954
- });
18955
- }
18956
- // A recursive lower-log join can resolve successfully while declining
18957
- // an individual top-level entry (for example, when one of its parents
18958
- // is temporarily unavailable). The public Log.join() API intentionally
18959
- // does not expose that per-entry result, so make local index presence the
18960
- // authority before publishing any SharedLog-side effects. A successful
18961
- // prepared-facts batch is atomic and already proves every input hash.
18962
- const admittedHashes = joinedPreparedFacts
18963
- ? new Set(allToMergeHashes)
18964
- : await this.log.hasMany(allToMergeHashes);
18965
- admittedMergeHashes = admittedHashes;
18966
- const admittedShallowEntries =
18967
- admittedHashes.size === allToMergeShallowEntries.length
18968
- ? allToMergeShallowEntries
18969
- : allToMergeShallowEntries.filter((entry) =>
19076
+ });
19077
+ }
19078
+ // A recursive lower-log join can resolve successfully while declining
19079
+ // an individual top-level entry (for example, when one of its parents
19080
+ // is temporarily unavailable). The public Log.join() API intentionally
19081
+ // does not expose that per-entry result, so make local index presence the
19082
+ // authority before publishing any SharedLog-side effects. A successful
19083
+ // prepared-facts batch is atomic and already proves every input hash.
19084
+ const admittedHashes = joinedPreparedFacts
19085
+ ? new Set(allToMergeHashes)
19086
+ : await this.log.hasMany(allToMergeHashes);
19087
+ admittedMergeHashes = admittedHashes;
19088
+ const admittedShallowEntries =
19089
+ admittedHashes.size === allToMergeShallowEntries.length
19090
+ ? allToMergeShallowEntries
19091
+ : allToMergeShallowEntries.filter((entry) =>
19092
+ admittedHashes.has(entry.hash),
19093
+ );
19094
+ if (!joinedPreparedFacts) {
19095
+ reusableCoordinatePersistItems =
19096
+ reusableCoordinatePersistItems.filter((item) =>
19097
+ admittedHashes.has(item.entry.hash),
19098
+ );
19099
+ coordinatePersistFallbackEntries =
19100
+ coordinatePersistFallbackEntries.filter((entry) =>
18970
19101
  admittedHashes.has(entry.hash),
18971
19102
  );
18972
- if (!joinedPreparedFacts) {
18973
- reusableCoordinatePersistItems =
18974
- reusableCoordinatePersistItems.filter((item) =>
18975
- admittedHashes.has(item.entry.hash),
18976
- );
18977
- coordinatePersistFallbackEntries =
18978
- coordinatePersistFallbackEntries.filter((entry) =>
18979
- admittedHashes.has(entry.hash),
18980
- );
18981
- }
18982
- const reusableCoordinatePersistItemCount =
18983
- reusableCoordinatePersistItems.length;
18984
- if (syncProfile) {
18985
- emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
18986
- name: "sharedLog.receive.lowerLogJoin",
18987
- component: "shared-log",
18988
- entries: allToMerge.length,
18989
- messages: 1,
18990
- details: {
18991
- hashOnlyEntryAdded,
18992
- batchHashOnlyEntryAdded,
18993
- programOnChange,
18994
- joinedPreparedFacts,
18995
- admittedEntries: admittedHashes.size,
18996
- nativePreparedCoordinatesFinished,
18997
- },
18998
- });
18999
- }
19000
- const coordinatePersistStartedAt = syncProfileStart(syncProfile);
19001
- if (nativePreparedCoordinatesFinished) {
19002
- // The lower-log prepared receive transaction already finished
19003
- // the native coordinate mirror/journal after entry-index commit.
19004
- } else if (nativePreparedCoordinateBatch) {
19005
- try {
19006
- nativeBackboneOnlyPersistedHashes =
19007
- await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
19103
+ }
19104
+ const reusableCoordinatePersistItemCount =
19105
+ reusableCoordinatePersistItems.length;
19106
+ if (syncProfile) {
19107
+ emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
19108
+ name: "sharedLog.receive.lowerLogJoin",
19109
+ component: "shared-log",
19110
+ entries: allToMerge.length,
19111
+ messages: 1,
19112
+ details: {
19113
+ hashOnlyEntryAdded,
19114
+ batchHashOnlyEntryAdded,
19115
+ programOnChange,
19116
+ joinedPreparedFacts,
19117
+ admittedEntries: admittedHashes.size,
19118
+ nativePreparedCoordinatesFinished,
19119
+ },
19120
+ });
19121
+ }
19122
+ const coordinatePersistStartedAt = syncProfileStart(syncProfile);
19123
+ if (nativePreparedCoordinatesFinished) {
19124
+ // The lower-log prepared receive transaction already finished
19125
+ // the native coordinate mirror/journal after entry-index commit.
19126
+ } else if (nativePreparedCoordinateBatch) {
19127
+ try {
19128
+ nativeBackboneOnlyPersistedHashes =
19129
+ await this._coordinates.finishBackboneOnlyReceiveCoordinateBatch(
19130
+ nativePreparedCoordinateBatch,
19131
+ syncProfile,
19132
+ );
19133
+ } catch (error) {
19134
+ this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
19008
19135
  nativePreparedCoordinateBatch,
19009
- syncProfile,
19010
19136
  );
19011
- } catch (error) {
19012
- this._coordinates.rollbackBackboneOnlyReceiveCoordinateBatch(
19013
- nativePreparedCoordinateBatch,
19014
- );
19015
- throw error;
19137
+ throw error;
19138
+ }
19139
+ } else {
19140
+ nativeBackboneOnlyPersistedHashes =
19141
+ await this._coordinates.persistBackboneOnlyReceiveCoordinateBatch(
19142
+ reusableCoordinatePersistItems,
19143
+ );
19016
19144
  }
19017
- } else {
19018
- nativeBackboneOnlyPersistedHashes =
19019
- await this._coordinates.persistBackboneOnlyReceiveCoordinateBatch(
19020
- reusableCoordinatePersistItems,
19021
- );
19022
- }
19023
- if (
19024
- nativeBackboneOnlyPersistedHashes &&
19025
- nativeBackboneOnlyPersistedHashes.size > 0
19026
- ) {
19027
- for (
19028
- let i = reusableCoordinatePersistItems.length - 1;
19029
- i >= 0;
19030
- i--
19145
+ if (
19146
+ nativeBackboneOnlyPersistedHashes &&
19147
+ nativeBackboneOnlyPersistedHashes.size > 0
19031
19148
  ) {
19032
- if (
19033
- nativeBackboneOnlyPersistedHashes.has(
19034
- reusableCoordinatePersistItems[i]!.entry.hash,
19035
- )
19149
+ for (
19150
+ let i = reusableCoordinatePersistItems.length - 1;
19151
+ i >= 0;
19152
+ i--
19036
19153
  ) {
19037
- reusableCoordinatePersistItems.splice(i, 1);
19154
+ if (
19155
+ nativeBackboneOnlyPersistedHashes.has(
19156
+ reusableCoordinatePersistItems[i]!.entry.hash,
19157
+ )
19158
+ ) {
19159
+ reusableCoordinatePersistItems.splice(i, 1);
19160
+ }
19038
19161
  }
19039
19162
  }
19040
- }
19041
- if (reusableCoordinatePersistItems.length > 0) {
19042
- await this._coordinates.persistCoordinatesBatch(
19043
- reusableCoordinatePersistItems,
19044
- );
19045
- }
19046
- if (coordinatePersistFallbackEntries.length > 0) {
19047
- await this.planEntryLeaderBatch(
19048
- coordinatePersistFallbackEntries.map((entry) => ({
19049
- entry,
19050
- replicas:
19051
- receiveReplicaCounts.get(entry.hash) ??
19052
- decodeReplicas(entry).getValue(this),
19053
- options: { roleAge: 0, persist: {} },
19054
- })),
19055
- );
19056
- }
19057
- if (syncProfile) {
19058
- emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
19059
- name: "sharedLog.receive.coordinatePersist",
19060
- component: "shared-log",
19061
- entries: entriesToPersist.length,
19062
- messages: 1,
19063
- details: {
19064
- reusedLeaderPlans: reusableCoordinatePersistItemCount,
19065
- nativeBackboneOnly:
19066
- nativeBackboneOnlyPersistedHashes?.size ?? 0,
19067
- },
19068
- });
19069
- }
19070
- for (const hash of admittedHashes) {
19071
- confirmedHashes.add(hash);
19072
- }
19073
- const checkedPruneStartedAt = syncProfileStart(syncProfile);
19074
- const ownershipChangedDuringReceive =
19075
- !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
19076
- if (ownershipChangedDuringReceive) {
19077
- const freshAuditRevision =
19078
- this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
19079
- const armFreshAuditRetry = () => {
19080
- for (const entry of admittedShallowEntries) {
19081
- this.scheduleCheckedPruneRetry(
19082
- { entry, leaders: new Map() },
19163
+ if (reusableCoordinatePersistItems.length > 0) {
19164
+ await this._coordinates.persistCoordinatesBatch(
19165
+ reusableCoordinatePersistItems,
19166
+ );
19167
+ }
19168
+ if (coordinatePersistFallbackEntries.length > 0) {
19169
+ await this.planEntryLeaderBatch(
19170
+ coordinatePersistFallbackEntries.map((entry) => ({
19171
+ entry,
19172
+ replicas:
19173
+ receiveReplicaCounts.get(entry.hash) ??
19174
+ decodeReplicas(entry).getValue(this),
19175
+ options: { roleAge: 0, persist: {} },
19176
+ })),
19177
+ );
19178
+ }
19179
+ if (syncProfile) {
19180
+ emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
19181
+ name: "sharedLog.receive.coordinatePersist",
19182
+ component: "shared-log",
19183
+ entries: entriesToPersist.length,
19184
+ messages: 1,
19185
+ details: {
19186
+ reusedLeaderPlans: reusableCoordinatePersistItemCount,
19187
+ nativeBackboneOnly:
19188
+ nativeBackboneOnlyPersistedHashes?.size ?? 0,
19189
+ },
19190
+ });
19191
+ }
19192
+ for (const hash of admittedHashes) {
19193
+ confirmedHashes.add(hash);
19194
+ }
19195
+ const checkedPruneStartedAt = syncProfileStart(syncProfile);
19196
+ const ownershipChangedDuringReceive =
19197
+ !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
19198
+ if (ownershipChangedDuringReceive) {
19199
+ const freshAuditRevision =
19200
+ this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
19201
+ const armFreshAuditRetry = () => {
19202
+ for (const entry of admittedShallowEntries) {
19203
+ this.scheduleCheckedPruneRetry(
19204
+ { entry, leaders: new Map() },
19205
+ receiveOwnershipLifecycleController,
19206
+ );
19207
+ }
19208
+ };
19209
+ try {
19210
+ await this.pruneJoinedEntriesNoLongerLed(
19211
+ admittedShallowEntries,
19212
+ {
19213
+ decodedReplicaCounts: receiveReplicaCounts,
19214
+ freshReceiveOwnerAudit: true,
19215
+ preserveExistingPruneOnLocalResult: true,
19216
+ profile: syncProfile,
19217
+ },
19083
19218
  receiveOwnershipLifecycleController,
19084
19219
  );
19220
+ this.throwIfReplicationOwnershipLifecycleInactive(
19221
+ receiveOwnershipLifecycleController,
19222
+ );
19223
+ if (
19224
+ !this.isReceiveOwnershipSnapshotStable(freshAuditRevision)
19225
+ ) {
19226
+ armFreshAuditRetry();
19227
+ }
19228
+ } catch {
19229
+ // The lower-log and coordinate commits are already durable. A
19230
+ // sender retry will filter these hashes as present, so retain a
19231
+ // bounded local obligation instead of failing the admitted receive.
19232
+ this.throwIfReplicationOwnershipLifecycleInactive(
19233
+ receiveOwnershipLifecycleController,
19234
+ );
19235
+ armFreshAuditRetry();
19085
19236
  }
19086
- };
19087
- try {
19237
+ } else {
19088
19238
  await this.pruneJoinedEntriesNoLongerLed(
19089
19239
  admittedShallowEntries,
19090
19240
  {
19091
19241
  decodedReplicaCounts: receiveReplicaCounts,
19092
- freshReceiveOwnerAudit: true,
19093
19242
  preserveExistingPruneOnLocalResult: true,
19243
+ reusableLeaderPlans: reusableCoordinatePlans,
19094
19244
  profile: syncProfile,
19095
19245
  },
19096
19246
  receiveOwnershipLifecycleController,
19097
19247
  );
19098
- this.throwIfReplicationOwnershipLifecycleInactive(
19099
- receiveOwnershipLifecycleController,
19100
- );
19101
- if (
19102
- !this.isReceiveOwnershipSnapshotStable(freshAuditRevision)
19103
- ) {
19104
- armFreshAuditRetry();
19105
- }
19106
- } catch {
19107
- // The lower-log and coordinate commits are already durable. A
19108
- // sender retry will filter these hashes as present, so retain a
19109
- // bounded local obligation instead of failing the admitted receive.
19110
- this.throwIfReplicationOwnershipLifecycleInactive(
19111
- receiveOwnershipLifecycleController,
19112
- );
19113
- armFreshAuditRetry();
19114
19248
  }
19115
- } else {
19116
- await this.pruneJoinedEntriesNoLongerLed(
19117
- admittedShallowEntries,
19118
- {
19119
- decodedReplicaCounts: receiveReplicaCounts,
19120
- preserveExistingPruneOnLocalResult: true,
19121
- reusableLeaderPlans: reusableCoordinatePlans,
19122
- profile: syncProfile,
19123
- },
19124
- receiveOwnershipLifecycleController,
19125
- );
19126
- }
19127
- if (syncProfile) {
19128
- emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
19129
- name: "sharedLog.receive.checkedPrune",
19130
- component: "shared-log",
19131
- entries: allToMerge.length,
19132
- messages: 1,
19133
- });
19134
- }
19249
+ if (syncProfile) {
19250
+ emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
19251
+ name: "sharedLog.receive.checkedPrune",
19252
+ component: "shared-log",
19253
+ entries: allToMerge.length,
19254
+ messages: 1,
19255
+ });
19256
+ }
19135
19257
 
19136
- for (const plan of joinPlans) {
19137
- plan.toDelete
19138
- ?.filter((entry) => admittedMergeHashes.has(entry.hash))
19139
- .map((entry) =>
19140
- this.pruneDebouncedFnAddIfNotKeeping({
19141
- key: entry.hash,
19142
- value: {
19143
- entry,
19144
- leaders: plan.leaders as Map<string, any>,
19145
- },
19146
- }),
19147
- );
19258
+ for (const plan of joinPlans) {
19259
+ plan.toDelete
19260
+ ?.filter((entry) => admittedMergeHashes.has(entry.hash))
19261
+ .map((entry) =>
19262
+ this.pruneDebouncedFnAddIfNotKeeping({
19263
+ key: entry.hash,
19264
+ value: {
19265
+ entry,
19266
+ leaders: plan.leaders as Map<string, any>,
19267
+ },
19268
+ }),
19269
+ );
19270
+ }
19271
+ this.rebalanceParticipationDebounced?.call();
19272
+ } finally {
19273
+ // Settle seam for the receive token. Every consumer that can
19274
+ // roll it back runs inline before control leaves this block: the
19275
+ // prepared-join callback resolves during the join await, and the
19276
+ // late finish/rollback arm runs above. This `finally` is what
19277
+ // closes the abandon arms (no prepared-join commit, a declined
19278
+ // native commit, a downgrade to the plain join) without having
19279
+ // to enumerate them.
19280
+ this._coordinates.settleResidentCoordinateSnapshot(
19281
+ nativeReceiveCoordinateBatch?.rollbackCoordinateEntries,
19282
+ );
19148
19283
  }
19149
- this.rebalanceParticipationDebounced?.call();
19150
19284
  }
19151
19285
 
19152
19286
  for (const plan of joinPlans) {