@peerbit/shared-log 16.0.13 → 16.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -115,6 +115,7 @@ import {
115
115
  AbortError,
116
116
  TimeoutError,
117
117
  debounceFixedInterval,
118
+ delay,
118
119
  waitFor,
119
120
  } from "@peerbit/time";
120
121
  import pDefer, { type DeferredPromise } from "p-defer";
@@ -147,6 +148,7 @@ import {
147
148
  CompatibilityModeRetiredError,
148
149
  NativeDurableCommitError,
149
150
  NoPeersError,
151
+ PersistedDeliveryError,
150
152
  isNotStartedError,
151
153
  } from "./errors.js";
152
154
  import {
@@ -162,6 +164,7 @@ import {
162
164
  RequestIPruneV2,
163
165
  ResponseIPrune,
164
166
  ResponseIPruneV2,
167
+ SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS,
165
168
  SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
166
169
  SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY,
167
170
  SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM,
@@ -306,6 +309,7 @@ import {
306
309
  import {
307
310
  ConfirmEntriesMessage,
308
311
  RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
312
+ RequestPersistedEntriesV1,
309
313
  SYNC_MESSAGE_PRIORITY,
310
314
  SimpleSyncronizer,
311
315
  } from "./sync/simple.js";
@@ -449,6 +453,7 @@ export {
449
453
  CompatibilityModeRetiredError,
450
454
  NativeDurableCommitError,
451
455
  NoPeersError,
456
+ PersistedDeliveryError,
452
457
  };
453
458
  export { MAX_U32, MAX_U64, type NumberFromType };
454
459
  export type {
@@ -498,25 +503,36 @@ const joinNativeCoordinateDirectory = (
498
503
  `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
499
504
 
500
505
  type DurableBlockSublevelStore = {
506
+ readonly supportsCrashSafeJournalCheckpoint?: boolean;
501
507
  sublevel(
502
508
  name: string,
503
509
  options?: {
504
510
  compactOnClose?: boolean;
505
511
  compactOnCloseMinJournalBytes?: number;
512
+ compactMaxJournalBytes?: number;
506
513
  durability?: "normal" | "strict";
507
514
  },
508
515
  ): MaybePromise<AnyStore>;
509
516
  };
510
517
 
518
+ const defaultNativeEntryBlockCompactMaxJournalBytes = 64 * 1024 * 1024;
519
+
511
520
  const createNativeDurableBlockStore = async (
512
521
  storage: DurableBlockSublevelStore,
513
522
  ): Promise<AnyBlockStore> =>
514
523
  new AnyBlockStore(
515
524
  await storage.sublevel("blocks", {
516
- // Strict mirrors remain WAL-backed across close. The available snapshot
517
- // rewrite is not a crash-atomic generation protocol, so thresholds must
518
- // not re-enable it behind this acknowledgement boundary.
525
+ // Strict mirrors remain WAL-backed. On POSIX Node, checkpoint only the
526
+ // historical suffix through the Rust store's fsync + atomic-rename path;
527
+ // browsers/custom backends and Windows retain the prior unbounded WAL until
528
+ // they expose an equally strong directory durability barrier.
519
529
  compactOnClose: false,
530
+ ...(storage.supportsCrashSafeJournalCheckpoint === true
531
+ ? {
532
+ compactMaxJournalBytes:
533
+ defaultNativeEntryBlockCompactMaxJournalBytes,
534
+ }
535
+ : {}),
520
536
  // A native append is acknowledged only after this mirror resolves. The
521
537
  // Rust store's normal immutable fast path may resolve before its WAL write;
522
538
  // strict mode waits for the journal write and sync, closing the SIGKILL gap.
@@ -540,6 +556,7 @@ type LeaderMap = Map<string, { intersecting: boolean }>;
540
556
  type LeaderSelectionOptions<R extends "u32" | "u64"> = {
541
557
  roleAge?: number;
542
558
  candidates?: Iterable<string>;
559
+ freshLeaderPlan?: boolean;
543
560
  onLeader?: (key: string) => void;
544
561
  persist?:
545
562
  | {
@@ -687,6 +704,10 @@ type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
687
704
  };
688
705
  };
689
706
 
707
+ type PersistedDeliveryPlanningEntry<T, R extends "u32" | "u64"> =
708
+ | ShallowOrFullEntry<T>
709
+ | EntryReplicated<R>;
710
+
690
711
  type NativeBackboneSimpleDocumentProjectionPlan = {
691
712
  documentVariantType?: "u8" | "string";
692
713
  documentVariantValue?: string;
@@ -897,10 +918,15 @@ const nativeStrictDurableTransactionJournalRecordBytes = (
897
918
  return new TextEncoder().encode(JSON.stringify(record));
898
919
  };
899
920
 
921
+ type TrustedLocalCommitEvidence = {
922
+ committedHashes: Set<string>;
923
+ };
924
+
900
925
  type PreparedPayloadCommitOnlyProperties =
901
926
  NativeBackboneDocumentCommitOptions & {
902
927
  skipMissingNextJoin?: boolean;
903
928
  resolveTrimmedEntries?: boolean;
929
+ localCommitEvidence?: TrustedLocalCommitEvidence;
904
930
  };
905
931
 
906
932
  type PreparedPayloadsManyIndependentProperties<T> = {
@@ -909,6 +935,7 @@ type PreparedPayloadsManyIndependentProperties<T> = {
909
935
  nexts?: ShallowOrFullEntry<T>[][];
910
936
  nativeBackboneDocumentIndexes?: NativeBackboneDocumentIndexCommitInput[];
911
937
  retainMaterializationBytes?: boolean;
938
+ localCommitEvidence?: TrustedLocalCommitEvidence;
912
939
  };
913
940
 
914
941
  type PreparedPayloadCommitOnlyResult<T, R extends "u32" | "u64"> = {
@@ -1670,10 +1697,23 @@ export type Args<
1670
1697
  : "u32",
1671
1698
  > = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D, R>;
1672
1699
 
1673
- export type DeliveryReliability = "ack" | "best-effort";
1700
+ /**
1701
+ * `persisted` waits, after the local commit, for `minAcks` distinct remote
1702
+ * leaders that advertised crash-safe receipt support. Each receipt proves that
1703
+ * the exact block, lower-log row, and replication-coordinate row crossed that
1704
+ * peer's storage barriers at the receipt instant. It is cooperative-peer
1705
+ * evidence, not a Byzantine proof or a promise that the peer will retain the
1706
+ * entry forever. Only current capable leaders count.
1707
+ */
1708
+ export type DeliveryReliability = "ack" | "best-effort" | "persisted";
1674
1709
 
1675
1710
  export type DeliveryOptions = {
1676
1711
  reliability?: DeliveryReliability;
1712
+ /**
1713
+ * Required for persisted delivery; counts distinct current remote leaders.
1714
+ * This does not increase the entry's replication degree, so the configured
1715
+ * replication must make at least this many capable remote leaders eligible.
1716
+ */
1677
1717
  minAcks?: number;
1678
1718
  requireRecipients?: boolean;
1679
1719
  /**
@@ -1681,10 +1721,131 @@ export type DeliveryOptions = {
1681
1721
  * its control lane, so this only changes the direct/fallback RPC path.
1682
1722
  */
1683
1723
  priority?: number;
1724
+ /**
1725
+ * Overall delivery deadline in milliseconds. For persisted delivery it
1726
+ * starts after the local append has returned and includes leader planning,
1727
+ * transfer, receipt requests, and final ownership/session validation. The
1728
+ * omitted persisted default is 10 seconds plus one admission-attempt budget
1729
+ * per transfer chunk and the minimum receipt sender-pacing time implied by
1730
+ * the batch size. An explicit timeout remains exact.
1731
+ */
1684
1732
  timeout?: number;
1685
1733
  signal?: AbortSignal;
1686
1734
  };
1687
1735
 
1736
+ type CrashSafeStorageBarrier = {
1737
+ readonly crashSafe: true;
1738
+ barrier(): MaybePromise<void>;
1739
+ };
1740
+
1741
+ type PersistedReceiptStorage = {
1742
+ block: CrashSafeStorageBarrier;
1743
+ lower: CrashSafeStorageBarrier;
1744
+ coordinate: CrashSafeStorageBarrier;
1745
+ };
1746
+
1747
+ type PersistedDeliveryDeadline = {
1748
+ deadline: number;
1749
+ signal: AbortSignal;
1750
+ dispose(): void;
1751
+ };
1752
+
1753
+ const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
1754
+ const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
1755
+ const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
1756
+ const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
1757
+ const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
1758
+ const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
1759
+ const PERSISTED_RECEIPT_RETRY_MS = 50;
1760
+ const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
1761
+ const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
1762
+ const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
1763
+ const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
1764
+ const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
1765
+ const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
1766
+ const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
1767
+ const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
1768
+ const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
1769
+ const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
1770
+ const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
1771
+ const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
1772
+
1773
+ type PersistedReceiptIngressBucket = {
1774
+ requestTokens: number;
1775
+ hashTokens: number;
1776
+ refilledAt: number;
1777
+ };
1778
+
1779
+ type PersistedReceiptNodeIngressBudget = PersistedReceiptIngressBucket & {
1780
+ peerSessions: Map<string, PersistedReceiptIngressBucket>;
1781
+ };
1782
+
1783
+ type PersistedReceiptNodeEgressBudget = {
1784
+ peerSessions: Map<string, PersistedReceiptIngressBucket>;
1785
+ };
1786
+
1787
+ const persistedReceiptIngressBudgets = new WeakMap<
1788
+ object,
1789
+ PersistedReceiptNodeIngressBudget
1790
+ >();
1791
+
1792
+ // Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
1793
+ // on the Peerbit node (rather than one SharedLog) prevents independent programs
1794
+ // from silently overrunning the same remote receiver together.
1795
+ const persistedReceiptEgressBudgets = new WeakMap<
1796
+ object,
1797
+ PersistedReceiptNodeEgressBudget
1798
+ >();
1799
+
1800
+ const refillPersistedReceiptIngressBucket = (
1801
+ bucket: PersistedReceiptIngressBucket,
1802
+ now: number,
1803
+ requestCapacity: number,
1804
+ hashCapacity: number,
1805
+ requestsPerSecond: number,
1806
+ hashesPerSecond: number,
1807
+ ) => {
1808
+ // Never move the refill watermark backwards. Date.now() can jump after a
1809
+ // clock correction; accepting that earlier timestamp would grant the same
1810
+ // elapsed interval again on the next request.
1811
+ if (now <= bucket.refilledAt) {
1812
+ return;
1813
+ }
1814
+ const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
1815
+ if (elapsedSeconds > 0) {
1816
+ bucket.requestTokens = Math.min(
1817
+ requestCapacity,
1818
+ bucket.requestTokens + elapsedSeconds * requestsPerSecond,
1819
+ );
1820
+ bucket.hashTokens = Math.min(
1821
+ hashCapacity,
1822
+ bucket.hashTokens + elapsedSeconds * hashesPerSecond,
1823
+ );
1824
+ }
1825
+ bucket.refilledAt = now;
1826
+ };
1827
+
1828
+ const persistedReceiptPacingFloorMs = (hashCount: number): number => {
1829
+ const boundedHashCount = Math.max(0, Math.floor(hashCount));
1830
+ const requestCount = Math.ceil(
1831
+ boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE,
1832
+ );
1833
+ return Math.max(
1834
+ 0,
1835
+ ((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
1836
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
1837
+ 1_000,
1838
+ ((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
1839
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
1840
+ 1_000,
1841
+ );
1842
+ };
1843
+
1844
+ const persistedTransferAdmissionBudgetMs = (hashCount: number): number =>
1845
+ Math.ceil(
1846
+ Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE,
1847
+ ) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
1848
+
1688
1849
  export type SharedLogFanoutOptions = {
1689
1850
  root?: string;
1690
1851
  channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
@@ -1698,6 +1859,19 @@ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
1698
1859
 
1699
1860
  type TrustedLogAppendOptions<T> = AppendOptions<T> & {
1700
1861
  __peerbitCanAppendAlreadyValidated?: boolean;
1862
+ __peerbitOnLocalCommit?: (hashes: readonly string[]) => void;
1863
+ };
1864
+
1865
+ const attachTrustedLocalCommitEvidence = <T>(
1866
+ options: AppendOptions<T>,
1867
+ evidence: TrustedLocalCommitEvidence | undefined,
1868
+ ): void => {
1869
+ if (!evidence) return;
1870
+ (options as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (hashes) => {
1871
+ for (const hash of hashes) {
1872
+ evidence.committedHashes.add(hash);
1873
+ }
1874
+ };
1701
1875
  };
1702
1876
 
1703
1877
  export type SharedAppendOptions<T> =
@@ -3008,6 +3182,37 @@ export class SharedLog<
3008
3182
  }
3009
3183
  }
3010
3184
 
3185
+ private async finishCommittedNativeStrictDurableTransaction<TValue>(
3186
+ handle: NativeStrictDurableTransactionHandle | undefined,
3187
+ finish: () => MaybePromise<TValue>,
3188
+ shouldWarnOnRetirementFailure: () => boolean = () => true,
3189
+ ): Promise<TValue> {
3190
+ let finishResult: TValue | undefined;
3191
+ let finishError: unknown;
3192
+ let finishFailed = false;
3193
+ try {
3194
+ finishResult = await finish();
3195
+ } catch (error) {
3196
+ finishError = error;
3197
+ finishFailed = true;
3198
+ }
3199
+ try {
3200
+ await this.completeNativeStrictDurableTransaction(handle);
3201
+ } catch (error) {
3202
+ // The acknowledged lower commit is already final. Completion poisons and
3203
+ // releases the in-memory transaction before throwing, so preserve an
3204
+ // earlier post-commit failure while leaving recovery to retire the intent.
3205
+ if (!finishFailed && !shouldWarnOnRetirementFailure()) {
3206
+ throw error;
3207
+ }
3208
+ warn(`Failed to retire committed native intent: ${String(error)}`);
3209
+ }
3210
+ if (finishFailed) {
3211
+ throw finishError;
3212
+ }
3213
+ return finishResult as TValue;
3214
+ }
3215
+
3011
3216
  private releaseNativeStrictDurableTransaction(
3012
3217
  handle: NativeStrictDurableTransactionHandle | undefined,
3013
3218
  cause: unknown = new Error(
@@ -3493,6 +3698,9 @@ export class SharedLog<
3493
3698
  // parallel map so existing capability-number consumers remain unchanged.
3494
3699
  private _peerSyncCapabilitySessions!: Map<string, bigint>;
3495
3700
  private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
3701
+ private _persistedReceiptStorage?: PersistedReceiptStorage;
3702
+ private _persistedReceiptRequestsInFlight!: Map<string, number>;
3703
+ private _persistedReceiptRequestsInFlightTotal!: number;
3496
3704
  // Pending live raw exchange-head gossip, coalesced per recipient set and
3497
3705
  // flushed at the end of the current event-loop turn (or when a batch cap
3498
3706
  // is hit). Only used when every recipient advertised raw capability.
@@ -3567,10 +3775,7 @@ export class SharedLog<
3567
3775
  return new ReplicationInfoV2SendCoordinator<R>({
3568
3776
  getRpc: () => this.rpc,
3569
3777
  getSelfKey: () => this.node.identity.publicKey,
3570
- getSenderTransportSession: () =>
3571
- BigInt(
3572
- (this.node.services.pubsub as unknown as { session: number }).session,
3573
- ),
3778
+ getSenderTransportSession: () => this.ownTransportSession(),
3574
3779
  getMyReplicationSegments: () => this.getMyReplicationSegments(),
3575
3780
  validatePersistedReplicationRangeSnapshot: (ranges) =>
3576
3781
  this.validatePersistedReplicationRangeSnapshot(ranges),
@@ -3595,12 +3800,37 @@ export class SharedLog<
3595
3800
  });
3596
3801
  }
3597
3802
 
3803
+ private resolvePersistedReceiptStorage():
3804
+ | PersistedReceiptStorage
3805
+ | undefined {
3806
+ if (this.log.appendDurability !== "strict") {
3807
+ return undefined;
3808
+ }
3809
+ const block = this.remoteBlocks.crashSafeDurability;
3810
+ const lower = this.log.entryIndex.properties.index.crashSafeDurability;
3811
+ const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
3812
+ if (!block || !lower || !coordinate) {
3813
+ return undefined;
3814
+ }
3815
+ return { block, lower, coordinate };
3816
+ }
3817
+
3818
+ private ownTransportSession(): bigint {
3819
+ return BigInt(
3820
+ (this.node.services.pubsub as unknown as { session: number | bigint })
3821
+ .session,
3822
+ );
3823
+ }
3824
+
3598
3825
  private replicationInfoV2ReceiveCapabilities(): number {
3599
3826
  return (
3600
3827
  SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
3601
3828
  SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
3602
3829
  SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
3603
3830
  SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
3831
+ (this._persistedReceiptStorage
3832
+ ? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
3833
+ : 0) |
3604
3834
  (this._logProperties?.sync?.rawExchangeHeads === true
3605
3835
  ? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
3606
3836
  : 0)
@@ -3616,9 +3846,7 @@ export class SharedLog<
3616
3846
  { receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
3617
3847
  > {
3618
3848
  const peerHash = properties.target.hashcode();
3619
- const receiverTransportSession = BigInt(
3620
- (this.node.services.pubsub as unknown as { session: number }).session,
3621
- );
3849
+ const receiverTransportSession = this.ownTransportSession();
3622
3850
  await this.rpc.send(
3623
3851
  new SyncCapabilitiesMessage({
3624
3852
  capabilities: this.replicationInfoV2ReceiveCapabilities(),
@@ -3644,9 +3872,7 @@ export class SharedLog<
3644
3872
  ) ||
3645
3873
  this._peerSessions.isReplicationInfoBlocked(peerHash) ||
3646
3874
  !this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
3647
- BigInt(
3648
- (this.node.services.pubsub as unknown as { session: number }).session,
3649
- ) !== receiverTransportSession
3875
+ this.ownTransportSession() !== receiverTransportSession
3650
3876
  ) {
3651
3877
  return undefined;
3652
3878
  }
@@ -3659,10 +3885,7 @@ export class SharedLog<
3659
3885
  private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
3660
3886
  return new ReplicationInfoV2ReceiveCoordinator({
3661
3887
  getSelfKey: () => this.node.identity.publicKey,
3662
- getReceiverTransportSession: () =>
3663
- BigInt(
3664
- (this.node.services.pubsub as unknown as { session: number }).session,
3665
- ),
3888
+ getReceiverTransportSession: () => this.ownTransportSession(),
3666
3889
  isClosed: () => this.closed,
3667
3890
  isPeerSessionCurrent: (peerHash, peerSession) =>
3668
3891
  this._peerSessions.isCurrent(peerHash, peerSession) &&
@@ -3832,6 +4055,9 @@ export class SharedLog<
3832
4055
  this._peerSyncCapabilities = new Map();
3833
4056
  this._peerSyncCapabilitySessions = new Map();
3834
4057
  this._peerSyncCapabilityTimestamps = new Map();
4058
+ this._persistedReceiptStorage = undefined;
4059
+ this._persistedReceiptRequestsInFlight = new Map();
4060
+ this._persistedReceiptRequestsInFlightTotal = 0;
3835
4061
  this._liveRawGossipBatches = new Map();
3836
4062
  this._liveRawGossipFlushScheduled = false;
3837
4063
  this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
@@ -4001,6 +4227,14 @@ export class SharedLog<
4001
4227
  this._fanoutChannel = undefined;
4002
4228
  }
4003
4229
 
4230
+ private ensureLogProviderHandle(fanoutService: FanoutTree): void {
4231
+ if (this._providerHandle || this._closeController.signal.aborted) return;
4232
+ this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
4233
+ ttlMs: 120_000,
4234
+ announceIntervalMs: 60_000,
4235
+ });
4236
+ }
4237
+
4004
4238
  private async _onFanoutData(detail: FanoutTreeDataEvent) {
4005
4239
  let envelope: FanoutEnvelope;
4006
4240
  try {
@@ -4127,11 +4361,23 @@ export class SharedLog<
4127
4361
  const reliability: DeliveryReliability = delivery.reliability ?? "ack";
4128
4362
  const deliveryTimeout = delivery.timeout;
4129
4363
  const deliverySignal = delivery.signal;
4130
- const requireRecipients = delivery.requireRecipients === true;
4364
+ const requireRecipients =
4365
+ reliability === "persisted" || delivery.requireRecipients === true;
4131
4366
  const minAcks =
4132
4367
  delivery.minAcks != null && Number.isFinite(delivery.minAcks)
4133
4368
  ? Math.max(0, Math.floor(delivery.minAcks))
4134
4369
  : undefined;
4370
+ if (reliability === "persisted") {
4371
+ if (
4372
+ delivery.minAcks == null ||
4373
+ !Number.isSafeInteger(delivery.minAcks) ||
4374
+ delivery.minAcks <= 0
4375
+ ) {
4376
+ throw new Error(
4377
+ 'persisted delivery requires a positive explicit "minAcks"',
4378
+ );
4379
+ }
4380
+ }
4135
4381
 
4136
4382
  const wrap =
4137
4383
  deliveryTimeout == null && deliverySignal == null
@@ -4206,6 +4452,82 @@ export class SharedLog<
4206
4452
  };
4207
4453
  }
4208
4454
 
4455
+ private validatePersistedReceiptRequestShape(
4456
+ request: RequestPersistedEntriesV1,
4457
+ ): void {
4458
+ if (
4459
+ request.hashes.length === 0 ||
4460
+ request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES
4461
+ ) {
4462
+ throw new Error(
4463
+ `Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`,
4464
+ );
4465
+ }
4466
+ let bytes = 0;
4467
+ const encoder = new TextEncoder();
4468
+ for (const hash of request.hashes) {
4469
+ if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
4470
+ throw new Error("Invalid persisted receipt hash batch");
4471
+ }
4472
+ bytes += encoder.encode(hash).byteLength;
4473
+ if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
4474
+ throw new Error("Invalid persisted receipt hash batch");
4475
+ }
4476
+ }
4477
+ }
4478
+
4479
+ private hasValidPersistedReceiptHashes(
4480
+ request: RequestPersistedEntriesV1,
4481
+ ): boolean {
4482
+ const seen = new Set<string>();
4483
+ try {
4484
+ for (const hash of request.hashes) {
4485
+ if (seen.has(hash)) return false;
4486
+ cidifyString(hash);
4487
+ seen.add(hash);
4488
+ }
4489
+ } catch {
4490
+ return false;
4491
+ }
4492
+ return true;
4493
+ }
4494
+
4495
+ private getPersistedDeliveryOptions(
4496
+ options?: SharedAppendOptions<T>,
4497
+ ): DeliveryOptions | undefined {
4498
+ const delivery = options?.delivery;
4499
+ if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
4500
+ return undefined;
4501
+ }
4502
+ const parsed = this._parseDeliveryOptions(options?.delivery);
4503
+ const target = (options as { target?: string } | undefined)?.target;
4504
+ if (target !== undefined && target !== "replicators") {
4505
+ throw new Error(
4506
+ 'persisted delivery requires target="replicators" (or an omitted target)',
4507
+ );
4508
+ }
4509
+ if (
4510
+ parsed.delivery?.timeout != null &&
4511
+ (!Number.isFinite(parsed.delivery.timeout) ||
4512
+ parsed.delivery.timeout <= 0 ||
4513
+ parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)
4514
+ ) {
4515
+ throw new Error(
4516
+ `persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
4517
+ );
4518
+ }
4519
+ if (parsed.delivery?.signal?.aborted) {
4520
+ throw parsed.delivery.signal.reason ?? new AbortError();
4521
+ }
4522
+ return parsed.delivery!;
4523
+ }
4524
+
4525
+ private assertPersistedDeliveryOptions(
4526
+ options?: SharedAppendOptions<T>,
4527
+ ): void {
4528
+ this.getPersistedDeliveryOptions(options);
4529
+ }
4530
+
4209
4531
  private async _getSortedRouteHints(targetHash: string): Promise<RouteHint[]> {
4210
4532
  const pubsub: any = this.node.services.pubsub as any;
4211
4533
  const maybeHints = await pubsub?.getUnifiedRouteHints?.(
@@ -4620,6 +4942,7 @@ export class SharedLog<
4620
4942
  plan: RawExchangeHeadSendPlan,
4621
4943
  to: string[] | Set<string>,
4622
4944
  options?: {
4945
+ acknowledge?: boolean;
4623
4946
  priority?: number;
4624
4947
  reserved?: Uint8Array;
4625
4948
  signal?: AbortSignal;
@@ -4634,7 +4957,7 @@ export class SharedLog<
4634
4957
  payload: Uint8Array,
4635
4958
  properties: { topics: string[] },
4636
4959
  options: {
4637
- mode: SilentDelivery;
4960
+ mode: SilentDelivery | AcknowledgeDelivery;
4638
4961
  priority?: number;
4639
4962
  signal?: AbortSignal;
4640
4963
  },
@@ -4722,7 +5045,9 @@ export class SharedLog<
4722
5045
  item.payload,
4723
5046
  { topics: [topic] },
4724
5047
  {
4725
- mode: new SilentDelivery({ redundancy: 1, to: [...to] }),
5048
+ mode: options?.acknowledge
5049
+ ? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
5050
+ : new SilentDelivery({ redundancy: 1, to: [...to] }),
4726
5051
  priority: options?.priority,
4727
5052
  signal: options?.signal,
4728
5053
  },
@@ -4749,34 +5074,762 @@ export class SharedLog<
4749
5074
  });
4750
5075
  }
4751
5076
  }
4752
- return sentMessages;
4753
- }
4754
-
4755
- /**
4756
- * `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
4757
- * responses: resolves the head/reference plan like the TS raw path and
4758
- * ships it fused when possible.
4759
- */
4760
- private async trySendFusedRawExchangeHeads(
4761
- hashes: string[],
4762
- to: string[],
4763
- options?: {
4764
- priority?: number;
4765
- reserved?: Uint8Array;
4766
- signal?: AbortSignal;
4767
- },
4768
- ): Promise<number | undefined> {
4769
- if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
4770
- return undefined;
4771
- }
4772
- const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
4773
- if (!plan) {
4774
- return undefined;
4775
- }
4776
- if (plan.hashes.length === 0) {
4777
- return 0;
4778
- }
4779
- return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
5077
+ return sentMessages;
5078
+ }
5079
+
5080
+ /**
5081
+ * `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
5082
+ * responses: resolves the head/reference plan like the TS raw path and
5083
+ * ships it fused when possible.
5084
+ */
5085
+ private async trySendFusedRawExchangeHeads(
5086
+ hashes: string[],
5087
+ to: string[],
5088
+ options?: {
5089
+ acknowledge?: boolean;
5090
+ priority?: number;
5091
+ reserved?: Uint8Array;
5092
+ signal?: AbortSignal;
5093
+ },
5094
+ ): Promise<number | undefined> {
5095
+ if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
5096
+ return undefined;
5097
+ }
5098
+ const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
5099
+ if (!plan) {
5100
+ return undefined;
5101
+ }
5102
+ if (plan.hashes.length === 0) {
5103
+ return 0;
5104
+ }
5105
+ return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
5106
+ }
5107
+
5108
+ private persistedReceiptPeerSession(
5109
+ peerHash: string,
5110
+ ): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
5111
+ const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
5112
+ const peerSession = this._peerSessions.current(peerHash);
5113
+ if (
5114
+ capabilitySession == null ||
5115
+ !peerSession ||
5116
+ peerSession.phase !== "open" ||
5117
+ !this._peerSessions.isCurrent(peerHash, peerSession) ||
5118
+ !this._peerSyncCapabilityTimestamps.has(peerHash) ||
5119
+ ((this._peerSyncCapabilities.get(peerHash) ?? 0) &
5120
+ SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
5121
+ 0
5122
+ ) {
5123
+ return undefined;
5124
+ }
5125
+ return { capabilitySession, peerSession };
5126
+ }
5127
+
5128
+ private async waitPersistedReceiptRetry(
5129
+ signal: AbortSignal,
5130
+ ms: number,
5131
+ ): Promise<void> {
5132
+ try {
5133
+ await delay(ms, { signal });
5134
+ } catch (error) {
5135
+ throw signal.aborted ? (signal.reason ?? error) : error;
5136
+ }
5137
+ }
5138
+
5139
+ private reservePersistedReceiptEgress(
5140
+ peer: string,
5141
+ capabilitySession: bigint,
5142
+ hashCount: number,
5143
+ now = Date.now(),
5144
+ ): number {
5145
+ let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
5146
+ if (!nodeBudget) {
5147
+ nodeBudget = { peerSessions: new Map() };
5148
+ persistedReceiptEgressBudgets.set(this.node, nodeBudget);
5149
+ }
5150
+ const peerSessionKey = `${peer}\0${capabilitySession}`;
5151
+ let bucket = nodeBudget.peerSessions.get(peerSessionKey);
5152
+ if (!bucket) {
5153
+ while (
5154
+ nodeBudget.peerSessions.size >=
5155
+ MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
5156
+ ) {
5157
+ const oldest = nodeBudget.peerSessions.keys().next().value;
5158
+ if (oldest === undefined) break;
5159
+ nodeBudget.peerSessions.delete(oldest);
5160
+ }
5161
+ bucket = {
5162
+ requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
5163
+ hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
5164
+ refilledAt: now,
5165
+ };
5166
+ nodeBudget.peerSessions.set(peerSessionKey, bucket);
5167
+ } else {
5168
+ nodeBudget.peerSessions.delete(peerSessionKey);
5169
+ nodeBudget.peerSessions.set(peerSessionKey, bucket);
5170
+ }
5171
+ refillPersistedReceiptIngressBucket(
5172
+ bucket,
5173
+ now,
5174
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
5175
+ PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
5176
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
5177
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
5178
+ );
5179
+ if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
5180
+ bucket.requestTokens -= 1;
5181
+ bucket.hashTokens -= hashCount;
5182
+ return 0;
5183
+ }
5184
+ return Math.max(
5185
+ 1,
5186
+ Math.ceil(
5187
+ Math.max(
5188
+ ((1 - bucket.requestTokens) /
5189
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
5190
+ 1_000,
5191
+ ((hashCount - bucket.hashTokens) /
5192
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
5193
+ 1_000,
5194
+ ),
5195
+ ),
5196
+ );
5197
+ }
5198
+
5199
+ private async waitForPersistedReceiptEgressAdmission(
5200
+ peer: string,
5201
+ capabilitySession: bigint,
5202
+ hashCount: number,
5203
+ signal: AbortSignal,
5204
+ ): Promise<void> {
5205
+ while (true) {
5206
+ if (signal.aborted) {
5207
+ throw signal.reason ?? new AbortError();
5208
+ }
5209
+ const waitMs = this.reservePersistedReceiptEgress(
5210
+ peer,
5211
+ capabilitySession,
5212
+ hashCount,
5213
+ );
5214
+ if (waitMs === 0) return;
5215
+ await this.waitPersistedReceiptRetry(signal, waitMs);
5216
+ }
5217
+ }
5218
+
5219
+ private persistedDeliveryTimeoutMs(
5220
+ delivery: DeliveryOptions,
5221
+ hashCount: number,
5222
+ ): number {
5223
+ return (
5224
+ delivery.timeout ??
5225
+ Math.min(
5226
+ MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
5227
+ DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
5228
+ persistedTransferAdmissionBudgetMs(hashCount) +
5229
+ Math.ceil(persistedReceiptPacingFloorMs(hashCount)),
5230
+ )
5231
+ );
5232
+ }
5233
+
5234
+ private async waitForPersistedTransferAdmission(
5235
+ peer: string,
5236
+ hashes: readonly string[],
5237
+ captured: { capabilitySession: bigint; peerSession: PeerSession },
5238
+ signal: AbortSignal,
5239
+ isStillCurrent: () => boolean,
5240
+ ): Promise<boolean> {
5241
+ const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
5242
+ while (!signal.aborted && isStillCurrent()) {
5243
+ const current = this.persistedReceiptPeerSession(peer);
5244
+ if (
5245
+ !current ||
5246
+ current.capabilitySession !== captured.capabilitySession ||
5247
+ current.peerSession !== captured.peerSession
5248
+ ) {
5249
+ return false;
5250
+ }
5251
+ if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
5252
+ return true;
5253
+ }
5254
+ const remaining = expiresAt - Date.now();
5255
+ if (remaining <= 0) return false;
5256
+ await this.waitPersistedReceiptRetry(
5257
+ signal,
5258
+ Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining),
5259
+ );
5260
+ }
5261
+ if (signal.aborted) throw signal.reason ?? new AbortError();
5262
+ return false;
5263
+ }
5264
+
5265
+ private createPersistedDeliveryDeadline(
5266
+ delivery: DeliveryOptions,
5267
+ ownershipLifecycleController: AbortController,
5268
+ hashCount = 1,
5269
+ ): PersistedDeliveryDeadline {
5270
+ const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
5271
+ if (
5272
+ !Number.isFinite(timeoutMs) ||
5273
+ timeoutMs <= 0 ||
5274
+ timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
5275
+ ) {
5276
+ throw new Error(
5277
+ `persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
5278
+ );
5279
+ }
5280
+ const minAcks = Math.floor(delivery.minAcks!);
5281
+ const deadlineController = new AbortController();
5282
+ const timeout = setTimeout(
5283
+ () =>
5284
+ deadlineController.abort(
5285
+ new TimeoutError(
5286
+ `Timed out waiting for ${minAcks} persisted remote replicas.`,
5287
+ ),
5288
+ ),
5289
+ timeoutMs,
5290
+ );
5291
+ timeout.unref?.();
5292
+ return {
5293
+ deadline: Date.now() + timeoutMs,
5294
+ signal: AbortSignal.any(
5295
+ [
5296
+ delivery.signal,
5297
+ this._closeController.signal,
5298
+ ownershipLifecycleController.signal,
5299
+ deadlineController.signal,
5300
+ ].filter((value): value is AbortSignal => !!value),
5301
+ ),
5302
+ dispose: () => clearTimeout(timeout),
5303
+ };
5304
+ }
5305
+
5306
+ private async planPersistedDeliveryLeaders(
5307
+ entries: PersistedDeliveryPlanningEntry<T, R>[],
5308
+ replicas: number,
5309
+ ownershipLifecycleController: AbortController,
5310
+ ): Promise<LeaderMap[]> {
5311
+ if (
5312
+ this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry
5313
+ ) {
5314
+ const leaders: LeaderMap[] = [];
5315
+ for (const entry of entries) {
5316
+ leaders.push(
5317
+ await this.findLeadersFromEntry(
5318
+ entry,
5319
+ replicas,
5320
+ { freshLeaderPlan: true },
5321
+ ownershipLifecycleController,
5322
+ ),
5323
+ );
5324
+ }
5325
+ return leaders;
5326
+ }
5327
+ const items: EntryLeaderBatchItem<R>[] = entries.map((entry) => ({
5328
+ entry,
5329
+ replicas,
5330
+ options: { freshLeaderPlan: true, persist: false },
5331
+ }));
5332
+ const nativeRoutingPlanner =
5333
+ this._nativeRangePlanner ?? this._nativeBackbone;
5334
+ if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
5335
+ const options = items[0]!.options!;
5336
+ const context = await this.createLeaderSelectionContext(
5337
+ options,
5338
+ ownershipLifecycleController,
5339
+ );
5340
+ this.throwIfReplicationOwnershipLifecycleInactive(
5341
+ ownershipLifecycleController,
5342
+ );
5343
+ const nativeOptions = this.createNativeLeaderOptions(context, options);
5344
+ const fullReplicaLeaders =
5345
+ nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(
5346
+ replicas,
5347
+ nativeOptions,
5348
+ );
5349
+ this.throwIfReplicationOwnershipLifecycleInactive(
5350
+ ownershipLifecycleController,
5351
+ );
5352
+ if (fullReplicaLeaders) {
5353
+ // Receipt settlement treats leader maps as read-only. Reusing the one
5354
+ // gid-independent routing result avoids one Map allocation per entry.
5355
+ return new Array<LeaderMap>(items.length).fill(fullReplicaLeaders);
5356
+ }
5357
+ if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
5358
+ const nativeLeaders =
5359
+ nativeRoutingPlanner.planLeaderSamplesForGidsBatch(
5360
+ items.map((item) => ({
5361
+ gid: this.getEntryGid(item.entry),
5362
+ replicas: item.replicas,
5363
+ })),
5364
+ nativeOptions,
5365
+ );
5366
+ this.throwIfReplicationOwnershipLifecycleInactive(
5367
+ ownershipLifecycleController,
5368
+ );
5369
+ if (nativeLeaders?.length === items.length) {
5370
+ return nativeLeaders;
5371
+ }
5372
+ }
5373
+ }
5374
+ return (
5375
+ await this.planEntryLeaderBatch(items, ownershipLifecycleController)
5376
+ ).map((plan) => plan.leaders);
5377
+ }
5378
+
5379
+ private async settlePersistedDelivery(
5380
+ input: PersistedDeliveryPlanningEntry<T, R>[],
5381
+ replicas: number,
5382
+ delivery: DeliveryOptions,
5383
+ ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
5384
+ persistedDeadline?: PersistedDeliveryDeadline,
5385
+ transferOnFirstRound = false,
5386
+ ): Promise<void> {
5387
+ const minAcks = Math.floor(delivery.minAcks!);
5388
+ const entries = new Map(input.map((entry) => [entry.hash, entry]));
5389
+ if (entries.size === 0) return;
5390
+
5391
+ const committedHashes = [...entries.keys()];
5392
+ const ownedDeadline = !persistedDeadline;
5393
+ const deadline =
5394
+ persistedDeadline ??
5395
+ this.createPersistedDeliveryDeadline(
5396
+ delivery,
5397
+ ownershipLifecycleController,
5398
+ entries.size,
5399
+ );
5400
+ const signal = deadline.signal;
5401
+ let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
5402
+ let initialTransferPending = transferOnFirstRound;
5403
+ let needsInitialLeaderCheck = true;
5404
+ const carriedAcknowledgements = new Map<
5405
+ string,
5406
+ Map<string, { capabilitySession: bigint; peerSession: PeerSession }>
5407
+ >(committedHashes.map((hash) => [hash, new Map()]));
5408
+ const repairsByPeer = new Map<
5409
+ string,
5410
+ {
5411
+ capabilitySession: bigint;
5412
+ peerSession: PeerSession;
5413
+ hashes: Set<string>;
5414
+ }
5415
+ >();
5416
+ let acknowledgementOwnershipRevision: number | undefined;
5417
+ const purgePeerDeliveryState = (peer: string) => {
5418
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5419
+ acknowledgements.delete(peer);
5420
+ }
5421
+ repairsByPeer.delete(peer);
5422
+ };
5423
+ try {
5424
+ while (true) {
5425
+ if (signal.aborted) {
5426
+ throw signal.reason ?? new AbortError();
5427
+ }
5428
+ this.throwIfReplicationOwnershipLifecycleInactive(
5429
+ ownershipLifecycleController,
5430
+ );
5431
+ const ownershipRevision =
5432
+ this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
5433
+ const isRoundOwnershipCurrent = () =>
5434
+ this.isReceiveOwnershipSnapshotStable(ownershipRevision);
5435
+ if (acknowledgementOwnershipRevision !== ownershipRevision) {
5436
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5437
+ acknowledgements.clear();
5438
+ }
5439
+ repairsByPeer.clear();
5440
+ acknowledgementOwnershipRevision = ownershipRevision;
5441
+ } else {
5442
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5443
+ for (const [peer, captured] of acknowledgements) {
5444
+ const current = this.persistedReceiptPeerSession(peer);
5445
+ if (
5446
+ !current ||
5447
+ current.capabilitySession !== captured.capabilitySession ||
5448
+ current.peerSession !== captured.peerSession
5449
+ ) {
5450
+ purgePeerDeliveryState(peer);
5451
+ }
5452
+ }
5453
+ }
5454
+ }
5455
+ if (!isRoundOwnershipCurrent()) {
5456
+ await this.waitPersistedReceiptRetry(
5457
+ signal,
5458
+ PERSISTED_RECEIPT_RETRY_MS,
5459
+ );
5460
+ continue;
5461
+ }
5462
+
5463
+ // Carry receipts only across retries in the exact same ownership and
5464
+ // transport epoch. A revision/session change purges them before they can
5465
+ // survive an away-and-back leader transition or combine with a later peer.
5466
+ const hashesByPeer = new Map<string, string[]>();
5467
+ const entryArray = [...entries.values()];
5468
+ const leadersByEntry = await this.planPersistedDeliveryLeaders(
5469
+ entryArray,
5470
+ replicas,
5471
+ ownershipLifecycleController,
5472
+ );
5473
+ if (!isRoundOwnershipCurrent()) continue;
5474
+ const selfHash = this.node.identity.publicKey.hashcode();
5475
+ if (needsInitialLeaderCheck) {
5476
+ needsInitialLeaderCheck = false;
5477
+ if (
5478
+ leadersByEntry.some(
5479
+ (leaders) =>
5480
+ leaders.size === 0 ||
5481
+ (leaders.size === 1 && leaders.has(selfHash)),
5482
+ )
5483
+ ) {
5484
+ throw new NoPeersError(this.rpc.topic);
5485
+ }
5486
+ }
5487
+ for (let index = 0; index < entryArray.length; index++) {
5488
+ const hash = entryArray[index]!.hash;
5489
+ const leaders = leadersByEntry[index]!;
5490
+ if (signal.aborted) {
5491
+ throw signal.reason ?? new AbortError();
5492
+ }
5493
+ if (!isRoundOwnershipCurrent()) break;
5494
+ const acknowledgements = carriedAcknowledgements.get(hash)!;
5495
+ for (const [peer, captured] of acknowledgements) {
5496
+ const current = this.persistedReceiptPeerSession(peer);
5497
+ if (
5498
+ !leaders.has(peer) ||
5499
+ !current ||
5500
+ current.capabilitySession !== captured.capabilitySession ||
5501
+ current.peerSession !== captured.peerSession
5502
+ ) {
5503
+ acknowledgements.delete(peer);
5504
+ }
5505
+ }
5506
+ if (acknowledgements.size >= minAcks) continue;
5507
+ for (const peer of leaders.keys()) {
5508
+ if (peer === selfHash) continue;
5509
+ const current = this.persistedReceiptPeerSession(peer);
5510
+ if (!current) continue;
5511
+ if (acknowledgements.has(peer)) continue;
5512
+ const hashes = hashesByPeer.get(peer) ?? [];
5513
+ hashes.push(hash);
5514
+ hashesByPeer.set(peer, hashes);
5515
+ }
5516
+ }
5517
+ if (!isRoundOwnershipCurrent()) continue;
5518
+
5519
+ const operationQueue = new PQueue({
5520
+ concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
5521
+ });
5522
+ const candidateWaves = Math.max(
5523
+ 1,
5524
+ Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL),
5525
+ );
5526
+ const roundController = new AbortController();
5527
+ const roundSignal = AbortSignal.any([signal, roundController.signal]);
5528
+ const getAttemptTimeout = () =>
5529
+ Math.max(
5530
+ 1,
5531
+ Math.min(
5532
+ maxAttemptMs,
5533
+ Math.floor((deadline.deadline - Date.now()) / candidateWaves),
5534
+ ),
5535
+ );
5536
+ const requests = new Set<Promise<void>>();
5537
+ const transferAllOnRound = initialTransferPending;
5538
+ try {
5539
+ for (const [peer, hashes] of hashesByPeer) {
5540
+ let request!: Promise<void>;
5541
+ request = (async () => {
5542
+ const captured = this.persistedReceiptPeerSession(peer);
5543
+ if (!captured) return;
5544
+ const previousRepair = repairsByPeer.get(peer);
5545
+ if (
5546
+ previousRepair &&
5547
+ (previousRepair.capabilitySession !==
5548
+ captured.capabilitySession ||
5549
+ previousRepair.peerSession !== captured.peerSession)
5550
+ ) {
5551
+ repairsByPeer.delete(peer);
5552
+ }
5553
+ const ensureRepairState = () => {
5554
+ let state = repairsByPeer.get(peer);
5555
+ if (!state) {
5556
+ state = {
5557
+ capabilitySession: captured.capabilitySession,
5558
+ peerSession: captured.peerSession,
5559
+ hashes: new Set<string>(),
5560
+ };
5561
+ repairsByPeer.set(peer, state);
5562
+ }
5563
+ return state;
5564
+ };
5565
+ const isPeerRoundCurrent = () => {
5566
+ if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
5567
+ return false;
5568
+ }
5569
+ const current = this.persistedReceiptPeerSession(peer);
5570
+ return (
5571
+ !!current &&
5572
+ current.capabilitySession === captured.capabilitySession &&
5573
+ current.peerSession === captured.peerSession
5574
+ );
5575
+ };
5576
+ const repairs = repairsByPeer.get(peer)?.hashes;
5577
+ const transferHashes = transferAllOnRound
5578
+ ? hashes
5579
+ : repairs
5580
+ ? hashes.filter((hash) => repairs.has(hash))
5581
+ : [];
5582
+ let attemptedHashCount = 0;
5583
+ if (transferHashes.length > 0) {
5584
+ try {
5585
+ await this.pushEntryHashes(peer, transferHashes, {
5586
+ acknowledge: false,
5587
+ chunkTimeout: getAttemptTimeout,
5588
+ chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
5589
+ isStillCurrent: isPeerRoundCurrent,
5590
+ onChunkAttempted: (chunk) => {
5591
+ attemptedHashCount += chunk.length;
5592
+ if (!transferAllOnRound) {
5593
+ const state = repairsByPeer.get(peer);
5594
+ for (const hash of chunk) state?.hashes.delete(hash);
5595
+ }
5596
+ },
5597
+ onChunkSent: async (chunk) => {
5598
+ return this.waitForPersistedTransferAdmission(
5599
+ peer,
5600
+ chunk,
5601
+ captured,
5602
+ roundSignal,
5603
+ isPeerRoundCurrent,
5604
+ );
5605
+ },
5606
+ operationQueue,
5607
+ priority: delivery.priority,
5608
+ repairHint: !transferAllOnRound,
5609
+ signal: roundSignal,
5610
+ });
5611
+ } catch {
5612
+ // A send can fail after remote admission. The attempted prefix
5613
+ // includes that uncertain chunk for authoritative receipt probing.
5614
+ }
5615
+ if (
5616
+ transferAllOnRound &&
5617
+ attemptedHashCount < transferHashes.length
5618
+ ) {
5619
+ const state = ensureRepairState();
5620
+ for (const hash of transferHashes.slice(attemptedHashCount)) {
5621
+ state.hashes.add(hash);
5622
+ }
5623
+ }
5624
+ }
5625
+ const receiptHashes = transferAllOnRound
5626
+ ? hashes.slice(0, attemptedHashCount)
5627
+ : hashes;
5628
+ if (!isPeerRoundCurrent()) {
5629
+ purgePeerDeliveryState(peer);
5630
+ return;
5631
+ }
5632
+ // Keep receipt chunks sequential per peer. The shared operation
5633
+ // queue bounds only active sends/requests; admission waits never
5634
+ // occupy its slots and cannot starve a later healthy candidate.
5635
+ for (
5636
+ let offset = 0;
5637
+ offset < receiptHashes.length;
5638
+ offset += PERSISTED_RECEIPT_CHUNK_SIZE
5639
+ ) {
5640
+ const requestedHashes = receiptHashes.slice(
5641
+ offset,
5642
+ offset + PERSISTED_RECEIPT_CHUNK_SIZE,
5643
+ );
5644
+ if (roundSignal.aborted || !isPeerRoundCurrent()) {
5645
+ if (!isRoundOwnershipCurrent()) roundController.abort();
5646
+ break;
5647
+ }
5648
+ let responses;
5649
+ try {
5650
+ await this.waitForPersistedReceiptEgressAdmission(
5651
+ peer,
5652
+ captured.capabilitySession,
5653
+ requestedHashes.length,
5654
+ roundSignal,
5655
+ );
5656
+ if (!isPeerRoundCurrent()) break;
5657
+ const attemptTimeout = getAttemptTimeout();
5658
+ responses =
5659
+ (await operationQueue.add(async () => {
5660
+ if (!isPeerRoundCurrent()) return [];
5661
+ return this.rpc.request(
5662
+ new RequestPersistedEntriesV1({
5663
+ expectedReceiverSession: captured.capabilitySession,
5664
+ hashes: requestedHashes,
5665
+ }),
5666
+ {
5667
+ mode: new SilentDelivery({
5668
+ to: [peer],
5669
+ redundancy: 1,
5670
+ }),
5671
+ amount: 1,
5672
+ priority: delivery.priority,
5673
+ timeout: attemptTimeout,
5674
+ signal: roundSignal,
5675
+ },
5676
+ );
5677
+ })) ?? [];
5678
+ } catch {
5679
+ if (roundSignal.aborted) break;
5680
+ // A peer can disconnect or miss this retry while the overall
5681
+ // quorum deadline remains active. Replan on the next round.
5682
+ break;
5683
+ }
5684
+ if (!isRoundOwnershipCurrent()) {
5685
+ roundController.abort();
5686
+ break;
5687
+ }
5688
+ const current = this.persistedReceiptPeerSession(peer);
5689
+ if (
5690
+ !current ||
5691
+ current.capabilitySession !== captured.capabilitySession ||
5692
+ current.peerSession !== captured.peerSession
5693
+ ) {
5694
+ purgePeerDeliveryState(peer);
5695
+ break;
5696
+ }
5697
+ const requested = new Set(requestedHashes);
5698
+ const confirmed = new Set<string>();
5699
+ let receivedValidConfirmation = false;
5700
+ for (const result of responses) {
5701
+ if (
5702
+ !(result.response instanceof ConfirmEntriesMessage) ||
5703
+ result.from?.hashcode() !== peer ||
5704
+ result.message.header.session !== captured.capabilitySession
5705
+ ) {
5706
+ continue;
5707
+ }
5708
+ const unique = new Set(result.response.hashes);
5709
+ if (
5710
+ unique.size !== result.response.hashes.length ||
5711
+ [...unique].some((hash) => !requested.has(hash))
5712
+ ) {
5713
+ continue;
5714
+ }
5715
+ receivedValidConfirmation = true;
5716
+ for (const hash of unique) {
5717
+ confirmed.add(hash);
5718
+ carriedAcknowledgements.get(hash)?.set(peer, captured);
5719
+ }
5720
+ }
5721
+ if (receivedValidConfirmation) {
5722
+ const state = ensureRepairState();
5723
+ for (const hash of requested) {
5724
+ if (confirmed.has(hash)) {
5725
+ state.hashes.delete(hash);
5726
+ } else {
5727
+ state.hashes.add(hash);
5728
+ }
5729
+ }
5730
+ }
5731
+ }
5732
+ })()
5733
+ .then(() => undefined)
5734
+ .catch(() => undefined)
5735
+ .finally(() => requests.delete(request));
5736
+ requests.add(request);
5737
+ }
5738
+
5739
+ const roundComplete = async (): Promise<boolean> => {
5740
+ if (signal.aborted) {
5741
+ throw signal.reason ?? new AbortError();
5742
+ }
5743
+ if (!isRoundOwnershipCurrent()) return false;
5744
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5745
+ if (acknowledgements.size < minAcks) return false;
5746
+ }
5747
+ const validatedLeaders = await this.planPersistedDeliveryLeaders(
5748
+ entryArray,
5749
+ replicas,
5750
+ ownershipLifecycleController,
5751
+ );
5752
+ for (let index = 0; index < entryArray.length; index++) {
5753
+ if (signal.aborted || !isRoundOwnershipCurrent()) {
5754
+ if (signal.aborted) {
5755
+ throw signal.reason ?? new AbortError();
5756
+ }
5757
+ return false;
5758
+ }
5759
+ const hash = entryArray[index]!.hash;
5760
+ const leaders = validatedLeaders[index]!;
5761
+ let valid = 0;
5762
+ const acknowledgements = carriedAcknowledgements.get(hash)!;
5763
+ for (const [peer, captured] of acknowledgements) {
5764
+ const current = this.persistedReceiptPeerSession(peer);
5765
+ if (
5766
+ !current ||
5767
+ current.capabilitySession !== captured.capabilitySession ||
5768
+ current.peerSession !== captured.peerSession
5769
+ ) {
5770
+ purgePeerDeliveryState(peer);
5771
+ continue;
5772
+ }
5773
+ if (!leaders.has(peer)) {
5774
+ acknowledgements.delete(peer);
5775
+ continue;
5776
+ }
5777
+ if (++valid >= minAcks) {
5778
+ break;
5779
+ }
5780
+ }
5781
+ if (valid < minAcks) return false;
5782
+ }
5783
+ this.throwIfReplicationOwnershipLifecycleInactive(
5784
+ ownershipLifecycleController,
5785
+ );
5786
+ if (signal.aborted) {
5787
+ throw signal.reason ?? new AbortError();
5788
+ }
5789
+ return isRoundOwnershipCurrent();
5790
+ };
5791
+
5792
+ while (requests.size > 0) {
5793
+ await Promise.race(requests);
5794
+ if (await roundComplete()) {
5795
+ return;
5796
+ }
5797
+ }
5798
+ if (await roundComplete()) {
5799
+ return;
5800
+ }
5801
+ } finally {
5802
+ roundController.abort();
5803
+ void Promise.allSettled([...requests]);
5804
+ }
5805
+ if (isRoundOwnershipCurrent()) {
5806
+ initialTransferPending = false;
5807
+ }
5808
+ // Keep early retries fair and responsive, then let a caller's longer
5809
+ // overall deadline accommodate a genuinely slow durability barrier.
5810
+ maxAttemptMs = Math.min(
5811
+ MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
5812
+ maxAttemptMs * 2,
5813
+ );
5814
+ await this.waitPersistedReceiptRetry(
5815
+ signal,
5816
+ Math.max(
5817
+ 0,
5818
+ Math.min(
5819
+ PERSISTED_RECEIPT_RETRY_MS,
5820
+ deadline.deadline - Date.now(),
5821
+ ),
5822
+ ),
5823
+ );
5824
+ }
5825
+ } catch (error) {
5826
+ if (error instanceof PersistedDeliveryError) {
5827
+ throw error;
5828
+ }
5829
+ throw new PersistedDeliveryError(error, committedHashes);
5830
+ } finally {
5831
+ if (ownedDeadline) deadline.dispose();
5832
+ }
4780
5833
  }
4781
5834
 
4782
5835
  private async _appendDeliverToReplicators(
@@ -7839,13 +8892,7 @@ export class SharedLog<
7839
8892
  try {
7840
8893
  const fanoutService = getSharedLogFanoutService(this.node.services);
7841
8894
  if (fanoutService?.provide && !this._providerHandle) {
7842
- this._providerHandle = fanoutService.provide(
7843
- `shared-log|${this.topic}`,
7844
- {
7845
- ttlMs: 120_000,
7846
- announceIntervalMs: 60_000,
7847
- },
7848
- );
8895
+ this.ensureLogProviderHandle(fanoutService);
7849
8896
  }
7850
8897
  } catch {
7851
8898
  // Best-effort only.
@@ -8357,63 +9404,135 @@ export class SharedLog<
8357
9404
  }
8358
9405
  }
8359
9406
 
8360
- private async pushRepairEntries(
9407
+ private async pushEntryHashChunk(
8361
9408
  target: string,
8362
- entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
8363
- isStillCurrent: () => boolean = () => true,
8364
- signal?: AbortSignal,
8365
- ) {
8366
- if (!isStillCurrent()) {
8367
- return;
8368
- }
8369
- const hashes = [...entries.keys()];
8370
- if (
9409
+ chunk: string[],
9410
+ options: {
9411
+ acknowledge?: boolean;
9412
+ priority?: number;
9413
+ repairHint?: boolean;
9414
+ signal?: AbortSignal;
9415
+ },
9416
+ isStillCurrent: () => boolean,
9417
+ ): Promise<boolean> {
9418
+ if (!isStillCurrent()) return false;
9419
+ const useRaw =
8371
9420
  this._logProperties?.sync?.rawExchangeHeads === true &&
8372
- this.peerSupportsRawExchangeHeads(target)
8373
- ) {
8374
- const reserved = new Uint8Array(4);
8375
- reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
9421
+ this.peerSupportsRawExchangeHeads(target);
9422
+ if (useRaw) {
9423
+ const reserved = options.repairHint ? new Uint8Array(4) : undefined;
9424
+ if (reserved) reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8376
9425
  const sentMessages = await this.trySendFusedRawExchangeHeads(
8377
- hashes,
9426
+ chunk,
8378
9427
  [target],
8379
- { priority: SYNC_MESSAGE_PRIORITY, reserved, signal },
9428
+ {
9429
+ acknowledge: options.acknowledge,
9430
+ priority: options.priority,
9431
+ reserved,
9432
+ signal: options.signal,
9433
+ },
8380
9434
  );
8381
- if (!isStillCurrent()) {
8382
- return;
8383
- }
8384
- if (sentMessages !== undefined) {
8385
- return;
9435
+ if (!isStillCurrent()) return false;
9436
+ if (sentMessages === undefined) {
9437
+ for await (const message of createRawExchangeHeadsMessages(
9438
+ this.log,
9439
+ chunk,
9440
+ this._logProperties?.sync?.profile,
9441
+ )) {
9442
+ if (!isStillCurrent()) return false;
9443
+ if (options.repairHint) {
9444
+ message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
9445
+ }
9446
+ await this.rpc.send(message, {
9447
+ priority: options.priority,
9448
+ mode: options.acknowledge
9449
+ ? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
9450
+ : new SilentDelivery({ to: [target], redundancy: 1 }),
9451
+ signal: options.signal,
9452
+ });
9453
+ }
8386
9454
  }
8387
- for await (const message of createRawExchangeHeadsMessages(
9455
+ } else {
9456
+ for await (const message of createExchangeHeadsMessages(
8388
9457
  this.log,
8389
- hashes,
8390
- this._logProperties?.sync?.profile,
9458
+ chunk,
8391
9459
  )) {
8392
- if (!isStillCurrent()) {
8393
- return;
9460
+ if (!isStillCurrent()) return false;
9461
+ if (options.repairHint) {
9462
+ message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8394
9463
  }
8395
- message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8396
9464
  await this.rpc.send(message, {
8397
- priority: SYNC_MESSAGE_PRIORITY,
8398
- mode: new SilentDelivery({ to: [target], redundancy: 1 }),
8399
- signal,
9465
+ priority: options.priority,
9466
+ mode: options.acknowledge
9467
+ ? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
9468
+ : new SilentDelivery({ to: [target], redundancy: 1 }),
9469
+ signal: options.signal,
8400
9470
  });
8401
9471
  }
8402
- return;
8403
9472
  }
8404
- for await (const message of createExchangeHeadsMessages(this.log, hashes)) {
8405
- if (!isStillCurrent()) {
8406
- return;
8407
- }
8408
- message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8409
- await this.rpc.send(message, {
8410
- priority: SYNC_MESSAGE_PRIORITY,
8411
- mode: new SilentDelivery({ to: [target], redundancy: 1 }),
8412
- signal,
8413
- });
9473
+ return isStillCurrent();
9474
+ }
9475
+
9476
+ private async pushEntryHashes(
9477
+ target: string,
9478
+ hashes: string[],
9479
+ options: {
9480
+ acknowledge?: boolean;
9481
+ chunkTimeout?: () => number;
9482
+ chunkSize?: number;
9483
+ isStillCurrent?: () => boolean;
9484
+ onChunkAttempted?: (hashes: readonly string[]) => void;
9485
+ onChunkSent?: (hashes: readonly string[]) => Promise<boolean>;
9486
+ operationQueue?: PQueue;
9487
+ priority?: number;
9488
+ repairHint?: boolean;
9489
+ signal?: AbortSignal;
9490
+ },
9491
+ ) {
9492
+ const isStillCurrent = options.isStillCurrent ?? (() => true);
9493
+ if (!isStillCurrent()) return;
9494
+ const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
9495
+ for (let offset = 0; offset < hashes.length; offset += chunkSize) {
9496
+ const chunk = hashes.slice(offset, offset + chunkSize);
9497
+ const pushChunk = () => {
9498
+ options.onChunkAttempted?.(chunk);
9499
+ const chunkTimeout = options.chunkTimeout?.();
9500
+ const chunkSignal =
9501
+ chunkTimeout == null
9502
+ ? options.signal
9503
+ : AbortSignal.any([
9504
+ ...(options.signal ? [options.signal] : []),
9505
+ AbortSignal.timeout(Math.max(1, chunkTimeout)),
9506
+ ]);
9507
+ return this.pushEntryHashChunk(
9508
+ target,
9509
+ chunk,
9510
+ { ...options, signal: chunkSignal },
9511
+ isStillCurrent,
9512
+ );
9513
+ };
9514
+ const pushed = options.operationQueue
9515
+ ? await options.operationQueue.add(pushChunk)
9516
+ : await pushChunk();
9517
+ if (pushed !== true) return;
9518
+ if (options.onChunkSent && !(await options.onChunkSent(chunk))) return;
8414
9519
  }
8415
9520
  }
8416
9521
 
9522
+ private async pushRepairEntries(
9523
+ target: string,
9524
+ entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
9525
+ isStillCurrent: () => boolean = () => true,
9526
+ signal?: AbortSignal,
9527
+ ) {
9528
+ return this.pushEntryHashes(target, [...entries.keys()], {
9529
+ isStillCurrent,
9530
+ priority: SYNC_MESSAGE_PRIORITY,
9531
+ repairHint: true,
9532
+ signal,
9533
+ });
9534
+ }
9535
+
8417
9536
  private async sendRepairEntriesWithTransport(
8418
9537
  target: string,
8419
9538
  entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
@@ -10384,6 +11503,7 @@ export class SharedLog<
10384
11503
  removed: ShallowOrFullEntry<T>[];
10385
11504
  }> {
10386
11505
  this.throwIfNativeDurableCommitFailed();
11506
+ const persistedDelivery = this.getPersistedDeliveryOptions(options);
10387
11507
  const ownershipLifecycleController =
10388
11508
  this.captureReplicationOwnershipLifecycle();
10389
11509
  if (this._isAdaptiveReplicating) {
@@ -10394,18 +11514,73 @@ export class SharedLog<
10394
11514
  options,
10395
11515
  ownershipLifecycleController,
10396
11516
  );
10397
- const result = await this.log.append(data, appendOptions);
10398
- this.throwIfReplicationOwnershipLifecycleInactive(
10399
- ownershipLifecycleController,
10400
- );
10401
- await this.processLocalAppend(result.entry, result.removed, options, {
10402
- minReplicasValue,
10403
- ownershipLifecycleController,
10404
- });
10405
- this.throwIfReplicationOwnershipLifecycleInactive(
10406
- ownershipLifecycleController,
10407
- );
10408
- return result;
11517
+ let committedHashes: readonly string[] | undefined;
11518
+ if (persistedDelivery) {
11519
+ (appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
11520
+ hashes,
11521
+ ) => {
11522
+ // The lower log reports the exact hashes immediately after their
11523
+ // irreversible local mutation and before entry initialization, trim,
11524
+ // or change callbacks can reject the append.
11525
+ committedHashes = hashes;
11526
+ };
11527
+ }
11528
+ let persistedDeadline: PersistedDeliveryDeadline | undefined;
11529
+ const throwIfDeliveryAborted = () => {
11530
+ if (persistedDeadline?.signal.aborted) {
11531
+ throw persistedDeadline.signal.reason ?? new AbortError();
11532
+ }
11533
+ };
11534
+ try {
11535
+ const result = await this.log.append(data, appendOptions);
11536
+ committedHashes ??= [result.entry.hash];
11537
+ persistedDeadline = persistedDelivery
11538
+ ? this.createPersistedDeliveryDeadline(
11539
+ persistedDelivery,
11540
+ ownershipLifecycleController,
11541
+ 1,
11542
+ )
11543
+ : undefined;
11544
+ this.throwIfReplicationOwnershipLifecycleInactive(
11545
+ ownershipLifecycleController,
11546
+ );
11547
+ throwIfDeliveryAborted();
11548
+ await this.processLocalAppend(result.entry, result.removed, options, {
11549
+ minReplicasValue,
11550
+ ownershipLifecycleController,
11551
+ });
11552
+ throwIfDeliveryAborted();
11553
+ if (persistedDelivery && persistedDeadline) {
11554
+ await this.settlePersistedDelivery(
11555
+ [result.entry],
11556
+ minReplicasValue,
11557
+ persistedDelivery,
11558
+ ownershipLifecycleController,
11559
+ persistedDeadline,
11560
+ );
11561
+ }
11562
+ this.throwIfReplicationOwnershipLifecycleInactive(
11563
+ ownershipLifecycleController,
11564
+ );
11565
+ return result;
11566
+ } catch (error) {
11567
+ if (persistedDelivery && committedHashes) {
11568
+ throw new PersistedDeliveryError(error, committedHashes);
11569
+ }
11570
+ throw error;
11571
+ } finally {
11572
+ persistedDeadline?.dispose();
11573
+ }
11574
+ }
11575
+
11576
+ private rejectPersistedDeliveryOnTrustedLocalAppend(
11577
+ options?: SharedAppendOptions<T>,
11578
+ ): void {
11579
+ if (this.getPersistedDeliveryOptions(options)) {
11580
+ throw new Error(
11581
+ "trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit",
11582
+ );
11583
+ }
10409
11584
  }
10410
11585
 
10411
11586
  // Trusted local append path for callers that already validated the entry.
@@ -10417,6 +11592,7 @@ export class SharedLog<
10417
11592
  removed: ShallowOrFullEntry<T>[];
10418
11593
  }> {
10419
11594
  this.throwIfNativeDurableCommitFailed();
11595
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10420
11596
  const ownershipLifecycleController =
10421
11597
  this.captureReplicationOwnershipLifecycle();
10422
11598
  if (options?.canAppend || options?.onChange) {
@@ -10457,6 +11633,7 @@ export class SharedLog<
10457
11633
  skipMissingNextJoin?: boolean;
10458
11634
  resolveTrimmedEntries?: boolean;
10459
11635
  payloadData?: Uint8Array;
11636
+ localCommitEvidence?: TrustedLocalCommitEvidence;
10460
11637
  },
10461
11638
  ): Promise<{
10462
11639
  entry: Entry<T>;
@@ -10466,6 +11643,7 @@ export class SharedLog<
10466
11643
  appendCommit: PreparedLocalAppendCommit<R>;
10467
11644
  }> {
10468
11645
  this.throwIfNativeDurableCommitFailed();
11646
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10469
11647
  const ownershipLifecycleController =
10470
11648
  this.captureReplicationOwnershipLifecycle();
10471
11649
  if (options?.canAppend || options?.onChange) {
@@ -10480,6 +11658,10 @@ export class SharedLog<
10480
11658
  const { appendOptions, minReplicasValue } =
10481
11659
  this.createLogAppendOptions(options);
10482
11660
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
11661
+ attachTrustedLocalCommitEvidence(
11662
+ appendOptions,
11663
+ properties?.localCommitEvidence,
11664
+ );
10483
11665
  const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(
10484
11666
  data,
10485
11667
  appendOptions,
@@ -10489,6 +11671,11 @@ export class SharedLog<
10489
11671
  payloadData: properties?.payloadData,
10490
11672
  },
10491
11673
  );
11674
+ if (properties?.localCommitEvidence) {
11675
+ properties.localCommitEvidence.committedHashes.add(
11676
+ result.appendFacts.hash,
11677
+ );
11678
+ }
10492
11679
  this.throwIfReplicationOwnershipLifecycleInactive(
10493
11680
  ownershipLifecycleController,
10494
11681
  );
@@ -10737,6 +11924,9 @@ export class SharedLog<
10737
11924
  skipMissingNextJoin: properties?.skipMissingNextJoin,
10738
11925
  resolveTrimmedEntries: properties?.resolveTrimmedEntries,
10739
11926
  payloadData,
11927
+ ...(properties?.localCommitEvidence
11928
+ ? { localCommitEvidence: properties.localCommitEvidence }
11929
+ : undefined),
10740
11930
  });
10741
11931
  }
10742
11932
 
@@ -10747,6 +11937,7 @@ export class SharedLog<
10747
11937
  properties?: PreparedPayloadCommitOnlyProperties,
10748
11938
  ): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
10749
11939
  this.throwIfNativeDurableCommitFailed();
11940
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10750
11941
  if (options?.canAppend || options?.onChange) {
10751
11942
  throw new Error(
10752
11943
  "appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks",
@@ -10770,6 +11961,10 @@ export class SharedLog<
10770
11961
  const { appendOptions, minReplicasValue } =
10771
11962
  this.createLogAppendOptions(options);
10772
11963
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
11964
+ attachTrustedLocalCommitEvidence(
11965
+ appendOptions,
11966
+ properties?.localCommitEvidence,
11967
+ );
10773
11968
  const deferHeadCoordinatePersistence =
10774
11969
  this.shouldDeferHeadCoordinatePersistence(options);
10775
11970
  const nativeBackboneResult =
@@ -10819,6 +12014,7 @@ export class SharedLog<
10819
12014
  properties?: PreparedPayloadCommitOnlyProperties,
10820
12015
  ): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
10821
12016
  this.throwIfNativeDurableCommitFailed();
12017
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10822
12018
  if (options?.canAppend || options?.onChange) {
10823
12019
  throw new Error(
10824
12020
  "appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks",
@@ -10842,6 +12038,10 @@ export class SharedLog<
10842
12038
  const { appendOptions, minReplicasValue } =
10843
12039
  this.createLogAppendOptions(options);
10844
12040
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
12041
+ attachTrustedLocalCommitEvidence(
12042
+ appendOptions,
12043
+ properties?.localCommitEvidence,
12044
+ );
10845
12045
  const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(
10846
12046
  payloadData,
10847
12047
  appendOptions,
@@ -10881,6 +12081,11 @@ export class SharedLog<
10881
12081
  includeAppendFactsBytes: !deferHeadCoordinatePersistence,
10882
12082
  });
10883
12083
  return mapMaybePromise(resultMaybe, (result) => {
12084
+ if (result && properties?.localCommitEvidence) {
12085
+ properties.localCommitEvidence.committedHashes.add(
12086
+ result.appendFacts.hash,
12087
+ );
12088
+ }
10884
12089
  this.throwIfReplicationOwnershipLifecycleInactive(
10885
12090
  ownershipLifecycleController,
10886
12091
  );
@@ -11461,25 +12666,28 @@ export class SharedLog<
11461
12666
  // boundary separates the catch above from this statement and
11462
12667
  // `rollback` can no longer fire. Nothing downstream rolls back
11463
12668
  // (the retire below only warns), so the token is terminal here.
11464
- this._coordinates.settleResidentCoordinateSnapshot(
11465
- lowerPublicationRollback?.coordinateEntries,
11466
- );
11467
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
11468
- prepared.appendFacts,
11469
- prepared.removed,
11470
- prepared.materializeEntry,
11471
- {
11472
- removedHashes: prepared.removedHashes,
11473
- removedGids: prepared.removedGids,
11474
- },
11475
- );
11476
- try {
11477
- await this.completeNativeStrictDurableTransaction(
11478
- nativeStrictTransaction,
12669
+ if (properties?.localCommitEvidence) {
12670
+ properties.localCommitEvidence.committedHashes.add(
12671
+ prepared.appendFacts.hash,
11479
12672
  );
11480
- } catch (error) {
11481
- warn(`Failed to retire committed native intent: ${String(error)}`);
11482
12673
  }
12674
+ await this.finishCommittedNativeStrictDurableTransaction(
12675
+ nativeStrictTransaction,
12676
+ () => {
12677
+ this._coordinates.settleResidentCoordinateSnapshot(
12678
+ lowerPublicationRollback?.coordinateEntries,
12679
+ );
12680
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
12681
+ prepared.appendFacts,
12682
+ prepared.removed,
12683
+ prepared.materializeEntry,
12684
+ {
12685
+ removedHashes: prepared.removedHashes,
12686
+ removedGids: prepared.removedGids,
12687
+ },
12688
+ );
12689
+ },
12690
+ );
11483
12691
  return finishResult;
11484
12692
  });
11485
12693
  }
@@ -12112,26 +13320,29 @@ export class SharedLog<
12112
13320
  // Success seam: the finalizer acknowledge above is the last
12113
13321
  // await inside the protected try, so `rollback` can no
12114
13322
  // longer fire and the retire below only warns.
12115
- this._coordinates.settleResidentCoordinateSnapshot(
12116
- rollbackCoordinateEntries,
12117
- );
12118
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
12119
- prepared.appendFacts,
12120
- prepared.removed,
12121
- prepared.materializeEntry,
12122
- {
12123
- forgetNativeCoordinates: false,
12124
- removedHashes: prepared.removedHashes,
12125
- removedGids: prepared.removedGids,
12126
- },
12127
- );
12128
- try {
12129
- await this.completeNativeStrictDurableTransaction(
12130
- nativeStrictTransaction,
13323
+ if (properties?.localCommitEvidence) {
13324
+ properties.localCommitEvidence.committedHashes.add(
13325
+ prepared.appendFacts.hash,
12131
13326
  );
12132
- } catch (error) {
12133
- warn(`Failed to retire committed native intent: ${String(error)}`);
12134
13327
  }
13328
+ await this.finishCommittedNativeStrictDurableTransaction(
13329
+ nativeStrictTransaction,
13330
+ () => {
13331
+ this._coordinates.settleResidentCoordinateSnapshot(
13332
+ rollbackCoordinateEntries,
13333
+ );
13334
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
13335
+ prepared.appendFacts,
13336
+ prepared.removed,
13337
+ prepared.materializeEntry,
13338
+ {
13339
+ forgetNativeCoordinates: false,
13340
+ removedHashes: prepared.removedHashes,
13341
+ removedGids: prepared.removedGids,
13342
+ },
13343
+ );
13344
+ },
13345
+ );
12135
13346
  if (
12136
13347
  commitBlocksInBackbone &&
12137
13348
  !runtimeOnlyCoordinates &&
@@ -13224,73 +14435,73 @@ export class SharedLog<
13224
14435
  nativeStrictTransaction,
13225
14436
  );
13226
14437
  });
13227
- this.throwIfReplicationOwnershipLifecycleInactive(
13228
- ownershipLifecycleController,
13229
- );
13230
14438
  } catch (error) {
13231
14439
  return rollbackBatch(error);
13232
14440
  }
13233
14441
  // Success seam: `rollbackBatch` has exactly one call site (the catch
13234
14442
  // above), and everything from here on escapes without any rollback.
13235
- this._coordinates.settleResidentCoordinateSnapshot(
13236
- batchCoordinateRollback,
13237
- );
13238
-
13239
- this.throwIfReplicationOwnershipLifecycleInactive(
13240
- ownershipLifecycleController,
13241
- );
13242
- const appendCommits: PreparedLocalAppendCommit<R>[] = [];
13243
- for (let i = 0; i < coordinateRows.length; i++) {
13244
- const {
13245
- facts,
13246
- backboneAppend,
13247
- coordinateFields,
13248
- plannedCoordinateDeleteHashes,
13249
- } = coordinateRows[i]!;
13250
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
13251
- facts,
13252
- [],
13253
- appended.materializeEntries[i]!,
13254
- {
13255
- forgetNativeCoordinates: false,
13256
- removedHashes: plannedCoordinateDeleteHashes,
13257
- removedGids:
13258
- backboneAppend.trimmedGids ??
13259
- (backboneAppend.trimmed.length > 0
13260
- ? backboneAppend.trimmed.map((entry) => entry.gid)
13261
- : undefined),
13262
- },
13263
- );
13264
- if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
13265
- this.remoteBlocks.notifyStoredDeferred(facts.hash);
14443
+ if (properties?.localCommitEvidence) {
14444
+ for (const facts of appended.appendFacts) {
14445
+ properties.localCommitEvidence.committedHashes.add(facts.hash);
13266
14446
  }
13267
- const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
13268
- facts,
13269
- {
13270
- hashNumber: backboneAppend.coordinate.hashNumber as NumberFromType<R>,
13271
- coordinateFields,
13272
- },
13273
- );
13274
- appendCommit.nativeBackboneDocumentIndexCommitted = true;
13275
- appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
13276
- appended.documentTrimmedHeadsProcessed?.[i];
13277
- appendCommit.documentPreviousContext =
13278
- backboneAppend.documentPreviousContext;
13279
- appendCommits.push(appendCommit);
13280
14447
  }
13281
- try {
13282
- await this.completeNativeStrictDurableTransaction(
14448
+ const appendCommits =
14449
+ await this.finishCommittedNativeStrictDurableTransaction(
13283
14450
  nativeStrictTransaction,
14451
+ () => {
14452
+ this._coordinates.settleResidentCoordinateSnapshot(
14453
+ batchCoordinateRollback,
14454
+ );
14455
+ this.throwIfReplicationOwnershipLifecycleInactive(
14456
+ ownershipLifecycleController,
14457
+ );
14458
+ const commits: PreparedLocalAppendCommit<R>[] = [];
14459
+ for (let i = 0; i < coordinateRows.length; i++) {
14460
+ const {
14461
+ facts,
14462
+ backboneAppend,
14463
+ coordinateFields,
14464
+ plannedCoordinateDeleteHashes,
14465
+ } = coordinateRows[i]!;
14466
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
14467
+ facts,
14468
+ [],
14469
+ appended.materializeEntries[i]!,
14470
+ {
14471
+ forgetNativeCoordinates: false,
14472
+ removedHashes: plannedCoordinateDeleteHashes,
14473
+ removedGids:
14474
+ backboneAppend.trimmedGids ??
14475
+ (backboneAppend.trimmed.length > 0
14476
+ ? backboneAppend.trimmed.map((entry) => entry.gid)
14477
+ : undefined),
14478
+ },
14479
+ );
14480
+ if (
14481
+ !runtimeOnlyCoordinates &&
14482
+ this.remoteBlocks.hasNotifyStoredHook()
14483
+ ) {
14484
+ this.remoteBlocks.notifyStoredDeferred(facts.hash);
14485
+ }
14486
+ const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
14487
+ facts,
14488
+ {
14489
+ hashNumber: backboneAppend.coordinate
14490
+ .hashNumber as NumberFromType<R>,
14491
+ coordinateFields,
14492
+ },
14493
+ );
14494
+ appendCommit.nativeBackboneDocumentIndexCommitted = true;
14495
+ appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
14496
+ appended.documentTrimmedHeadsProcessed?.[i];
14497
+ appendCommit.documentPreviousContext =
14498
+ backboneAppend.documentPreviousContext;
14499
+ commits.push(appendCommit);
14500
+ }
14501
+ return commits;
14502
+ },
14503
+ () => this.isRepairLifecycleActive(ownershipLifecycleController),
13284
14504
  );
13285
- this.throwIfReplicationOwnershipLifecycleInactive(
13286
- ownershipLifecycleController,
13287
- );
13288
- } catch (error) {
13289
- if (!this.isRepairLifecycleActive(ownershipLifecycleController)) {
13290
- throw error;
13291
- }
13292
- warn(`Failed to retire committed native intent: ${String(error)}`);
13293
- }
13294
14505
  this.throwIfReplicationOwnershipLifecycleInactive(
13295
14506
  ownershipLifecycleController,
13296
14507
  );
@@ -13322,6 +14533,7 @@ export class SharedLog<
13322
14533
  | undefined
13323
14534
  > {
13324
14535
  this.throwIfNativeDurableCommitFailed();
14536
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
13325
14537
  if (data.length === 0) {
13326
14538
  return { entries: [], removed: [], appendCommits: [] };
13327
14539
  }
@@ -13339,6 +14551,10 @@ export class SharedLog<
13339
14551
  const { appendOptions, minReplicasValue } =
13340
14552
  this.createLogAppendOptions(options);
13341
14553
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
14554
+ attachTrustedLocalCommitEvidence(
14555
+ appendOptions,
14556
+ properties?.localCommitEvidence,
14557
+ );
13342
14558
  const nativeBackboneBatch =
13343
14559
  await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(
13344
14560
  data,
@@ -13361,6 +14577,11 @@ export class SharedLog<
13361
14577
  payloadDatas: properties?.payloadDatas,
13362
14578
  nexts: properties?.nexts,
13363
14579
  });
14580
+ if (result && properties?.localCommitEvidence) {
14581
+ for (const facts of result.appendFacts) {
14582
+ properties.localCommitEvidence.committedHashes.add(facts.hash);
14583
+ }
14584
+ }
13364
14585
  this.throwIfReplicationOwnershipLifecycleInactive(
13365
14586
  ownershipLifecycleController,
13366
14587
  );
@@ -13490,6 +14711,9 @@ export class SharedLog<
13490
14711
  nativeBackboneDocumentIndexes:
13491
14712
  properties?.nativeBackboneDocumentIndexes,
13492
14713
  retainMaterializationBytes: properties?.retainMaterializationBytes,
14714
+ ...(properties?.localCommitEvidence
14715
+ ? { localCommitEvidence: properties.localCommitEvidence }
14716
+ : undefined),
13493
14717
  },
13494
14718
  );
13495
14719
  }
@@ -13502,9 +14726,15 @@ export class SharedLog<
13502
14726
  removed: ShallowOrFullEntry<T>[];
13503
14727
  }> {
13504
14728
  this.throwIfNativeDurableCommitFailed();
14729
+ const persistedDelivery = this.getPersistedDeliveryOptions(options);
13505
14730
  if (data.length === 0) {
13506
14731
  return { entries: [], removed: [] };
13507
14732
  }
14733
+ if (persistedDelivery) {
14734
+ throw new Error(
14735
+ "persisted delivery is not supported for chained appendMany; use independent document puts",
14736
+ );
14737
+ }
13508
14738
  const ownershipLifecycleController =
13509
14739
  this.captureReplicationOwnershipLifecycle();
13510
14740
  if (this._isAdaptiveReplicating) {
@@ -13581,6 +14811,123 @@ export class SharedLog<
13581
14811
  return result;
13582
14812
  }
13583
14813
 
14814
+ /**
14815
+ * Deliver entries that were already committed locally, then wait for the
14816
+ * requested persisted remote quorum. This is the post-commit seam used by
14817
+ * higher-level transactional/batched writers so a receipt timeout never
14818
+ * rolls back or hides their successful local commit.
14819
+ */
14820
+ private async deliverPersistedPlanningEntries(
14821
+ resolveEntries: () => PersistedDeliveryPlanningEntry<T, R>[],
14822
+ committedHashes: string[],
14823
+ options: SharedAppendOptions<T>,
14824
+ ): Promise<void> {
14825
+ let persistedDeadline: PersistedDeliveryDeadline | undefined;
14826
+ try {
14827
+ const delivery = this.getPersistedDeliveryOptions(options);
14828
+ if (!delivery) {
14829
+ throw new Error(
14830
+ 'deliverPersistedEntries requires reliability="persisted"',
14831
+ );
14832
+ }
14833
+ const ownershipLifecycleController =
14834
+ this.captureReplicationOwnershipLifecycle();
14835
+ const deadline = this.createPersistedDeliveryDeadline(
14836
+ delivery,
14837
+ ownershipLifecycleController,
14838
+ committedHashes.length,
14839
+ );
14840
+ persistedDeadline = deadline;
14841
+ const throwIfDeliveryAborted = () => {
14842
+ if (deadline.signal.aborted) {
14843
+ throw deadline.signal.reason ?? new AbortError();
14844
+ }
14845
+ if (Date.now() >= deadline.deadline) {
14846
+ throw new TimeoutError(
14847
+ `Timed out waiting for ${Math.floor(delivery.minAcks!)} persisted remote replicas.`,
14848
+ );
14849
+ }
14850
+ };
14851
+ const entries = resolveEntries();
14852
+ throwIfDeliveryAborted();
14853
+ if (entries.length === 0) return;
14854
+ const { minReplicasValue } = this.createLogAppendOptions(
14855
+ options,
14856
+ ownershipLifecycleController,
14857
+ );
14858
+ throwIfDeliveryAborted();
14859
+ await this.settlePersistedDelivery(
14860
+ entries,
14861
+ minReplicasValue,
14862
+ delivery,
14863
+ ownershipLifecycleController,
14864
+ deadline,
14865
+ true,
14866
+ );
14867
+ } catch (error) {
14868
+ throw new PersistedDeliveryError(error, committedHashes);
14869
+ } finally {
14870
+ persistedDeadline?.dispose();
14871
+ }
14872
+ }
14873
+
14874
+ private createPersistedDeliveryPlanningEntries(
14875
+ appendCommits: PreparedLocalAppendCommit<R>[],
14876
+ materializeEntries: () => Entry<T>[],
14877
+ ): PersistedDeliveryPlanningEntry<T, R>[] {
14878
+ let materializedEntries: Entry<T>[] | undefined;
14879
+ const requiresFullEntries =
14880
+ this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
14881
+ const getMaterializedEntry = (
14882
+ appendCommit: PreparedLocalAppendCommit<R>,
14883
+ index: number,
14884
+ ) => {
14885
+ materializedEntries ??= materializeEntries();
14886
+ const entry = materializedEntries[index];
14887
+ if (!entry || entry.hash !== appendCommit.hash) {
14888
+ throw new Error(
14889
+ `Persisted delivery materializer did not return committed entry ${appendCommit.hash}`,
14890
+ );
14891
+ }
14892
+ return entry;
14893
+ };
14894
+ return appendCommits.map((appendCommit, index) => {
14895
+ if (!requiresFullEntries && appendCommit.coordinateFields) {
14896
+ return this._coordinates.materializeResidentCoordinateEntry(
14897
+ appendCommit.coordinateFields,
14898
+ );
14899
+ }
14900
+ return getMaterializedEntry(appendCommit, index);
14901
+ });
14902
+ }
14903
+
14904
+ private deliverPersistedAppendCommits(
14905
+ appendCommits: PreparedLocalAppendCommit<R>[],
14906
+ materializeEntries: () => Entry<T>[],
14907
+ options: SharedAppendOptions<T>,
14908
+ ): Promise<void> {
14909
+ return this.deliverPersistedPlanningEntries(
14910
+ () =>
14911
+ this.createPersistedDeliveryPlanningEntries(
14912
+ appendCommits,
14913
+ materializeEntries,
14914
+ ),
14915
+ appendCommits.map((appendCommit) => appendCommit.hash),
14916
+ options,
14917
+ );
14918
+ }
14919
+
14920
+ async deliverPersistedEntries(
14921
+ entries: Entry<T>[],
14922
+ options: SharedAppendOptions<T>,
14923
+ ): Promise<void> {
14924
+ return this.deliverPersistedPlanningEntries(
14925
+ () => entries,
14926
+ entries.map((entry) => entry.hash),
14927
+ options,
14928
+ );
14929
+ }
14930
+
13584
14931
  private canCoalesceLocalAppendMany(
13585
14932
  entries: Entry<T>[],
13586
14933
  options?: SharedAppendOptions<T>,
@@ -14722,6 +16069,9 @@ export class SharedLog<
14722
16069
  this._peerSyncCapabilities = new Map();
14723
16070
  this._peerSyncCapabilitySessions = new Map();
14724
16071
  this._peerSyncCapabilityTimestamps = new Map();
16072
+ this._persistedReceiptStorage = undefined;
16073
+ this._persistedReceiptRequestsInFlight = new Map();
16074
+ this._persistedReceiptRequestsInFlightTotal = 0;
14725
16075
  this._liveRawGossipBatches = new Map();
14726
16076
  this._liveRawGossipFlushScheduled = false;
14727
16077
  this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
@@ -14864,6 +16214,43 @@ export class SharedLog<
14864
16214
 
14865
16215
  const fanoutService = getSharedLogFanoutService(this.node.services);
14866
16216
  const blockProviderNamespace = (cid: string) => `cid:${cid}`;
16217
+ const logProviderNamespace = `shared-log|${this.topic}`;
16218
+ const announceBlockProvider = async (cid: string): Promise<void> => {
16219
+ try {
16220
+ await fanoutService?.announceProvider(blockProviderNamespace(cid), {
16221
+ ttlMs: 120_000,
16222
+ bootstrapMaxPeers: 2,
16223
+ });
16224
+ } catch {
16225
+ // Provider publication is best-effort.
16226
+ }
16227
+ };
16228
+ const announceBlockProviders = async (cids: string[]): Promise<void> => {
16229
+ const batchedAnnounce = fanoutService?.announceProviders;
16230
+ if (typeof batchedAnnounce === "function") {
16231
+ const namespaces = function* () {
16232
+ for (const cid of cids) yield blockProviderNamespace(cid);
16233
+ };
16234
+ await batchedAnnounce.call(fanoutService, namespaces(), {
16235
+ ttlMs: 120_000,
16236
+ bootstrapMaxPeers: 2,
16237
+ });
16238
+ return;
16239
+ }
16240
+
16241
+ // Tolerate a skewed runtime that supplied an older FanoutTree service.
16242
+ let nextIndex = 0;
16243
+ const worker = async () => {
16244
+ for (;;) {
16245
+ const index = nextIndex++;
16246
+ if (index >= cids.length) return;
16247
+ await announceBlockProvider(cids[index]!);
16248
+ }
16249
+ };
16250
+ await Promise.all(
16251
+ Array.from({ length: Math.min(8, cids.length) }, () => worker()),
16252
+ );
16253
+ };
14867
16254
  const [replicationIndex, logIndex] = await Promise.all([
14868
16255
  logScope.scope("replication"),
14869
16256
  logScope.scope("log"),
@@ -14963,17 +16350,23 @@ export class SharedLog<
14963
16350
 
14964
16351
  let directoryProviders: string[] = [];
14965
16352
  try {
14966
- directoryProviders =
14967
- (await fanoutService?.queryProviders(
14968
- blockProviderNamespace(cid),
14969
- {
14970
- want: maxPeers,
14971
- timeoutMs: 2_000,
14972
- queryTimeoutMs: 500,
14973
- bootstrapMaxPeers: 2,
14974
- signal: opts?.signal,
14975
- },
14976
- )) ?? [];
16353
+ const query = (namespace: string) =>
16354
+ fanoutService?.queryProviders(namespace, {
16355
+ want: maxPeers,
16356
+ timeoutMs: 2_000,
16357
+ queryTimeoutMs: 500,
16358
+ bootstrapMaxPeers: 2,
16359
+ signal: opts?.signal,
16360
+ }) ?? Promise.resolve([]);
16361
+ const results = await Promise.allSettled([
16362
+ query(blockProviderNamespace(cid)),
16363
+ query(logProviderNamespace),
16364
+ ]);
16365
+ for (const result of results) {
16366
+ if (result.status === "fulfilled") {
16367
+ directoryProviders.push(...result.value);
16368
+ }
16369
+ }
14977
16370
  } catch {
14978
16371
  // Ignore discovery failures; local evidence remains usable.
14979
16372
  }
@@ -14981,11 +16374,7 @@ export class SharedLog<
14981
16374
  const selected: string[] = [];
14982
16375
  const selectedSet = new Set<string>();
14983
16376
  const add = (peer: string | undefined) => {
14984
- if (
14985
- !peer ||
14986
- selectedSet.has(peer) ||
14987
- selected.length >= maxPeers
14988
- ) {
16377
+ if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
14989
16378
  return;
14990
16379
  }
14991
16380
  selectedSet.add(peer);
@@ -15003,31 +16392,46 @@ export class SharedLog<
15003
16392
  return selected;
15004
16393
  },
15005
16394
  watchProviders: fanoutService
15006
- ? (cid, opts) =>
15007
- fanoutService.watchProviders(blockProviderNamespace(cid), {
15008
- signal: opts.signal,
15009
- want: 8,
15010
- ttlMs: 10_000,
15011
- renewIntervalMs: 5_000,
15012
- bootstrapMaxPeers: 2,
15013
- onProviders: (providers) =>
15014
- opts.onProviders(providers.map((provider) => provider.hash)),
15015
- })
15016
- : undefined,
15017
- onPut: fanoutService
15018
- ? async (cid) => {
15019
- // Best-effort directory announce for "get without remote.from" workflows.
16395
+ ? (cid, opts) => {
16396
+ const watch = (namespace: string) =>
16397
+ fanoutService.watchProviders(namespace, {
16398
+ signal: opts.signal,
16399
+ want: 8,
16400
+ ttlMs: 10_000,
16401
+ renewIntervalMs: 5_000,
16402
+ bootstrapMaxPeers: 2,
16403
+ onProviders: (providers) =>
16404
+ opts.onProviders(providers.map((provider) => provider.hash)),
16405
+ });
16406
+ const cidWatch = watch(blockProviderNamespace(cid));
15020
16407
  try {
15021
- await fanoutService.announceProvider(
15022
- blockProviderNamespace(cid),
15023
- {
15024
- ttlMs: 120_000,
15025
- bootstrapMaxPeers: 2,
16408
+ const logWatch = watch(logProviderNamespace);
16409
+ return {
16410
+ close: () => {
16411
+ cidWatch.close();
16412
+ logWatch.close();
15026
16413
  },
15027
- );
16414
+ };
16415
+ } catch (error) {
16416
+ cidWatch.close();
16417
+ throw error;
16418
+ }
16419
+ }
16420
+ : undefined,
16421
+ onPut: fanoutService ? announceBlockProvider : undefined,
16422
+ onPutMany: fanoutService
16423
+ ? (cids) => {
16424
+ // A renewable log-wide provider lease makes every CID in a
16425
+ // stored batch discoverable to current readers. Retain the
16426
+ // per-CID directory publications for released readers that only
16427
+ // know the legacy namespace; the bounded workers avoid creating
16428
+ // one in-flight promise per block.
16429
+ try {
16430
+ this.ensureLogProviderHandle(fanoutService);
15028
16431
  } catch {
15029
16432
  // ignore announce failures
15030
16433
  }
16434
+ return announceBlockProviders(cids);
15031
16435
  }
15032
16436
  : undefined,
15033
16437
  });
@@ -15279,6 +16683,7 @@ export class SharedLog<
15279
16683
  },
15280
16684
  indexer: logIndex,
15281
16685
  });
16686
+ this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
15282
16687
  try {
15283
16688
  const recovered =
15284
16689
  await this.recoverNativeStrictDurableTransactionIntent();
@@ -16077,13 +17482,12 @@ export class SharedLog<
16077
17482
  this.topic,
16078
17483
  );
16079
17484
  // We do this here, because these calls requires this.closed == false
16080
- void this.pruneOfflineReplicators()
16081
- .catch((error) => {
16082
- if (isNotStartedError(error as Error)) {
16083
- return;
16084
- }
16085
- logger.error(error);
16086
- });
17485
+ void this.pruneOfflineReplicators().catch((error) => {
17486
+ if (isNotStartedError(error as Error)) {
17487
+ return;
17488
+ }
17489
+ logger.error(error);
17490
+ });
16087
17491
 
16088
17492
  this._liveness.startReplicatorLivenessSweep();
16089
17493
 
@@ -16402,7 +17806,8 @@ export class SharedLog<
16402
17806
  for (const peer of reachable) {
16403
17807
  if (confirmed?.has(peer)) liveEvidenceTiers[0].push(peer);
16404
17808
  else if (contacted?.has(peer)) liveEvidenceTiers[1].push(peer);
16405
- else if (this.uniqueReplicators.has(peer)) liveEvidenceTiers[2].push(peer);
17809
+ else if (this.uniqueReplicators.has(peer))
17810
+ liveEvidenceTiers[2].push(peer);
16406
17811
  }
16407
17812
  const liveKnown: string[] = [];
16408
17813
  for (let tier = 0; tier < liveEvidenceTiers.length; tier++) {
@@ -16972,8 +18377,7 @@ export class SharedLog<
16972
18377
  ): Promise<void> {
16973
18378
  if (!gids) return;
16974
18379
  const candidates = [...new Set(gids)].filter(
16975
- (gid) =>
16976
- history === this._gidPeersHistory && !!gid && history.has(gid),
18380
+ (gid) => history === this._gidPeersHistory && !!gid && history.has(gid),
16977
18381
  );
16978
18382
  if (candidates.length === 0) return;
16979
18383
  const hasHeads = await this.hasAnyHeadForGidSets(
@@ -17007,9 +18411,7 @@ export class SharedLog<
17007
18411
  for (const gid of gids) {
17008
18412
  if (!gid || !history.has(gid)) continue;
17009
18413
  if (state.pending.has(gid)) continue;
17010
- if (
17011
- state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY
17012
- ) {
18414
+ if (state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY) {
17013
18415
  // History is only a suppression memo. Under sustained synchronous
17014
18416
  // production, forgetting an overflow row is the bounded, safe fallback:
17015
18417
  // it can cause redundant delivery but cannot affect ownership or data.
@@ -17808,6 +19210,9 @@ export class SharedLog<
17808
19210
  this._peerSyncCapabilities?.clear();
17809
19211
  this._peerSyncCapabilitySessions?.clear();
17810
19212
  this._peerSyncCapabilityTimestamps?.clear();
19213
+ this._persistedReceiptStorage = undefined;
19214
+ this._persistedReceiptRequestsInFlight?.clear();
19215
+ this._persistedReceiptRequestsInFlightTotal = 0;
17811
19216
  this._v2Receive?.clearForClose();
17812
19217
  this._v2Send?.clearForClose();
17813
19218
  this._liveRawGossipBatches?.clear();
@@ -18580,7 +19985,11 @@ export class SharedLog<
18580
19985
  async onMessage(
18581
19986
  msg: TransportMessage,
18582
19987
  context: RequestContext,
18583
- ): Promise<void> {
19988
+ ): Promise<void>;
19989
+ async onMessage(
19990
+ msg: TransportMessage,
19991
+ context: RequestContext,
19992
+ ): Promise<any> {
18584
19993
  const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
18585
19994
  ? msg
18586
19995
  : undefined;
@@ -18656,6 +20065,7 @@ export class SharedLog<
18656
20065
  if (
18657
20066
  !context.from.equals(this.node.identity.publicKey) &&
18658
20067
  !(msg instanceof RequestReplicationInfoV2Message) &&
20068
+ !(msg instanceof RequestPersistedEntriesV1) &&
18659
20069
  !isReplicationInfoV2Message(msg)
18660
20070
  ) {
18661
20071
  this._liveness.markReplicatorActivity(receiveFromHash);
@@ -20102,24 +21512,30 @@ export class SharedLog<
20102
21512
  );
20103
21513
  }
20104
21514
  if (syncProfile) {
20105
- emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
20106
- name: "sharedLog.receive.coordinatePersist",
20107
- component: "shared-log",
20108
- entries: entriesToPersist.length,
20109
- messages: 1,
20110
- details: {
20111
- reusedLeaderPlans: reusableCoordinatePersistItemCount,
20112
- nativeBackboneOnly:
20113
- nativeBackboneOnlyPersistedHashes?.size ?? 0,
21515
+ emitSyncProfileDuration(
21516
+ syncProfile,
21517
+ coordinatePersistStartedAt,
21518
+ {
21519
+ name: "sharedLog.receive.coordinatePersist",
21520
+ component: "shared-log",
21521
+ entries: entriesToPersist.length,
21522
+ messages: 1,
21523
+ details: {
21524
+ reusedLeaderPlans: reusableCoordinatePersistItemCount,
21525
+ nativeBackboneOnly:
21526
+ nativeBackboneOnlyPersistedHashes?.size ?? 0,
21527
+ },
20114
21528
  },
20115
- });
21529
+ );
20116
21530
  }
20117
21531
  for (const hash of admittedHashes) {
20118
21532
  confirmedHashes.add(hash);
20119
21533
  }
20120
21534
  const checkedPruneStartedAt = syncProfileStart(syncProfile);
20121
21535
  const ownershipChangedDuringReceive =
20122
- !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
21536
+ !this.isReceiveOwnershipSnapshotStable(
21537
+ receiveOwnershipRevision,
21538
+ );
20123
21539
  if (ownershipChangedDuringReceive) {
20124
21540
  const freshAuditRevision =
20125
21541
  this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
@@ -20296,7 +21712,13 @@ export class SharedLog<
20296
21712
  };
20297
21713
  // The prelude already threw when `context.from` was missing.
20298
21714
  const laneRequestContext = context as ReceiveRequestContext;
20299
- if (msg instanceof RequestIPruneV2) {
21715
+ if (msg instanceof RequestPersistedEntriesV1) {
21716
+ return await this.handleRequestPersistedEntriesV1(
21717
+ msg,
21718
+ laneRequestContext,
21719
+ lane,
21720
+ );
21721
+ } else if (msg instanceof RequestIPruneV2) {
20300
21722
  await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
20301
21723
  } else if (msg instanceof ResponseIPruneV2) {
20302
21724
  await this.handleResponseIPruneV2(msg, laneRequestContext, lane);
@@ -20503,6 +21925,322 @@ export class SharedLog<
20503
21925
  // wire-stash release, lease release, durable-poison recheck).
20504
21926
  // -----------------------------------------------------------------
20505
21927
 
21928
+ private isPersistedReceiptRequestSessionCurrent(
21929
+ request: RequestPersistedEntriesV1,
21930
+ context: ReceiveRequestContext,
21931
+ lane: ReceiveLaneContext,
21932
+ ): boolean {
21933
+ const session = lane.session;
21934
+ return (
21935
+ !!this._persistedReceiptStorage &&
21936
+ !context.from.equals(this.node.identity.publicKey) &&
21937
+ request.expectedReceiverSession === this.ownTransportSession() &&
21938
+ session !== null &&
21939
+ session.phase === "open" &&
21940
+ this._peerSessions.isCurrent(lane.fromHash, session) &&
21941
+ !this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
21942
+ this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
21943
+ this._peerSyncCapabilitySessions.get(lane.fromHash) ===
21944
+ context.message.header.session &&
21945
+ this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
21946
+ this.isRepairLifecycleActive(lane.ownershipLifecycleController)
21947
+ );
21948
+ }
21949
+
21950
+ private admitPersistedReceiptIngress(
21951
+ peer: string,
21952
+ transportSession: bigint,
21953
+ hashCount: number,
21954
+ now = Date.now(),
21955
+ ): boolean {
21956
+ // Charge malformed empty/oversized vectors too. The request was already
21957
+ // decoded by the transport, so letting an invalid shape bypass this bucket
21958
+ // would leave an authenticated peer with an unmetered validation/logging
21959
+ // path. Clamp only the accounting cost; shape validation still rejects the
21960
+ // request below.
21961
+ const hashCost = Math.max(
21962
+ 1,
21963
+ Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount),
21964
+ );
21965
+ let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
21966
+ if (!nodeBudget) {
21967
+ nodeBudget = {
21968
+ requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
21969
+ hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
21970
+ refilledAt: now,
21971
+ peerSessions: new Map(),
21972
+ };
21973
+ persistedReceiptIngressBudgets.set(this.node, nodeBudget);
21974
+ }
21975
+ refillPersistedReceiptIngressBucket(
21976
+ nodeBudget,
21977
+ now,
21978
+ PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
21979
+ PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
21980
+ PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND,
21981
+ PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND,
21982
+ );
21983
+
21984
+ const peerSessionKey = `${peer}\0${transportSession}`;
21985
+ let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
21986
+ if (!peerBudget) {
21987
+ while (
21988
+ nodeBudget.peerSessions.size >=
21989
+ MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
21990
+ ) {
21991
+ const oldest = nodeBudget.peerSessions.keys().next().value;
21992
+ if (oldest === undefined) break;
21993
+ nodeBudget.peerSessions.delete(oldest);
21994
+ }
21995
+ peerBudget = {
21996
+ requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
21997
+ hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
21998
+ refilledAt: now,
21999
+ };
22000
+ nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
22001
+ } else {
22002
+ // Refresh insertion order so bounded eviction prefers inactive sessions.
22003
+ nodeBudget.peerSessions.delete(peerSessionKey);
22004
+ nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
22005
+ }
22006
+ refillPersistedReceiptIngressBucket(
22007
+ peerBudget,
22008
+ now,
22009
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
22010
+ PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
22011
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
22012
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
22013
+ );
22014
+
22015
+ if (
22016
+ nodeBudget.requestTokens < 1 ||
22017
+ nodeBudget.hashTokens < hashCost ||
22018
+ peerBudget.requestTokens < 1 ||
22019
+ peerBudget.hashTokens < hashCost
22020
+ ) {
22021
+ return false;
22022
+ }
22023
+ nodeBudget.requestTokens -= 1;
22024
+ nodeBudget.hashTokens -= hashCost;
22025
+ peerBudget.requestTokens -= 1;
22026
+ peerBudget.hashTokens -= hashCost;
22027
+ return true;
22028
+ }
22029
+
22030
+ private async handleRequestPersistedEntriesV1(
22031
+ request: RequestPersistedEntriesV1,
22032
+ context: ReceiveRequestContext,
22033
+ lane: ReceiveLaneContext,
22034
+ ): Promise<ConfirmEntriesMessage | undefined> {
22035
+ if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
22036
+ return undefined;
22037
+ }
22038
+ if (
22039
+ !this.admitPersistedReceiptIngress(
22040
+ lane.fromHash,
22041
+ context.message.header.session,
22042
+ request.hashes.length,
22043
+ )
22044
+ ) {
22045
+ return undefined;
22046
+ }
22047
+ // Charge the request before shape validation and attacker-controlled CID
22048
+ // parsing. Invalid shapes are rejected quietly here so they cannot turn the
22049
+ // outer receive error logger into a post-budget work amplifier.
22050
+ try {
22051
+ this.validatePersistedReceiptRequestShape(request);
22052
+ } catch {
22053
+ return undefined;
22054
+ }
22055
+ if (!this.hasValidPersistedReceiptHashes(request)) {
22056
+ return undefined;
22057
+ }
22058
+ const peerInFlight =
22059
+ this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
22060
+ if (
22061
+ peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
22062
+ this._persistedReceiptRequestsInFlightTotal >=
22063
+ MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL
22064
+ ) {
22065
+ return undefined;
22066
+ }
22067
+ this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
22068
+ this._persistedReceiptRequestsInFlightTotal++;
22069
+
22070
+ try {
22071
+ // A positive receipt requires the block itself. Reject entirely absent
22072
+ // batches before the serialized mutation lane and, critically, before a
22073
+ // request can force unrelated pending index/coordinate journals durable.
22074
+ // A concurrent put can yield a harmless false negative: the sender retries.
22075
+ const blockPresence = await this.remoteBlocks.localStore.hasMany(
22076
+ request.hashes,
22077
+ );
22078
+ const presentBlockHashes = request.hashes.filter(
22079
+ (_hash, index) => blockPresence[index] === true,
22080
+ );
22081
+ if (
22082
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22083
+ ) {
22084
+ return undefined;
22085
+ }
22086
+ if (presentBlockHashes.length === 0) {
22087
+ return new ConfirmEntriesMessage({ hashes: [] });
22088
+ }
22089
+ this._liveness.markReplicatorActivity(lane.fromHash);
22090
+ return await this.withReplicationRangeMutationQueue(async () => {
22091
+ if (
22092
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22093
+ ) {
22094
+ return undefined;
22095
+ }
22096
+ this.throwIfNativeDurableCommitFailed();
22097
+ const storage = this.resolvePersistedReceiptStorage();
22098
+ if (!storage) {
22099
+ return undefined;
22100
+ }
22101
+
22102
+ await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
22103
+ await this._coordinates.flushNativeBackboneCoordinateJournal();
22104
+ const candidates = new Map(
22105
+ await Promise.all(
22106
+ presentBlockHashes.map(async (hash) => {
22107
+ const [blockPresent, lowerRow, coordinate] = await Promise.all([
22108
+ this.remoteBlocks.localStore.has(hash),
22109
+ this.log.entryIndex.properties.index.get(toId(hash)),
22110
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(
22111
+ hash,
22112
+ ),
22113
+ ]);
22114
+ return [hash, { blockPresent, lowerRow, coordinate }] as const;
22115
+ }),
22116
+ ),
22117
+ );
22118
+ const presentHashes = presentBlockHashes.filter((hash) => {
22119
+ const candidate = candidates.get(hash)!;
22120
+ return (
22121
+ candidate.blockPresent &&
22122
+ candidate.lowerRow != null &&
22123
+ candidate.coordinate != null
22124
+ );
22125
+ });
22126
+ if (presentHashes.length === 0) {
22127
+ return new ConfirmEntriesMessage({ hashes: [] });
22128
+ }
22129
+ await Promise.all(
22130
+ [...new Set([storage.block, storage.lower, storage.coordinate])].map(
22131
+ (store) => store.barrier(),
22132
+ ),
22133
+ );
22134
+ this.throwIfNativeDurableCommitFailed();
22135
+ if (
22136
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22137
+ ) {
22138
+ return undefined;
22139
+ }
22140
+
22141
+ const ownershipRevision =
22142
+ this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
22143
+ if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
22144
+ return new ConfirmEntriesMessage({ hashes: [] });
22145
+ }
22146
+
22147
+ const selfHash = this.node.identity.publicKey.hashcode();
22148
+ const confirmed: string[] = [];
22149
+ for (const hash of presentHashes) {
22150
+ const candidate = candidates.get(hash)!;
22151
+ if (
22152
+ !candidate.blockPresent ||
22153
+ !candidate.lowerRow ||
22154
+ !candidate.coordinate ||
22155
+ this._checkedPrune.hasActiveWork(hash) ||
22156
+ !this.isPersistedReceiptRequestSessionCurrent(
22157
+ request,
22158
+ context,
22159
+ lane,
22160
+ ) ||
22161
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision)
22162
+ ) {
22163
+ continue;
22164
+ }
22165
+
22166
+ const [blockPresent, lowerRow, coordinate] = await Promise.all([
22167
+ this.remoteBlocks.localStore.has(hash),
22168
+ this.log.entryIndex.properties.index.get(toId(hash)),
22169
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
22170
+ ]);
22171
+ if (!blockPresent || !lowerRow || !coordinate) {
22172
+ continue;
22173
+ }
22174
+
22175
+ const replicas = decodeReplicas(coordinate).getValue(this);
22176
+ const leaders = await this.findLeadersFromEntry(
22177
+ coordinate,
22178
+ replicas,
22179
+ { freshLeaderPlan: true },
22180
+ lane.ownershipLifecycleController,
22181
+ );
22182
+ if (!leaders.has(selfHash)) {
22183
+ continue;
22184
+ }
22185
+ if (
22186
+ !this._checkedPrune.hasActiveWork(hash) &&
22187
+ this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
22188
+ this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22189
+ ) {
22190
+ confirmed.push(hash);
22191
+ }
22192
+ }
22193
+
22194
+ if (
22195
+ confirmed.length === 0 ||
22196
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
22197
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22198
+ ) {
22199
+ return new ConfirmEntriesMessage({ hashes: [] });
22200
+ }
22201
+ const finalRows = await Promise.all(
22202
+ confirmed.map(async (hash) => {
22203
+ const [block, lower, coordinate] = await Promise.all([
22204
+ this.remoteBlocks.localStore.has(hash),
22205
+ this.log.entryIndex.properties.index.get(toId(hash)),
22206
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
22207
+ ]);
22208
+ return { hash, block, lower, coordinate };
22209
+ }),
22210
+ );
22211
+ if (
22212
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
22213
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22214
+ ) {
22215
+ return new ConfirmEntriesMessage({ hashes: [] });
22216
+ }
22217
+ return new ConfirmEntriesMessage({
22218
+ hashes: finalRows
22219
+ .filter(
22220
+ (row) =>
22221
+ row.block &&
22222
+ row.lower &&
22223
+ row.coordinate &&
22224
+ !this._checkedPrune.hasActiveWork(row.hash),
22225
+ )
22226
+ .map((row) => row.hash),
22227
+ });
22228
+ }, lane.ownershipLifecycleController);
22229
+ } finally {
22230
+ const remaining =
22231
+ (this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
22232
+ if (remaining <= 0) {
22233
+ this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
22234
+ } else {
22235
+ this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
22236
+ }
22237
+ this._persistedReceiptRequestsInFlightTotal = Math.max(
22238
+ 0,
22239
+ this._persistedReceiptRequestsInFlightTotal - 1,
22240
+ );
22241
+ }
22242
+ }
22243
+
20506
22244
  private async handleRequestIPruneV2(
20507
22245
  msg: RequestIPruneV2,
20508
22246
  context: ReceiveRequestContext,