@peerbit/shared-log 16.0.14 → 16.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -115,6 +115,7 @@ import {
115
115
  AbortError,
116
116
  TimeoutError,
117
117
  debounceFixedInterval,
118
+ delay,
118
119
  waitFor,
119
120
  } from "@peerbit/time";
120
121
  import pDefer, { type DeferredPromise } from "p-defer";
@@ -147,6 +148,7 @@ import {
147
148
  CompatibilityModeRetiredError,
148
149
  NativeDurableCommitError,
149
150
  NoPeersError,
151
+ PersistedDeliveryError,
150
152
  isNotStartedError,
151
153
  } from "./errors.js";
152
154
  import {
@@ -162,6 +164,7 @@ import {
162
164
  RequestIPruneV2,
163
165
  ResponseIPrune,
164
166
  ResponseIPruneV2,
167
+ SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS,
165
168
  SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
166
169
  SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY,
167
170
  SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM,
@@ -306,6 +309,7 @@ import {
306
309
  import {
307
310
  ConfirmEntriesMessage,
308
311
  RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
312
+ RequestPersistedEntriesV1,
309
313
  SYNC_MESSAGE_PRIORITY,
310
314
  SimpleSyncronizer,
311
315
  } from "./sync/simple.js";
@@ -449,6 +453,7 @@ export {
449
453
  CompatibilityModeRetiredError,
450
454
  NativeDurableCommitError,
451
455
  NoPeersError,
456
+ PersistedDeliveryError,
452
457
  };
453
458
  export { MAX_U32, MAX_U64, type NumberFromType };
454
459
  export type {
@@ -551,6 +556,7 @@ type LeaderMap = Map<string, { intersecting: boolean }>;
551
556
  type LeaderSelectionOptions<R extends "u32" | "u64"> = {
552
557
  roleAge?: number;
553
558
  candidates?: Iterable<string>;
559
+ freshLeaderPlan?: boolean;
554
560
  onLeader?: (key: string) => void;
555
561
  persist?:
556
562
  | {
@@ -698,6 +704,10 @@ type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
698
704
  };
699
705
  };
700
706
 
707
+ type PersistedDeliveryPlanningEntry<T, R extends "u32" | "u64"> =
708
+ | ShallowOrFullEntry<T>
709
+ | EntryReplicated<R>;
710
+
701
711
  type NativeBackboneSimpleDocumentProjectionPlan = {
702
712
  documentVariantType?: "u8" | "string";
703
713
  documentVariantValue?: string;
@@ -908,10 +918,15 @@ const nativeStrictDurableTransactionJournalRecordBytes = (
908
918
  return new TextEncoder().encode(JSON.stringify(record));
909
919
  };
910
920
 
921
+ type TrustedLocalCommitEvidence = {
922
+ committedHashes: Set<string>;
923
+ };
924
+
911
925
  type PreparedPayloadCommitOnlyProperties =
912
926
  NativeBackboneDocumentCommitOptions & {
913
927
  skipMissingNextJoin?: boolean;
914
928
  resolveTrimmedEntries?: boolean;
929
+ localCommitEvidence?: TrustedLocalCommitEvidence;
915
930
  };
916
931
 
917
932
  type PreparedPayloadsManyIndependentProperties<T> = {
@@ -920,6 +935,7 @@ type PreparedPayloadsManyIndependentProperties<T> = {
920
935
  nexts?: ShallowOrFullEntry<T>[][];
921
936
  nativeBackboneDocumentIndexes?: NativeBackboneDocumentIndexCommitInput[];
922
937
  retainMaterializationBytes?: boolean;
938
+ localCommitEvidence?: TrustedLocalCommitEvidence;
923
939
  };
924
940
 
925
941
  type PreparedPayloadCommitOnlyResult<T, R extends "u32" | "u64"> = {
@@ -1681,10 +1697,23 @@ export type Args<
1681
1697
  : "u32",
1682
1698
  > = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D, R>;
1683
1699
 
1684
- export type DeliveryReliability = "ack" | "best-effort";
1700
+ /**
1701
+ * `persisted` waits, after the local commit, for `minAcks` distinct remote
1702
+ * leaders that advertised crash-safe receipt support. Each receipt proves that
1703
+ * the exact block, lower-log row, and replication-coordinate row crossed that
1704
+ * peer's storage barriers at the receipt instant. It is cooperative-peer
1705
+ * evidence, not a Byzantine proof or a promise that the peer will retain the
1706
+ * entry forever. Only current capable leaders count.
1707
+ */
1708
+ export type DeliveryReliability = "ack" | "best-effort" | "persisted";
1685
1709
 
1686
1710
  export type DeliveryOptions = {
1687
1711
  reliability?: DeliveryReliability;
1712
+ /**
1713
+ * Required for persisted delivery; counts distinct current remote leaders.
1714
+ * This does not increase the entry's replication degree, so the configured
1715
+ * replication must make at least this many capable remote leaders eligible.
1716
+ */
1688
1717
  minAcks?: number;
1689
1718
  requireRecipients?: boolean;
1690
1719
  /**
@@ -1692,10 +1721,131 @@ export type DeliveryOptions = {
1692
1721
  * its control lane, so this only changes the direct/fallback RPC path.
1693
1722
  */
1694
1723
  priority?: number;
1724
+ /**
1725
+ * Overall delivery deadline in milliseconds. For persisted delivery it
1726
+ * starts after the local append has returned and includes leader planning,
1727
+ * transfer, receipt requests, and final ownership/session validation. The
1728
+ * omitted persisted default is 10 seconds plus one admission-attempt budget
1729
+ * per transfer chunk and the minimum receipt sender-pacing time implied by
1730
+ * the batch size. An explicit timeout remains exact.
1731
+ */
1695
1732
  timeout?: number;
1696
1733
  signal?: AbortSignal;
1697
1734
  };
1698
1735
 
1736
+ type CrashSafeStorageBarrier = {
1737
+ readonly crashSafe: true;
1738
+ barrier(): MaybePromise<void>;
1739
+ };
1740
+
1741
+ type PersistedReceiptStorage = {
1742
+ block: CrashSafeStorageBarrier;
1743
+ lower: CrashSafeStorageBarrier;
1744
+ coordinate: CrashSafeStorageBarrier;
1745
+ };
1746
+
1747
+ type PersistedDeliveryDeadline = {
1748
+ deadline: number;
1749
+ signal: AbortSignal;
1750
+ dispose(): void;
1751
+ };
1752
+
1753
+ const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
1754
+ const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
1755
+ const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
1756
+ const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
1757
+ const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
1758
+ const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
1759
+ const PERSISTED_RECEIPT_RETRY_MS = 50;
1760
+ const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
1761
+ const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
1762
+ const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
1763
+ const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
1764
+ const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
1765
+ const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
1766
+ const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
1767
+ const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
1768
+ const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
1769
+ const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
1770
+ const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
1771
+ const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
1772
+
1773
+ type PersistedReceiptIngressBucket = {
1774
+ requestTokens: number;
1775
+ hashTokens: number;
1776
+ refilledAt: number;
1777
+ };
1778
+
1779
+ type PersistedReceiptNodeIngressBudget = PersistedReceiptIngressBucket & {
1780
+ peerSessions: Map<string, PersistedReceiptIngressBucket>;
1781
+ };
1782
+
1783
+ type PersistedReceiptNodeEgressBudget = {
1784
+ peerSessions: Map<string, PersistedReceiptIngressBucket>;
1785
+ };
1786
+
1787
+ const persistedReceiptIngressBudgets = new WeakMap<
1788
+ object,
1789
+ PersistedReceiptNodeIngressBudget
1790
+ >();
1791
+
1792
+ // Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
1793
+ // on the Peerbit node (rather than one SharedLog) prevents independent programs
1794
+ // from silently overrunning the same remote receiver together.
1795
+ const persistedReceiptEgressBudgets = new WeakMap<
1796
+ object,
1797
+ PersistedReceiptNodeEgressBudget
1798
+ >();
1799
+
1800
+ const refillPersistedReceiptIngressBucket = (
1801
+ bucket: PersistedReceiptIngressBucket,
1802
+ now: number,
1803
+ requestCapacity: number,
1804
+ hashCapacity: number,
1805
+ requestsPerSecond: number,
1806
+ hashesPerSecond: number,
1807
+ ) => {
1808
+ // Never move the refill watermark backwards. Date.now() can jump after a
1809
+ // clock correction; accepting that earlier timestamp would grant the same
1810
+ // elapsed interval again on the next request.
1811
+ if (now <= bucket.refilledAt) {
1812
+ return;
1813
+ }
1814
+ const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
1815
+ if (elapsedSeconds > 0) {
1816
+ bucket.requestTokens = Math.min(
1817
+ requestCapacity,
1818
+ bucket.requestTokens + elapsedSeconds * requestsPerSecond,
1819
+ );
1820
+ bucket.hashTokens = Math.min(
1821
+ hashCapacity,
1822
+ bucket.hashTokens + elapsedSeconds * hashesPerSecond,
1823
+ );
1824
+ }
1825
+ bucket.refilledAt = now;
1826
+ };
1827
+
1828
+ const persistedReceiptPacingFloorMs = (hashCount: number): number => {
1829
+ const boundedHashCount = Math.max(0, Math.floor(hashCount));
1830
+ const requestCount = Math.ceil(
1831
+ boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE,
1832
+ );
1833
+ return Math.max(
1834
+ 0,
1835
+ ((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
1836
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
1837
+ 1_000,
1838
+ ((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
1839
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
1840
+ 1_000,
1841
+ );
1842
+ };
1843
+
1844
+ const persistedTransferAdmissionBudgetMs = (hashCount: number): number =>
1845
+ Math.ceil(
1846
+ Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE,
1847
+ ) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
1848
+
1699
1849
  export type SharedLogFanoutOptions = {
1700
1850
  root?: string;
1701
1851
  channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
@@ -1709,6 +1859,19 @@ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
1709
1859
 
1710
1860
  type TrustedLogAppendOptions<T> = AppendOptions<T> & {
1711
1861
  __peerbitCanAppendAlreadyValidated?: boolean;
1862
+ __peerbitOnLocalCommit?: (hashes: readonly string[]) => void;
1863
+ };
1864
+
1865
+ const attachTrustedLocalCommitEvidence = <T>(
1866
+ options: AppendOptions<T>,
1867
+ evidence: TrustedLocalCommitEvidence | undefined,
1868
+ ): void => {
1869
+ if (!evidence) return;
1870
+ (options as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (hashes) => {
1871
+ for (const hash of hashes) {
1872
+ evidence.committedHashes.add(hash);
1873
+ }
1874
+ };
1712
1875
  };
1713
1876
 
1714
1877
  export type SharedAppendOptions<T> =
@@ -3019,6 +3182,37 @@ export class SharedLog<
3019
3182
  }
3020
3183
  }
3021
3184
 
3185
+ private async finishCommittedNativeStrictDurableTransaction<TValue>(
3186
+ handle: NativeStrictDurableTransactionHandle | undefined,
3187
+ finish: () => MaybePromise<TValue>,
3188
+ shouldWarnOnRetirementFailure: () => boolean = () => true,
3189
+ ): Promise<TValue> {
3190
+ let finishResult: TValue | undefined;
3191
+ let finishError: unknown;
3192
+ let finishFailed = false;
3193
+ try {
3194
+ finishResult = await finish();
3195
+ } catch (error) {
3196
+ finishError = error;
3197
+ finishFailed = true;
3198
+ }
3199
+ try {
3200
+ await this.completeNativeStrictDurableTransaction(handle);
3201
+ } catch (error) {
3202
+ // The acknowledged lower commit is already final. Completion poisons and
3203
+ // releases the in-memory transaction before throwing, so preserve an
3204
+ // earlier post-commit failure while leaving recovery to retire the intent.
3205
+ if (!finishFailed && !shouldWarnOnRetirementFailure()) {
3206
+ throw error;
3207
+ }
3208
+ warn(`Failed to retire committed native intent: ${String(error)}`);
3209
+ }
3210
+ if (finishFailed) {
3211
+ throw finishError;
3212
+ }
3213
+ return finishResult as TValue;
3214
+ }
3215
+
3022
3216
  private releaseNativeStrictDurableTransaction(
3023
3217
  handle: NativeStrictDurableTransactionHandle | undefined,
3024
3218
  cause: unknown = new Error(
@@ -3504,6 +3698,9 @@ export class SharedLog<
3504
3698
  // parallel map so existing capability-number consumers remain unchanged.
3505
3699
  private _peerSyncCapabilitySessions!: Map<string, bigint>;
3506
3700
  private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
3701
+ private _persistedReceiptStorage?: PersistedReceiptStorage;
3702
+ private _persistedReceiptRequestsInFlight!: Map<string, number>;
3703
+ private _persistedReceiptRequestsInFlightTotal!: number;
3507
3704
  // Pending live raw exchange-head gossip, coalesced per recipient set and
3508
3705
  // flushed at the end of the current event-loop turn (or when a batch cap
3509
3706
  // is hit). Only used when every recipient advertised raw capability.
@@ -3578,10 +3775,7 @@ export class SharedLog<
3578
3775
  return new ReplicationInfoV2SendCoordinator<R>({
3579
3776
  getRpc: () => this.rpc,
3580
3777
  getSelfKey: () => this.node.identity.publicKey,
3581
- getSenderTransportSession: () =>
3582
- BigInt(
3583
- (this.node.services.pubsub as unknown as { session: number }).session,
3584
- ),
3778
+ getSenderTransportSession: () => this.ownTransportSession(),
3585
3779
  getMyReplicationSegments: () => this.getMyReplicationSegments(),
3586
3780
  validatePersistedReplicationRangeSnapshot: (ranges) =>
3587
3781
  this.validatePersistedReplicationRangeSnapshot(ranges),
@@ -3606,12 +3800,37 @@ export class SharedLog<
3606
3800
  });
3607
3801
  }
3608
3802
 
3803
+ private resolvePersistedReceiptStorage():
3804
+ | PersistedReceiptStorage
3805
+ | undefined {
3806
+ if (this.log.appendDurability !== "strict") {
3807
+ return undefined;
3808
+ }
3809
+ const block = this.remoteBlocks.crashSafeDurability;
3810
+ const lower = this.log.entryIndex.properties.index.crashSafeDurability;
3811
+ const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
3812
+ if (!block || !lower || !coordinate) {
3813
+ return undefined;
3814
+ }
3815
+ return { block, lower, coordinate };
3816
+ }
3817
+
3818
+ private ownTransportSession(): bigint {
3819
+ return BigInt(
3820
+ (this.node.services.pubsub as unknown as { session: number | bigint })
3821
+ .session,
3822
+ );
3823
+ }
3824
+
3609
3825
  private replicationInfoV2ReceiveCapabilities(): number {
3610
3826
  return (
3611
3827
  SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
3612
3828
  SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
3613
3829
  SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
3614
3830
  SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
3831
+ (this._persistedReceiptStorage
3832
+ ? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
3833
+ : 0) |
3615
3834
  (this._logProperties?.sync?.rawExchangeHeads === true
3616
3835
  ? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
3617
3836
  : 0)
@@ -3627,9 +3846,7 @@ export class SharedLog<
3627
3846
  { receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
3628
3847
  > {
3629
3848
  const peerHash = properties.target.hashcode();
3630
- const receiverTransportSession = BigInt(
3631
- (this.node.services.pubsub as unknown as { session: number }).session,
3632
- );
3849
+ const receiverTransportSession = this.ownTransportSession();
3633
3850
  await this.rpc.send(
3634
3851
  new SyncCapabilitiesMessage({
3635
3852
  capabilities: this.replicationInfoV2ReceiveCapabilities(),
@@ -3655,9 +3872,7 @@ export class SharedLog<
3655
3872
  ) ||
3656
3873
  this._peerSessions.isReplicationInfoBlocked(peerHash) ||
3657
3874
  !this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
3658
- BigInt(
3659
- (this.node.services.pubsub as unknown as { session: number }).session,
3660
- ) !== receiverTransportSession
3875
+ this.ownTransportSession() !== receiverTransportSession
3661
3876
  ) {
3662
3877
  return undefined;
3663
3878
  }
@@ -3670,10 +3885,7 @@ export class SharedLog<
3670
3885
  private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
3671
3886
  return new ReplicationInfoV2ReceiveCoordinator({
3672
3887
  getSelfKey: () => this.node.identity.publicKey,
3673
- getReceiverTransportSession: () =>
3674
- BigInt(
3675
- (this.node.services.pubsub as unknown as { session: number }).session,
3676
- ),
3888
+ getReceiverTransportSession: () => this.ownTransportSession(),
3677
3889
  isClosed: () => this.closed,
3678
3890
  isPeerSessionCurrent: (peerHash, peerSession) =>
3679
3891
  this._peerSessions.isCurrent(peerHash, peerSession) &&
@@ -3843,6 +4055,9 @@ export class SharedLog<
3843
4055
  this._peerSyncCapabilities = new Map();
3844
4056
  this._peerSyncCapabilitySessions = new Map();
3845
4057
  this._peerSyncCapabilityTimestamps = new Map();
4058
+ this._persistedReceiptStorage = undefined;
4059
+ this._persistedReceiptRequestsInFlight = new Map();
4060
+ this._persistedReceiptRequestsInFlightTotal = 0;
3846
4061
  this._liveRawGossipBatches = new Map();
3847
4062
  this._liveRawGossipFlushScheduled = false;
3848
4063
  this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
@@ -4012,6 +4227,14 @@ export class SharedLog<
4012
4227
  this._fanoutChannel = undefined;
4013
4228
  }
4014
4229
 
4230
+ private ensureLogProviderHandle(fanoutService: FanoutTree): void {
4231
+ if (this._providerHandle || this._closeController.signal.aborted) return;
4232
+ this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
4233
+ ttlMs: 120_000,
4234
+ announceIntervalMs: 60_000,
4235
+ });
4236
+ }
4237
+
4015
4238
  private async _onFanoutData(detail: FanoutTreeDataEvent) {
4016
4239
  let envelope: FanoutEnvelope;
4017
4240
  try {
@@ -4138,11 +4361,23 @@ export class SharedLog<
4138
4361
  const reliability: DeliveryReliability = delivery.reliability ?? "ack";
4139
4362
  const deliveryTimeout = delivery.timeout;
4140
4363
  const deliverySignal = delivery.signal;
4141
- const requireRecipients = delivery.requireRecipients === true;
4364
+ const requireRecipients =
4365
+ reliability === "persisted" || delivery.requireRecipients === true;
4142
4366
  const minAcks =
4143
4367
  delivery.minAcks != null && Number.isFinite(delivery.minAcks)
4144
4368
  ? Math.max(0, Math.floor(delivery.minAcks))
4145
4369
  : undefined;
4370
+ if (reliability === "persisted") {
4371
+ if (
4372
+ delivery.minAcks == null ||
4373
+ !Number.isSafeInteger(delivery.minAcks) ||
4374
+ delivery.minAcks <= 0
4375
+ ) {
4376
+ throw new Error(
4377
+ 'persisted delivery requires a positive explicit "minAcks"',
4378
+ );
4379
+ }
4380
+ }
4146
4381
 
4147
4382
  const wrap =
4148
4383
  deliveryTimeout == null && deliverySignal == null
@@ -4217,6 +4452,82 @@ export class SharedLog<
4217
4452
  };
4218
4453
  }
4219
4454
 
4455
+ private validatePersistedReceiptRequestShape(
4456
+ request: RequestPersistedEntriesV1,
4457
+ ): void {
4458
+ if (
4459
+ request.hashes.length === 0 ||
4460
+ request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES
4461
+ ) {
4462
+ throw new Error(
4463
+ `Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`,
4464
+ );
4465
+ }
4466
+ let bytes = 0;
4467
+ const encoder = new TextEncoder();
4468
+ for (const hash of request.hashes) {
4469
+ if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
4470
+ throw new Error("Invalid persisted receipt hash batch");
4471
+ }
4472
+ bytes += encoder.encode(hash).byteLength;
4473
+ if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
4474
+ throw new Error("Invalid persisted receipt hash batch");
4475
+ }
4476
+ }
4477
+ }
4478
+
4479
+ private hasValidPersistedReceiptHashes(
4480
+ request: RequestPersistedEntriesV1,
4481
+ ): boolean {
4482
+ const seen = new Set<string>();
4483
+ try {
4484
+ for (const hash of request.hashes) {
4485
+ if (seen.has(hash)) return false;
4486
+ cidifyString(hash);
4487
+ seen.add(hash);
4488
+ }
4489
+ } catch {
4490
+ return false;
4491
+ }
4492
+ return true;
4493
+ }
4494
+
4495
+ private getPersistedDeliveryOptions(
4496
+ options?: SharedAppendOptions<T>,
4497
+ ): DeliveryOptions | undefined {
4498
+ const delivery = options?.delivery;
4499
+ if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
4500
+ return undefined;
4501
+ }
4502
+ const parsed = this._parseDeliveryOptions(options?.delivery);
4503
+ const target = (options as { target?: string } | undefined)?.target;
4504
+ if (target !== undefined && target !== "replicators") {
4505
+ throw new Error(
4506
+ 'persisted delivery requires target="replicators" (or an omitted target)',
4507
+ );
4508
+ }
4509
+ if (
4510
+ parsed.delivery?.timeout != null &&
4511
+ (!Number.isFinite(parsed.delivery.timeout) ||
4512
+ parsed.delivery.timeout <= 0 ||
4513
+ parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)
4514
+ ) {
4515
+ throw new Error(
4516
+ `persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
4517
+ );
4518
+ }
4519
+ if (parsed.delivery?.signal?.aborted) {
4520
+ throw parsed.delivery.signal.reason ?? new AbortError();
4521
+ }
4522
+ return parsed.delivery!;
4523
+ }
4524
+
4525
+ private assertPersistedDeliveryOptions(
4526
+ options?: SharedAppendOptions<T>,
4527
+ ): void {
4528
+ this.getPersistedDeliveryOptions(options);
4529
+ }
4530
+
4220
4531
  private async _getSortedRouteHints(targetHash: string): Promise<RouteHint[]> {
4221
4532
  const pubsub: any = this.node.services.pubsub as any;
4222
4533
  const maybeHints = await pubsub?.getUnifiedRouteHints?.(
@@ -4631,6 +4942,7 @@ export class SharedLog<
4631
4942
  plan: RawExchangeHeadSendPlan,
4632
4943
  to: string[] | Set<string>,
4633
4944
  options?: {
4945
+ acknowledge?: boolean;
4634
4946
  priority?: number;
4635
4947
  reserved?: Uint8Array;
4636
4948
  signal?: AbortSignal;
@@ -4645,7 +4957,7 @@ export class SharedLog<
4645
4957
  payload: Uint8Array,
4646
4958
  properties: { topics: string[] },
4647
4959
  options: {
4648
- mode: SilentDelivery;
4960
+ mode: SilentDelivery | AcknowledgeDelivery;
4649
4961
  priority?: number;
4650
4962
  signal?: AbortSignal;
4651
4963
  },
@@ -4733,7 +5045,9 @@ export class SharedLog<
4733
5045
  item.payload,
4734
5046
  { topics: [topic] },
4735
5047
  {
4736
- mode: new SilentDelivery({ redundancy: 1, to: [...to] }),
5048
+ mode: options?.acknowledge
5049
+ ? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
5050
+ : new SilentDelivery({ redundancy: 1, to: [...to] }),
4737
5051
  priority: options?.priority,
4738
5052
  signal: options?.signal,
4739
5053
  },
@@ -4760,34 +5074,762 @@ export class SharedLog<
4760
5074
  });
4761
5075
  }
4762
5076
  }
4763
- return sentMessages;
4764
- }
4765
-
4766
- /**
4767
- * `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
4768
- * responses: resolves the head/reference plan like the TS raw path and
4769
- * ships it fused when possible.
4770
- */
4771
- private async trySendFusedRawExchangeHeads(
4772
- hashes: string[],
4773
- to: string[],
4774
- options?: {
4775
- priority?: number;
4776
- reserved?: Uint8Array;
4777
- signal?: AbortSignal;
4778
- },
4779
- ): Promise<number | undefined> {
4780
- if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
4781
- return undefined;
4782
- }
4783
- const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
4784
- if (!plan) {
4785
- return undefined;
4786
- }
4787
- if (plan.hashes.length === 0) {
4788
- return 0;
4789
- }
4790
- return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
5077
+ return sentMessages;
5078
+ }
5079
+
5080
+ /**
5081
+ * `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
5082
+ * responses: resolves the head/reference plan like the TS raw path and
5083
+ * ships it fused when possible.
5084
+ */
5085
+ private async trySendFusedRawExchangeHeads(
5086
+ hashes: string[],
5087
+ to: string[],
5088
+ options?: {
5089
+ acknowledge?: boolean;
5090
+ priority?: number;
5091
+ reserved?: Uint8Array;
5092
+ signal?: AbortSignal;
5093
+ },
5094
+ ): Promise<number | undefined> {
5095
+ if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
5096
+ return undefined;
5097
+ }
5098
+ const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
5099
+ if (!plan) {
5100
+ return undefined;
5101
+ }
5102
+ if (plan.hashes.length === 0) {
5103
+ return 0;
5104
+ }
5105
+ return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
5106
+ }
5107
+
5108
+ private persistedReceiptPeerSession(
5109
+ peerHash: string,
5110
+ ): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
5111
+ const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
5112
+ const peerSession = this._peerSessions.current(peerHash);
5113
+ if (
5114
+ capabilitySession == null ||
5115
+ !peerSession ||
5116
+ peerSession.phase !== "open" ||
5117
+ !this._peerSessions.isCurrent(peerHash, peerSession) ||
5118
+ !this._peerSyncCapabilityTimestamps.has(peerHash) ||
5119
+ ((this._peerSyncCapabilities.get(peerHash) ?? 0) &
5120
+ SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
5121
+ 0
5122
+ ) {
5123
+ return undefined;
5124
+ }
5125
+ return { capabilitySession, peerSession };
5126
+ }
5127
+
5128
+ private async waitPersistedReceiptRetry(
5129
+ signal: AbortSignal,
5130
+ ms: number,
5131
+ ): Promise<void> {
5132
+ try {
5133
+ await delay(ms, { signal });
5134
+ } catch (error) {
5135
+ throw signal.aborted ? (signal.reason ?? error) : error;
5136
+ }
5137
+ }
5138
+
5139
+ private reservePersistedReceiptEgress(
5140
+ peer: string,
5141
+ capabilitySession: bigint,
5142
+ hashCount: number,
5143
+ now = Date.now(),
5144
+ ): number {
5145
+ let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
5146
+ if (!nodeBudget) {
5147
+ nodeBudget = { peerSessions: new Map() };
5148
+ persistedReceiptEgressBudgets.set(this.node, nodeBudget);
5149
+ }
5150
+ const peerSessionKey = `${peer}\0${capabilitySession}`;
5151
+ let bucket = nodeBudget.peerSessions.get(peerSessionKey);
5152
+ if (!bucket) {
5153
+ while (
5154
+ nodeBudget.peerSessions.size >=
5155
+ MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
5156
+ ) {
5157
+ const oldest = nodeBudget.peerSessions.keys().next().value;
5158
+ if (oldest === undefined) break;
5159
+ nodeBudget.peerSessions.delete(oldest);
5160
+ }
5161
+ bucket = {
5162
+ requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
5163
+ hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
5164
+ refilledAt: now,
5165
+ };
5166
+ nodeBudget.peerSessions.set(peerSessionKey, bucket);
5167
+ } else {
5168
+ nodeBudget.peerSessions.delete(peerSessionKey);
5169
+ nodeBudget.peerSessions.set(peerSessionKey, bucket);
5170
+ }
5171
+ refillPersistedReceiptIngressBucket(
5172
+ bucket,
5173
+ now,
5174
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
5175
+ PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
5176
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
5177
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
5178
+ );
5179
+ if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
5180
+ bucket.requestTokens -= 1;
5181
+ bucket.hashTokens -= hashCount;
5182
+ return 0;
5183
+ }
5184
+ return Math.max(
5185
+ 1,
5186
+ Math.ceil(
5187
+ Math.max(
5188
+ ((1 - bucket.requestTokens) /
5189
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
5190
+ 1_000,
5191
+ ((hashCount - bucket.hashTokens) /
5192
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
5193
+ 1_000,
5194
+ ),
5195
+ ),
5196
+ );
5197
+ }
5198
+
5199
+ private async waitForPersistedReceiptEgressAdmission(
5200
+ peer: string,
5201
+ capabilitySession: bigint,
5202
+ hashCount: number,
5203
+ signal: AbortSignal,
5204
+ ): Promise<void> {
5205
+ while (true) {
5206
+ if (signal.aborted) {
5207
+ throw signal.reason ?? new AbortError();
5208
+ }
5209
+ const waitMs = this.reservePersistedReceiptEgress(
5210
+ peer,
5211
+ capabilitySession,
5212
+ hashCount,
5213
+ );
5214
+ if (waitMs === 0) return;
5215
+ await this.waitPersistedReceiptRetry(signal, waitMs);
5216
+ }
5217
+ }
5218
+
5219
+ private persistedDeliveryTimeoutMs(
5220
+ delivery: DeliveryOptions,
5221
+ hashCount: number,
5222
+ ): number {
5223
+ return (
5224
+ delivery.timeout ??
5225
+ Math.min(
5226
+ MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
5227
+ DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
5228
+ persistedTransferAdmissionBudgetMs(hashCount) +
5229
+ Math.ceil(persistedReceiptPacingFloorMs(hashCount)),
5230
+ )
5231
+ );
5232
+ }
5233
+
5234
+ private async waitForPersistedTransferAdmission(
5235
+ peer: string,
5236
+ hashes: readonly string[],
5237
+ captured: { capabilitySession: bigint; peerSession: PeerSession },
5238
+ signal: AbortSignal,
5239
+ isStillCurrent: () => boolean,
5240
+ ): Promise<boolean> {
5241
+ const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
5242
+ while (!signal.aborted && isStillCurrent()) {
5243
+ const current = this.persistedReceiptPeerSession(peer);
5244
+ if (
5245
+ !current ||
5246
+ current.capabilitySession !== captured.capabilitySession ||
5247
+ current.peerSession !== captured.peerSession
5248
+ ) {
5249
+ return false;
5250
+ }
5251
+ if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
5252
+ return true;
5253
+ }
5254
+ const remaining = expiresAt - Date.now();
5255
+ if (remaining <= 0) return false;
5256
+ await this.waitPersistedReceiptRetry(
5257
+ signal,
5258
+ Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining),
5259
+ );
5260
+ }
5261
+ if (signal.aborted) throw signal.reason ?? new AbortError();
5262
+ return false;
5263
+ }
5264
+
5265
+ private createPersistedDeliveryDeadline(
5266
+ delivery: DeliveryOptions,
5267
+ ownershipLifecycleController: AbortController,
5268
+ hashCount = 1,
5269
+ ): PersistedDeliveryDeadline {
5270
+ const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
5271
+ if (
5272
+ !Number.isFinite(timeoutMs) ||
5273
+ timeoutMs <= 0 ||
5274
+ timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
5275
+ ) {
5276
+ throw new Error(
5277
+ `persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
5278
+ );
5279
+ }
5280
+ const minAcks = Math.floor(delivery.minAcks!);
5281
+ const deadlineController = new AbortController();
5282
+ const timeout = setTimeout(
5283
+ () =>
5284
+ deadlineController.abort(
5285
+ new TimeoutError(
5286
+ `Timed out waiting for ${minAcks} persisted remote replicas.`,
5287
+ ),
5288
+ ),
5289
+ timeoutMs,
5290
+ );
5291
+ timeout.unref?.();
5292
+ return {
5293
+ deadline: Date.now() + timeoutMs,
5294
+ signal: AbortSignal.any(
5295
+ [
5296
+ delivery.signal,
5297
+ this._closeController.signal,
5298
+ ownershipLifecycleController.signal,
5299
+ deadlineController.signal,
5300
+ ].filter((value): value is AbortSignal => !!value),
5301
+ ),
5302
+ dispose: () => clearTimeout(timeout),
5303
+ };
5304
+ }
5305
+
5306
+ private async planPersistedDeliveryLeaders(
5307
+ entries: PersistedDeliveryPlanningEntry<T, R>[],
5308
+ replicas: number,
5309
+ ownershipLifecycleController: AbortController,
5310
+ ): Promise<LeaderMap[]> {
5311
+ if (
5312
+ this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry
5313
+ ) {
5314
+ const leaders: LeaderMap[] = [];
5315
+ for (const entry of entries) {
5316
+ leaders.push(
5317
+ await this.findLeadersFromEntry(
5318
+ entry,
5319
+ replicas,
5320
+ { freshLeaderPlan: true },
5321
+ ownershipLifecycleController,
5322
+ ),
5323
+ );
5324
+ }
5325
+ return leaders;
5326
+ }
5327
+ const items: EntryLeaderBatchItem<R>[] = entries.map((entry) => ({
5328
+ entry,
5329
+ replicas,
5330
+ options: { freshLeaderPlan: true, persist: false },
5331
+ }));
5332
+ const nativeRoutingPlanner =
5333
+ this._nativeRangePlanner ?? this._nativeBackbone;
5334
+ if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
5335
+ const options = items[0]!.options!;
5336
+ const context = await this.createLeaderSelectionContext(
5337
+ options,
5338
+ ownershipLifecycleController,
5339
+ );
5340
+ this.throwIfReplicationOwnershipLifecycleInactive(
5341
+ ownershipLifecycleController,
5342
+ );
5343
+ const nativeOptions = this.createNativeLeaderOptions(context, options);
5344
+ const fullReplicaLeaders =
5345
+ nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(
5346
+ replicas,
5347
+ nativeOptions,
5348
+ );
5349
+ this.throwIfReplicationOwnershipLifecycleInactive(
5350
+ ownershipLifecycleController,
5351
+ );
5352
+ if (fullReplicaLeaders) {
5353
+ // Receipt settlement treats leader maps as read-only. Reusing the one
5354
+ // gid-independent routing result avoids one Map allocation per entry.
5355
+ return new Array<LeaderMap>(items.length).fill(fullReplicaLeaders);
5356
+ }
5357
+ if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
5358
+ const nativeLeaders =
5359
+ nativeRoutingPlanner.planLeaderSamplesForGidsBatch(
5360
+ items.map((item) => ({
5361
+ gid: this.getEntryGid(item.entry),
5362
+ replicas: item.replicas,
5363
+ })),
5364
+ nativeOptions,
5365
+ );
5366
+ this.throwIfReplicationOwnershipLifecycleInactive(
5367
+ ownershipLifecycleController,
5368
+ );
5369
+ if (nativeLeaders?.length === items.length) {
5370
+ return nativeLeaders;
5371
+ }
5372
+ }
5373
+ }
5374
+ return (
5375
+ await this.planEntryLeaderBatch(items, ownershipLifecycleController)
5376
+ ).map((plan) => plan.leaders);
5377
+ }
5378
+
5379
+ private async settlePersistedDelivery(
5380
+ input: PersistedDeliveryPlanningEntry<T, R>[],
5381
+ replicas: number,
5382
+ delivery: DeliveryOptions,
5383
+ ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
5384
+ persistedDeadline?: PersistedDeliveryDeadline,
5385
+ transferOnFirstRound = false,
5386
+ ): Promise<void> {
5387
+ const minAcks = Math.floor(delivery.minAcks!);
5388
+ const entries = new Map(input.map((entry) => [entry.hash, entry]));
5389
+ if (entries.size === 0) return;
5390
+
5391
+ const committedHashes = [...entries.keys()];
5392
+ const ownedDeadline = !persistedDeadline;
5393
+ const deadline =
5394
+ persistedDeadline ??
5395
+ this.createPersistedDeliveryDeadline(
5396
+ delivery,
5397
+ ownershipLifecycleController,
5398
+ entries.size,
5399
+ );
5400
+ const signal = deadline.signal;
5401
+ let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
5402
+ let initialTransferPending = transferOnFirstRound;
5403
+ let needsInitialLeaderCheck = true;
5404
+ const carriedAcknowledgements = new Map<
5405
+ string,
5406
+ Map<string, { capabilitySession: bigint; peerSession: PeerSession }>
5407
+ >(committedHashes.map((hash) => [hash, new Map()]));
5408
+ const repairsByPeer = new Map<
5409
+ string,
5410
+ {
5411
+ capabilitySession: bigint;
5412
+ peerSession: PeerSession;
5413
+ hashes: Set<string>;
5414
+ }
5415
+ >();
5416
+ let acknowledgementOwnershipRevision: number | undefined;
5417
+ const purgePeerDeliveryState = (peer: string) => {
5418
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5419
+ acknowledgements.delete(peer);
5420
+ }
5421
+ repairsByPeer.delete(peer);
5422
+ };
5423
+ try {
5424
+ while (true) {
5425
+ if (signal.aborted) {
5426
+ throw signal.reason ?? new AbortError();
5427
+ }
5428
+ this.throwIfReplicationOwnershipLifecycleInactive(
5429
+ ownershipLifecycleController,
5430
+ );
5431
+ const ownershipRevision =
5432
+ this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
5433
+ const isRoundOwnershipCurrent = () =>
5434
+ this.isReceiveOwnershipSnapshotStable(ownershipRevision);
5435
+ if (acknowledgementOwnershipRevision !== ownershipRevision) {
5436
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5437
+ acknowledgements.clear();
5438
+ }
5439
+ repairsByPeer.clear();
5440
+ acknowledgementOwnershipRevision = ownershipRevision;
5441
+ } else {
5442
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5443
+ for (const [peer, captured] of acknowledgements) {
5444
+ const current = this.persistedReceiptPeerSession(peer);
5445
+ if (
5446
+ !current ||
5447
+ current.capabilitySession !== captured.capabilitySession ||
5448
+ current.peerSession !== captured.peerSession
5449
+ ) {
5450
+ purgePeerDeliveryState(peer);
5451
+ }
5452
+ }
5453
+ }
5454
+ }
5455
+ if (!isRoundOwnershipCurrent()) {
5456
+ await this.waitPersistedReceiptRetry(
5457
+ signal,
5458
+ PERSISTED_RECEIPT_RETRY_MS,
5459
+ );
5460
+ continue;
5461
+ }
5462
+
5463
+ // Carry receipts only across retries in the exact same ownership and
5464
+ // transport epoch. A revision/session change purges them before they can
5465
+ // survive an away-and-back leader transition or combine with a later peer.
5466
+ const hashesByPeer = new Map<string, string[]>();
5467
+ const entryArray = [...entries.values()];
5468
+ const leadersByEntry = await this.planPersistedDeliveryLeaders(
5469
+ entryArray,
5470
+ replicas,
5471
+ ownershipLifecycleController,
5472
+ );
5473
+ if (!isRoundOwnershipCurrent()) continue;
5474
+ const selfHash = this.node.identity.publicKey.hashcode();
5475
+ if (needsInitialLeaderCheck) {
5476
+ needsInitialLeaderCheck = false;
5477
+ if (
5478
+ leadersByEntry.some(
5479
+ (leaders) =>
5480
+ leaders.size === 0 ||
5481
+ (leaders.size === 1 && leaders.has(selfHash)),
5482
+ )
5483
+ ) {
5484
+ throw new NoPeersError(this.rpc.topic);
5485
+ }
5486
+ }
5487
+ for (let index = 0; index < entryArray.length; index++) {
5488
+ const hash = entryArray[index]!.hash;
5489
+ const leaders = leadersByEntry[index]!;
5490
+ if (signal.aborted) {
5491
+ throw signal.reason ?? new AbortError();
5492
+ }
5493
+ if (!isRoundOwnershipCurrent()) break;
5494
+ const acknowledgements = carriedAcknowledgements.get(hash)!;
5495
+ for (const [peer, captured] of acknowledgements) {
5496
+ const current = this.persistedReceiptPeerSession(peer);
5497
+ if (
5498
+ !leaders.has(peer) ||
5499
+ !current ||
5500
+ current.capabilitySession !== captured.capabilitySession ||
5501
+ current.peerSession !== captured.peerSession
5502
+ ) {
5503
+ acknowledgements.delete(peer);
5504
+ }
5505
+ }
5506
+ if (acknowledgements.size >= minAcks) continue;
5507
+ for (const peer of leaders.keys()) {
5508
+ if (peer === selfHash) continue;
5509
+ const current = this.persistedReceiptPeerSession(peer);
5510
+ if (!current) continue;
5511
+ if (acknowledgements.has(peer)) continue;
5512
+ const hashes = hashesByPeer.get(peer) ?? [];
5513
+ hashes.push(hash);
5514
+ hashesByPeer.set(peer, hashes);
5515
+ }
5516
+ }
5517
+ if (!isRoundOwnershipCurrent()) continue;
5518
+
5519
+ const operationQueue = new PQueue({
5520
+ concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
5521
+ });
5522
+ const candidateWaves = Math.max(
5523
+ 1,
5524
+ Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL),
5525
+ );
5526
+ const roundController = new AbortController();
5527
+ const roundSignal = AbortSignal.any([signal, roundController.signal]);
5528
+ const getAttemptTimeout = () =>
5529
+ Math.max(
5530
+ 1,
5531
+ Math.min(
5532
+ maxAttemptMs,
5533
+ Math.floor((deadline.deadline - Date.now()) / candidateWaves),
5534
+ ),
5535
+ );
5536
+ const requests = new Set<Promise<void>>();
5537
+ const transferAllOnRound = initialTransferPending;
5538
+ try {
5539
+ for (const [peer, hashes] of hashesByPeer) {
5540
+ let request!: Promise<void>;
5541
+ request = (async () => {
5542
+ const captured = this.persistedReceiptPeerSession(peer);
5543
+ if (!captured) return;
5544
+ const previousRepair = repairsByPeer.get(peer);
5545
+ if (
5546
+ previousRepair &&
5547
+ (previousRepair.capabilitySession !==
5548
+ captured.capabilitySession ||
5549
+ previousRepair.peerSession !== captured.peerSession)
5550
+ ) {
5551
+ repairsByPeer.delete(peer);
5552
+ }
5553
+ const ensureRepairState = () => {
5554
+ let state = repairsByPeer.get(peer);
5555
+ if (!state) {
5556
+ state = {
5557
+ capabilitySession: captured.capabilitySession,
5558
+ peerSession: captured.peerSession,
5559
+ hashes: new Set<string>(),
5560
+ };
5561
+ repairsByPeer.set(peer, state);
5562
+ }
5563
+ return state;
5564
+ };
5565
+ const isPeerRoundCurrent = () => {
5566
+ if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
5567
+ return false;
5568
+ }
5569
+ const current = this.persistedReceiptPeerSession(peer);
5570
+ return (
5571
+ !!current &&
5572
+ current.capabilitySession === captured.capabilitySession &&
5573
+ current.peerSession === captured.peerSession
5574
+ );
5575
+ };
5576
+ const repairs = repairsByPeer.get(peer)?.hashes;
5577
+ const transferHashes = transferAllOnRound
5578
+ ? hashes
5579
+ : repairs
5580
+ ? hashes.filter((hash) => repairs.has(hash))
5581
+ : [];
5582
+ let attemptedHashCount = 0;
5583
+ if (transferHashes.length > 0) {
5584
+ try {
5585
+ await this.pushEntryHashes(peer, transferHashes, {
5586
+ acknowledge: false,
5587
+ chunkTimeout: getAttemptTimeout,
5588
+ chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
5589
+ isStillCurrent: isPeerRoundCurrent,
5590
+ onChunkAttempted: (chunk) => {
5591
+ attemptedHashCount += chunk.length;
5592
+ if (!transferAllOnRound) {
5593
+ const state = repairsByPeer.get(peer);
5594
+ for (const hash of chunk) state?.hashes.delete(hash);
5595
+ }
5596
+ },
5597
+ onChunkSent: async (chunk) => {
5598
+ return this.waitForPersistedTransferAdmission(
5599
+ peer,
5600
+ chunk,
5601
+ captured,
5602
+ roundSignal,
5603
+ isPeerRoundCurrent,
5604
+ );
5605
+ },
5606
+ operationQueue,
5607
+ priority: delivery.priority,
5608
+ repairHint: !transferAllOnRound,
5609
+ signal: roundSignal,
5610
+ });
5611
+ } catch {
5612
+ // A send can fail after remote admission. The attempted prefix
5613
+ // includes that uncertain chunk for authoritative receipt probing.
5614
+ }
5615
+ if (
5616
+ transferAllOnRound &&
5617
+ attemptedHashCount < transferHashes.length
5618
+ ) {
5619
+ const state = ensureRepairState();
5620
+ for (const hash of transferHashes.slice(attemptedHashCount)) {
5621
+ state.hashes.add(hash);
5622
+ }
5623
+ }
5624
+ }
5625
+ const receiptHashes = transferAllOnRound
5626
+ ? hashes.slice(0, attemptedHashCount)
5627
+ : hashes;
5628
+ if (!isPeerRoundCurrent()) {
5629
+ purgePeerDeliveryState(peer);
5630
+ return;
5631
+ }
5632
+ // Keep receipt chunks sequential per peer. The shared operation
5633
+ // queue bounds only active sends/requests; admission waits never
5634
+ // occupy its slots and cannot starve a later healthy candidate.
5635
+ for (
5636
+ let offset = 0;
5637
+ offset < receiptHashes.length;
5638
+ offset += PERSISTED_RECEIPT_CHUNK_SIZE
5639
+ ) {
5640
+ const requestedHashes = receiptHashes.slice(
5641
+ offset,
5642
+ offset + PERSISTED_RECEIPT_CHUNK_SIZE,
5643
+ );
5644
+ if (roundSignal.aborted || !isPeerRoundCurrent()) {
5645
+ if (!isRoundOwnershipCurrent()) roundController.abort();
5646
+ break;
5647
+ }
5648
+ let responses;
5649
+ try {
5650
+ await this.waitForPersistedReceiptEgressAdmission(
5651
+ peer,
5652
+ captured.capabilitySession,
5653
+ requestedHashes.length,
5654
+ roundSignal,
5655
+ );
5656
+ if (!isPeerRoundCurrent()) break;
5657
+ const attemptTimeout = getAttemptTimeout();
5658
+ responses =
5659
+ (await operationQueue.add(async () => {
5660
+ if (!isPeerRoundCurrent()) return [];
5661
+ return this.rpc.request(
5662
+ new RequestPersistedEntriesV1({
5663
+ expectedReceiverSession: captured.capabilitySession,
5664
+ hashes: requestedHashes,
5665
+ }),
5666
+ {
5667
+ mode: new SilentDelivery({
5668
+ to: [peer],
5669
+ redundancy: 1,
5670
+ }),
5671
+ amount: 1,
5672
+ priority: delivery.priority,
5673
+ timeout: attemptTimeout,
5674
+ signal: roundSignal,
5675
+ },
5676
+ );
5677
+ })) ?? [];
5678
+ } catch {
5679
+ if (roundSignal.aborted) break;
5680
+ // A peer can disconnect or miss this retry while the overall
5681
+ // quorum deadline remains active. Replan on the next round.
5682
+ break;
5683
+ }
5684
+ if (!isRoundOwnershipCurrent()) {
5685
+ roundController.abort();
5686
+ break;
5687
+ }
5688
+ const current = this.persistedReceiptPeerSession(peer);
5689
+ if (
5690
+ !current ||
5691
+ current.capabilitySession !== captured.capabilitySession ||
5692
+ current.peerSession !== captured.peerSession
5693
+ ) {
5694
+ purgePeerDeliveryState(peer);
5695
+ break;
5696
+ }
5697
+ const requested = new Set(requestedHashes);
5698
+ const confirmed = new Set<string>();
5699
+ let receivedValidConfirmation = false;
5700
+ for (const result of responses) {
5701
+ if (
5702
+ !(result.response instanceof ConfirmEntriesMessage) ||
5703
+ result.from?.hashcode() !== peer ||
5704
+ result.message.header.session !== captured.capabilitySession
5705
+ ) {
5706
+ continue;
5707
+ }
5708
+ const unique = new Set(result.response.hashes);
5709
+ if (
5710
+ unique.size !== result.response.hashes.length ||
5711
+ [...unique].some((hash) => !requested.has(hash))
5712
+ ) {
5713
+ continue;
5714
+ }
5715
+ receivedValidConfirmation = true;
5716
+ for (const hash of unique) {
5717
+ confirmed.add(hash);
5718
+ carriedAcknowledgements.get(hash)?.set(peer, captured);
5719
+ }
5720
+ }
5721
+ if (receivedValidConfirmation) {
5722
+ const state = ensureRepairState();
5723
+ for (const hash of requested) {
5724
+ if (confirmed.has(hash)) {
5725
+ state.hashes.delete(hash);
5726
+ } else {
5727
+ state.hashes.add(hash);
5728
+ }
5729
+ }
5730
+ }
5731
+ }
5732
+ })()
5733
+ .then(() => undefined)
5734
+ .catch(() => undefined)
5735
+ .finally(() => requests.delete(request));
5736
+ requests.add(request);
5737
+ }
5738
+
5739
+ const roundComplete = async (): Promise<boolean> => {
5740
+ if (signal.aborted) {
5741
+ throw signal.reason ?? new AbortError();
5742
+ }
5743
+ if (!isRoundOwnershipCurrent()) return false;
5744
+ for (const acknowledgements of carriedAcknowledgements.values()) {
5745
+ if (acknowledgements.size < minAcks) return false;
5746
+ }
5747
+ const validatedLeaders = await this.planPersistedDeliveryLeaders(
5748
+ entryArray,
5749
+ replicas,
5750
+ ownershipLifecycleController,
5751
+ );
5752
+ for (let index = 0; index < entryArray.length; index++) {
5753
+ if (signal.aborted || !isRoundOwnershipCurrent()) {
5754
+ if (signal.aborted) {
5755
+ throw signal.reason ?? new AbortError();
5756
+ }
5757
+ return false;
5758
+ }
5759
+ const hash = entryArray[index]!.hash;
5760
+ const leaders = validatedLeaders[index]!;
5761
+ let valid = 0;
5762
+ const acknowledgements = carriedAcknowledgements.get(hash)!;
5763
+ for (const [peer, captured] of acknowledgements) {
5764
+ const current = this.persistedReceiptPeerSession(peer);
5765
+ if (
5766
+ !current ||
5767
+ current.capabilitySession !== captured.capabilitySession ||
5768
+ current.peerSession !== captured.peerSession
5769
+ ) {
5770
+ purgePeerDeliveryState(peer);
5771
+ continue;
5772
+ }
5773
+ if (!leaders.has(peer)) {
5774
+ acknowledgements.delete(peer);
5775
+ continue;
5776
+ }
5777
+ if (++valid >= minAcks) {
5778
+ break;
5779
+ }
5780
+ }
5781
+ if (valid < minAcks) return false;
5782
+ }
5783
+ this.throwIfReplicationOwnershipLifecycleInactive(
5784
+ ownershipLifecycleController,
5785
+ );
5786
+ if (signal.aborted) {
5787
+ throw signal.reason ?? new AbortError();
5788
+ }
5789
+ return isRoundOwnershipCurrent();
5790
+ };
5791
+
5792
+ while (requests.size > 0) {
5793
+ await Promise.race(requests);
5794
+ if (await roundComplete()) {
5795
+ return;
5796
+ }
5797
+ }
5798
+ if (await roundComplete()) {
5799
+ return;
5800
+ }
5801
+ } finally {
5802
+ roundController.abort();
5803
+ void Promise.allSettled([...requests]);
5804
+ }
5805
+ if (isRoundOwnershipCurrent()) {
5806
+ initialTransferPending = false;
5807
+ }
5808
+ // Keep early retries fair and responsive, then let a caller's longer
5809
+ // overall deadline accommodate a genuinely slow durability barrier.
5810
+ maxAttemptMs = Math.min(
5811
+ MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
5812
+ maxAttemptMs * 2,
5813
+ );
5814
+ await this.waitPersistedReceiptRetry(
5815
+ signal,
5816
+ Math.max(
5817
+ 0,
5818
+ Math.min(
5819
+ PERSISTED_RECEIPT_RETRY_MS,
5820
+ deadline.deadline - Date.now(),
5821
+ ),
5822
+ ),
5823
+ );
5824
+ }
5825
+ } catch (error) {
5826
+ if (error instanceof PersistedDeliveryError) {
5827
+ throw error;
5828
+ }
5829
+ throw new PersistedDeliveryError(error, committedHashes);
5830
+ } finally {
5831
+ if (ownedDeadline) deadline.dispose();
5832
+ }
4791
5833
  }
4792
5834
 
4793
5835
  private async _appendDeliverToReplicators(
@@ -7850,13 +8892,7 @@ export class SharedLog<
7850
8892
  try {
7851
8893
  const fanoutService = getSharedLogFanoutService(this.node.services);
7852
8894
  if (fanoutService?.provide && !this._providerHandle) {
7853
- this._providerHandle = fanoutService.provide(
7854
- `shared-log|${this.topic}`,
7855
- {
7856
- ttlMs: 120_000,
7857
- announceIntervalMs: 60_000,
7858
- },
7859
- );
8895
+ this.ensureLogProviderHandle(fanoutService);
7860
8896
  }
7861
8897
  } catch {
7862
8898
  // Best-effort only.
@@ -8368,63 +9404,135 @@ export class SharedLog<
8368
9404
  }
8369
9405
  }
8370
9406
 
8371
- private async pushRepairEntries(
9407
+ private async pushEntryHashChunk(
8372
9408
  target: string,
8373
- entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
8374
- isStillCurrent: () => boolean = () => true,
8375
- signal?: AbortSignal,
8376
- ) {
8377
- if (!isStillCurrent()) {
8378
- return;
8379
- }
8380
- const hashes = [...entries.keys()];
8381
- if (
9409
+ chunk: string[],
9410
+ options: {
9411
+ acknowledge?: boolean;
9412
+ priority?: number;
9413
+ repairHint?: boolean;
9414
+ signal?: AbortSignal;
9415
+ },
9416
+ isStillCurrent: () => boolean,
9417
+ ): Promise<boolean> {
9418
+ if (!isStillCurrent()) return false;
9419
+ const useRaw =
8382
9420
  this._logProperties?.sync?.rawExchangeHeads === true &&
8383
- this.peerSupportsRawExchangeHeads(target)
8384
- ) {
8385
- const reserved = new Uint8Array(4);
8386
- reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
9421
+ this.peerSupportsRawExchangeHeads(target);
9422
+ if (useRaw) {
9423
+ const reserved = options.repairHint ? new Uint8Array(4) : undefined;
9424
+ if (reserved) reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8387
9425
  const sentMessages = await this.trySendFusedRawExchangeHeads(
8388
- hashes,
9426
+ chunk,
8389
9427
  [target],
8390
- { priority: SYNC_MESSAGE_PRIORITY, reserved, signal },
9428
+ {
9429
+ acknowledge: options.acknowledge,
9430
+ priority: options.priority,
9431
+ reserved,
9432
+ signal: options.signal,
9433
+ },
8391
9434
  );
8392
- if (!isStillCurrent()) {
8393
- return;
8394
- }
8395
- if (sentMessages !== undefined) {
8396
- return;
9435
+ if (!isStillCurrent()) return false;
9436
+ if (sentMessages === undefined) {
9437
+ for await (const message of createRawExchangeHeadsMessages(
9438
+ this.log,
9439
+ chunk,
9440
+ this._logProperties?.sync?.profile,
9441
+ )) {
9442
+ if (!isStillCurrent()) return false;
9443
+ if (options.repairHint) {
9444
+ message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
9445
+ }
9446
+ await this.rpc.send(message, {
9447
+ priority: options.priority,
9448
+ mode: options.acknowledge
9449
+ ? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
9450
+ : new SilentDelivery({ to: [target], redundancy: 1 }),
9451
+ signal: options.signal,
9452
+ });
9453
+ }
8397
9454
  }
8398
- for await (const message of createRawExchangeHeadsMessages(
9455
+ } else {
9456
+ for await (const message of createExchangeHeadsMessages(
8399
9457
  this.log,
8400
- hashes,
8401
- this._logProperties?.sync?.profile,
9458
+ chunk,
8402
9459
  )) {
8403
- if (!isStillCurrent()) {
8404
- return;
9460
+ if (!isStillCurrent()) return false;
9461
+ if (options.repairHint) {
9462
+ message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8405
9463
  }
8406
- message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8407
9464
  await this.rpc.send(message, {
8408
- priority: SYNC_MESSAGE_PRIORITY,
8409
- mode: new SilentDelivery({ to: [target], redundancy: 1 }),
8410
- signal,
9465
+ priority: options.priority,
9466
+ mode: options.acknowledge
9467
+ ? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
9468
+ : new SilentDelivery({ to: [target], redundancy: 1 }),
9469
+ signal: options.signal,
8411
9470
  });
8412
9471
  }
8413
- return;
8414
9472
  }
8415
- for await (const message of createExchangeHeadsMessages(this.log, hashes)) {
8416
- if (!isStillCurrent()) {
8417
- return;
8418
- }
8419
- message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
8420
- await this.rpc.send(message, {
8421
- priority: SYNC_MESSAGE_PRIORITY,
8422
- mode: new SilentDelivery({ to: [target], redundancy: 1 }),
8423
- signal,
8424
- });
9473
+ return isStillCurrent();
9474
+ }
9475
+
9476
+ private async pushEntryHashes(
9477
+ target: string,
9478
+ hashes: string[],
9479
+ options: {
9480
+ acknowledge?: boolean;
9481
+ chunkTimeout?: () => number;
9482
+ chunkSize?: number;
9483
+ isStillCurrent?: () => boolean;
9484
+ onChunkAttempted?: (hashes: readonly string[]) => void;
9485
+ onChunkSent?: (hashes: readonly string[]) => Promise<boolean>;
9486
+ operationQueue?: PQueue;
9487
+ priority?: number;
9488
+ repairHint?: boolean;
9489
+ signal?: AbortSignal;
9490
+ },
9491
+ ) {
9492
+ const isStillCurrent = options.isStillCurrent ?? (() => true);
9493
+ if (!isStillCurrent()) return;
9494
+ const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
9495
+ for (let offset = 0; offset < hashes.length; offset += chunkSize) {
9496
+ const chunk = hashes.slice(offset, offset + chunkSize);
9497
+ const pushChunk = () => {
9498
+ options.onChunkAttempted?.(chunk);
9499
+ const chunkTimeout = options.chunkTimeout?.();
9500
+ const chunkSignal =
9501
+ chunkTimeout == null
9502
+ ? options.signal
9503
+ : AbortSignal.any([
9504
+ ...(options.signal ? [options.signal] : []),
9505
+ AbortSignal.timeout(Math.max(1, chunkTimeout)),
9506
+ ]);
9507
+ return this.pushEntryHashChunk(
9508
+ target,
9509
+ chunk,
9510
+ { ...options, signal: chunkSignal },
9511
+ isStillCurrent,
9512
+ );
9513
+ };
9514
+ const pushed = options.operationQueue
9515
+ ? await options.operationQueue.add(pushChunk)
9516
+ : await pushChunk();
9517
+ if (pushed !== true) return;
9518
+ if (options.onChunkSent && !(await options.onChunkSent(chunk))) return;
8425
9519
  }
8426
9520
  }
8427
9521
 
9522
+ private async pushRepairEntries(
9523
+ target: string,
9524
+ entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
9525
+ isStillCurrent: () => boolean = () => true,
9526
+ signal?: AbortSignal,
9527
+ ) {
9528
+ return this.pushEntryHashes(target, [...entries.keys()], {
9529
+ isStillCurrent,
9530
+ priority: SYNC_MESSAGE_PRIORITY,
9531
+ repairHint: true,
9532
+ signal,
9533
+ });
9534
+ }
9535
+
8428
9536
  private async sendRepairEntriesWithTransport(
8429
9537
  target: string,
8430
9538
  entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
@@ -10395,6 +11503,7 @@ export class SharedLog<
10395
11503
  removed: ShallowOrFullEntry<T>[];
10396
11504
  }> {
10397
11505
  this.throwIfNativeDurableCommitFailed();
11506
+ const persistedDelivery = this.getPersistedDeliveryOptions(options);
10398
11507
  const ownershipLifecycleController =
10399
11508
  this.captureReplicationOwnershipLifecycle();
10400
11509
  if (this._isAdaptiveReplicating) {
@@ -10405,18 +11514,73 @@ export class SharedLog<
10405
11514
  options,
10406
11515
  ownershipLifecycleController,
10407
11516
  );
10408
- const result = await this.log.append(data, appendOptions);
10409
- this.throwIfReplicationOwnershipLifecycleInactive(
10410
- ownershipLifecycleController,
10411
- );
10412
- await this.processLocalAppend(result.entry, result.removed, options, {
10413
- minReplicasValue,
10414
- ownershipLifecycleController,
10415
- });
10416
- this.throwIfReplicationOwnershipLifecycleInactive(
10417
- ownershipLifecycleController,
10418
- );
10419
- return result;
11517
+ let committedHashes: readonly string[] | undefined;
11518
+ if (persistedDelivery) {
11519
+ (appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
11520
+ hashes,
11521
+ ) => {
11522
+ // The lower log reports the exact hashes immediately after their
11523
+ // irreversible local mutation and before entry initialization, trim,
11524
+ // or change callbacks can reject the append.
11525
+ committedHashes = hashes;
11526
+ };
11527
+ }
11528
+ let persistedDeadline: PersistedDeliveryDeadline | undefined;
11529
+ const throwIfDeliveryAborted = () => {
11530
+ if (persistedDeadline?.signal.aborted) {
11531
+ throw persistedDeadline.signal.reason ?? new AbortError();
11532
+ }
11533
+ };
11534
+ try {
11535
+ const result = await this.log.append(data, appendOptions);
11536
+ committedHashes ??= [result.entry.hash];
11537
+ persistedDeadline = persistedDelivery
11538
+ ? this.createPersistedDeliveryDeadline(
11539
+ persistedDelivery,
11540
+ ownershipLifecycleController,
11541
+ 1,
11542
+ )
11543
+ : undefined;
11544
+ this.throwIfReplicationOwnershipLifecycleInactive(
11545
+ ownershipLifecycleController,
11546
+ );
11547
+ throwIfDeliveryAborted();
11548
+ await this.processLocalAppend(result.entry, result.removed, options, {
11549
+ minReplicasValue,
11550
+ ownershipLifecycleController,
11551
+ });
11552
+ throwIfDeliveryAborted();
11553
+ if (persistedDelivery && persistedDeadline) {
11554
+ await this.settlePersistedDelivery(
11555
+ [result.entry],
11556
+ minReplicasValue,
11557
+ persistedDelivery,
11558
+ ownershipLifecycleController,
11559
+ persistedDeadline,
11560
+ );
11561
+ }
11562
+ this.throwIfReplicationOwnershipLifecycleInactive(
11563
+ ownershipLifecycleController,
11564
+ );
11565
+ return result;
11566
+ } catch (error) {
11567
+ if (persistedDelivery && committedHashes) {
11568
+ throw new PersistedDeliveryError(error, committedHashes);
11569
+ }
11570
+ throw error;
11571
+ } finally {
11572
+ persistedDeadline?.dispose();
11573
+ }
11574
+ }
11575
+
11576
+ private rejectPersistedDeliveryOnTrustedLocalAppend(
11577
+ options?: SharedAppendOptions<T>,
11578
+ ): void {
11579
+ if (this.getPersistedDeliveryOptions(options)) {
11580
+ throw new Error(
11581
+ "trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit",
11582
+ );
11583
+ }
10420
11584
  }
10421
11585
 
10422
11586
  // Trusted local append path for callers that already validated the entry.
@@ -10428,6 +11592,7 @@ export class SharedLog<
10428
11592
  removed: ShallowOrFullEntry<T>[];
10429
11593
  }> {
10430
11594
  this.throwIfNativeDurableCommitFailed();
11595
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10431
11596
  const ownershipLifecycleController =
10432
11597
  this.captureReplicationOwnershipLifecycle();
10433
11598
  if (options?.canAppend || options?.onChange) {
@@ -10468,6 +11633,7 @@ export class SharedLog<
10468
11633
  skipMissingNextJoin?: boolean;
10469
11634
  resolveTrimmedEntries?: boolean;
10470
11635
  payloadData?: Uint8Array;
11636
+ localCommitEvidence?: TrustedLocalCommitEvidence;
10471
11637
  },
10472
11638
  ): Promise<{
10473
11639
  entry: Entry<T>;
@@ -10477,6 +11643,7 @@ export class SharedLog<
10477
11643
  appendCommit: PreparedLocalAppendCommit<R>;
10478
11644
  }> {
10479
11645
  this.throwIfNativeDurableCommitFailed();
11646
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10480
11647
  const ownershipLifecycleController =
10481
11648
  this.captureReplicationOwnershipLifecycle();
10482
11649
  if (options?.canAppend || options?.onChange) {
@@ -10491,6 +11658,10 @@ export class SharedLog<
10491
11658
  const { appendOptions, minReplicasValue } =
10492
11659
  this.createLogAppendOptions(options);
10493
11660
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
11661
+ attachTrustedLocalCommitEvidence(
11662
+ appendOptions,
11663
+ properties?.localCommitEvidence,
11664
+ );
10494
11665
  const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(
10495
11666
  data,
10496
11667
  appendOptions,
@@ -10500,6 +11671,11 @@ export class SharedLog<
10500
11671
  payloadData: properties?.payloadData,
10501
11672
  },
10502
11673
  );
11674
+ if (properties?.localCommitEvidence) {
11675
+ properties.localCommitEvidence.committedHashes.add(
11676
+ result.appendFacts.hash,
11677
+ );
11678
+ }
10503
11679
  this.throwIfReplicationOwnershipLifecycleInactive(
10504
11680
  ownershipLifecycleController,
10505
11681
  );
@@ -10748,6 +11924,9 @@ export class SharedLog<
10748
11924
  skipMissingNextJoin: properties?.skipMissingNextJoin,
10749
11925
  resolveTrimmedEntries: properties?.resolveTrimmedEntries,
10750
11926
  payloadData,
11927
+ ...(properties?.localCommitEvidence
11928
+ ? { localCommitEvidence: properties.localCommitEvidence }
11929
+ : undefined),
10751
11930
  });
10752
11931
  }
10753
11932
 
@@ -10758,6 +11937,7 @@ export class SharedLog<
10758
11937
  properties?: PreparedPayloadCommitOnlyProperties,
10759
11938
  ): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
10760
11939
  this.throwIfNativeDurableCommitFailed();
11940
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10761
11941
  if (options?.canAppend || options?.onChange) {
10762
11942
  throw new Error(
10763
11943
  "appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks",
@@ -10781,6 +11961,10 @@ export class SharedLog<
10781
11961
  const { appendOptions, minReplicasValue } =
10782
11962
  this.createLogAppendOptions(options);
10783
11963
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
11964
+ attachTrustedLocalCommitEvidence(
11965
+ appendOptions,
11966
+ properties?.localCommitEvidence,
11967
+ );
10784
11968
  const deferHeadCoordinatePersistence =
10785
11969
  this.shouldDeferHeadCoordinatePersistence(options);
10786
11970
  const nativeBackboneResult =
@@ -10830,6 +12014,7 @@ export class SharedLog<
10830
12014
  properties?: PreparedPayloadCommitOnlyProperties,
10831
12015
  ): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
10832
12016
  this.throwIfNativeDurableCommitFailed();
12017
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
10833
12018
  if (options?.canAppend || options?.onChange) {
10834
12019
  throw new Error(
10835
12020
  "appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks",
@@ -10853,6 +12038,10 @@ export class SharedLog<
10853
12038
  const { appendOptions, minReplicasValue } =
10854
12039
  this.createLogAppendOptions(options);
10855
12040
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
12041
+ attachTrustedLocalCommitEvidence(
12042
+ appendOptions,
12043
+ properties?.localCommitEvidence,
12044
+ );
10856
12045
  const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(
10857
12046
  payloadData,
10858
12047
  appendOptions,
@@ -10892,6 +12081,11 @@ export class SharedLog<
10892
12081
  includeAppendFactsBytes: !deferHeadCoordinatePersistence,
10893
12082
  });
10894
12083
  return mapMaybePromise(resultMaybe, (result) => {
12084
+ if (result && properties?.localCommitEvidence) {
12085
+ properties.localCommitEvidence.committedHashes.add(
12086
+ result.appendFacts.hash,
12087
+ );
12088
+ }
10895
12089
  this.throwIfReplicationOwnershipLifecycleInactive(
10896
12090
  ownershipLifecycleController,
10897
12091
  );
@@ -11472,25 +12666,28 @@ export class SharedLog<
11472
12666
  // boundary separates the catch above from this statement and
11473
12667
  // `rollback` can no longer fire. Nothing downstream rolls back
11474
12668
  // (the retire below only warns), so the token is terminal here.
11475
- this._coordinates.settleResidentCoordinateSnapshot(
11476
- lowerPublicationRollback?.coordinateEntries,
11477
- );
11478
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
11479
- prepared.appendFacts,
11480
- prepared.removed,
11481
- prepared.materializeEntry,
11482
- {
11483
- removedHashes: prepared.removedHashes,
11484
- removedGids: prepared.removedGids,
11485
- },
11486
- );
11487
- try {
11488
- await this.completeNativeStrictDurableTransaction(
11489
- nativeStrictTransaction,
12669
+ if (properties?.localCommitEvidence) {
12670
+ properties.localCommitEvidence.committedHashes.add(
12671
+ prepared.appendFacts.hash,
11490
12672
  );
11491
- } catch (error) {
11492
- warn(`Failed to retire committed native intent: ${String(error)}`);
11493
12673
  }
12674
+ await this.finishCommittedNativeStrictDurableTransaction(
12675
+ nativeStrictTransaction,
12676
+ () => {
12677
+ this._coordinates.settleResidentCoordinateSnapshot(
12678
+ lowerPublicationRollback?.coordinateEntries,
12679
+ );
12680
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
12681
+ prepared.appendFacts,
12682
+ prepared.removed,
12683
+ prepared.materializeEntry,
12684
+ {
12685
+ removedHashes: prepared.removedHashes,
12686
+ removedGids: prepared.removedGids,
12687
+ },
12688
+ );
12689
+ },
12690
+ );
11494
12691
  return finishResult;
11495
12692
  });
11496
12693
  }
@@ -12123,26 +13320,29 @@ export class SharedLog<
12123
13320
  // Success seam: the finalizer acknowledge above is the last
12124
13321
  // await inside the protected try, so `rollback` can no
12125
13322
  // longer fire and the retire below only warns.
12126
- this._coordinates.settleResidentCoordinateSnapshot(
12127
- rollbackCoordinateEntries,
12128
- );
12129
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
12130
- prepared.appendFacts,
12131
- prepared.removed,
12132
- prepared.materializeEntry,
12133
- {
12134
- forgetNativeCoordinates: false,
12135
- removedHashes: prepared.removedHashes,
12136
- removedGids: prepared.removedGids,
12137
- },
12138
- );
12139
- try {
12140
- await this.completeNativeStrictDurableTransaction(
12141
- nativeStrictTransaction,
13323
+ if (properties?.localCommitEvidence) {
13324
+ properties.localCommitEvidence.committedHashes.add(
13325
+ prepared.appendFacts.hash,
12142
13326
  );
12143
- } catch (error) {
12144
- warn(`Failed to retire committed native intent: ${String(error)}`);
12145
13327
  }
13328
+ await this.finishCommittedNativeStrictDurableTransaction(
13329
+ nativeStrictTransaction,
13330
+ () => {
13331
+ this._coordinates.settleResidentCoordinateSnapshot(
13332
+ rollbackCoordinateEntries,
13333
+ );
13334
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
13335
+ prepared.appendFacts,
13336
+ prepared.removed,
13337
+ prepared.materializeEntry,
13338
+ {
13339
+ forgetNativeCoordinates: false,
13340
+ removedHashes: prepared.removedHashes,
13341
+ removedGids: prepared.removedGids,
13342
+ },
13343
+ );
13344
+ },
13345
+ );
12146
13346
  if (
12147
13347
  commitBlocksInBackbone &&
12148
13348
  !runtimeOnlyCoordinates &&
@@ -13235,73 +14435,73 @@ export class SharedLog<
13235
14435
  nativeStrictTransaction,
13236
14436
  );
13237
14437
  });
13238
- this.throwIfReplicationOwnershipLifecycleInactive(
13239
- ownershipLifecycleController,
13240
- );
13241
14438
  } catch (error) {
13242
14439
  return rollbackBatch(error);
13243
14440
  }
13244
14441
  // Success seam: `rollbackBatch` has exactly one call site (the catch
13245
14442
  // above), and everything from here on escapes without any rollback.
13246
- this._coordinates.settleResidentCoordinateSnapshot(
13247
- batchCoordinateRollback,
13248
- );
13249
-
13250
- this.throwIfReplicationOwnershipLifecycleInactive(
13251
- ownershipLifecycleController,
13252
- );
13253
- const appendCommits: PreparedLocalAppendCommit<R>[] = [];
13254
- for (let i = 0; i < coordinateRows.length; i++) {
13255
- const {
13256
- facts,
13257
- backboneAppend,
13258
- coordinateFields,
13259
- plannedCoordinateDeleteHashes,
13260
- } = coordinateRows[i]!;
13261
- this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
13262
- facts,
13263
- [],
13264
- appended.materializeEntries[i]!,
13265
- {
13266
- forgetNativeCoordinates: false,
13267
- removedHashes: plannedCoordinateDeleteHashes,
13268
- removedGids:
13269
- backboneAppend.trimmedGids ??
13270
- (backboneAppend.trimmed.length > 0
13271
- ? backboneAppend.trimmed.map((entry) => entry.gid)
13272
- : undefined),
13273
- },
13274
- );
13275
- if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
13276
- this.remoteBlocks.notifyStoredDeferred(facts.hash);
14443
+ if (properties?.localCommitEvidence) {
14444
+ for (const facts of appended.appendFacts) {
14445
+ properties.localCommitEvidence.committedHashes.add(facts.hash);
13277
14446
  }
13278
- const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
13279
- facts,
13280
- {
13281
- hashNumber: backboneAppend.coordinate.hashNumber as NumberFromType<R>,
13282
- coordinateFields,
13283
- },
13284
- );
13285
- appendCommit.nativeBackboneDocumentIndexCommitted = true;
13286
- appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
13287
- appended.documentTrimmedHeadsProcessed?.[i];
13288
- appendCommit.documentPreviousContext =
13289
- backboneAppend.documentPreviousContext;
13290
- appendCommits.push(appendCommit);
13291
14447
  }
13292
- try {
13293
- await this.completeNativeStrictDurableTransaction(
14448
+ const appendCommits =
14449
+ await this.finishCommittedNativeStrictDurableTransaction(
13294
14450
  nativeStrictTransaction,
14451
+ () => {
14452
+ this._coordinates.settleResidentCoordinateSnapshot(
14453
+ batchCoordinateRollback,
14454
+ );
14455
+ this.throwIfReplicationOwnershipLifecycleInactive(
14456
+ ownershipLifecycleController,
14457
+ );
14458
+ const commits: PreparedLocalAppendCommit<R>[] = [];
14459
+ for (let i = 0; i < coordinateRows.length; i++) {
14460
+ const {
14461
+ facts,
14462
+ backboneAppend,
14463
+ coordinateFields,
14464
+ plannedCoordinateDeleteHashes,
14465
+ } = coordinateRows[i]!;
14466
+ this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
14467
+ facts,
14468
+ [],
14469
+ appended.materializeEntries[i]!,
14470
+ {
14471
+ forgetNativeCoordinates: false,
14472
+ removedHashes: plannedCoordinateDeleteHashes,
14473
+ removedGids:
14474
+ backboneAppend.trimmedGids ??
14475
+ (backboneAppend.trimmed.length > 0
14476
+ ? backboneAppend.trimmed.map((entry) => entry.gid)
14477
+ : undefined),
14478
+ },
14479
+ );
14480
+ if (
14481
+ !runtimeOnlyCoordinates &&
14482
+ this.remoteBlocks.hasNotifyStoredHook()
14483
+ ) {
14484
+ this.remoteBlocks.notifyStoredDeferred(facts.hash);
14485
+ }
14486
+ const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
14487
+ facts,
14488
+ {
14489
+ hashNumber: backboneAppend.coordinate
14490
+ .hashNumber as NumberFromType<R>,
14491
+ coordinateFields,
14492
+ },
14493
+ );
14494
+ appendCommit.nativeBackboneDocumentIndexCommitted = true;
14495
+ appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
14496
+ appended.documentTrimmedHeadsProcessed?.[i];
14497
+ appendCommit.documentPreviousContext =
14498
+ backboneAppend.documentPreviousContext;
14499
+ commits.push(appendCommit);
14500
+ }
14501
+ return commits;
14502
+ },
14503
+ () => this.isRepairLifecycleActive(ownershipLifecycleController),
13295
14504
  );
13296
- this.throwIfReplicationOwnershipLifecycleInactive(
13297
- ownershipLifecycleController,
13298
- );
13299
- } catch (error) {
13300
- if (!this.isRepairLifecycleActive(ownershipLifecycleController)) {
13301
- throw error;
13302
- }
13303
- warn(`Failed to retire committed native intent: ${String(error)}`);
13304
- }
13305
14505
  this.throwIfReplicationOwnershipLifecycleInactive(
13306
14506
  ownershipLifecycleController,
13307
14507
  );
@@ -13333,6 +14533,7 @@ export class SharedLog<
13333
14533
  | undefined
13334
14534
  > {
13335
14535
  this.throwIfNativeDurableCommitFailed();
14536
+ this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
13336
14537
  if (data.length === 0) {
13337
14538
  return { entries: [], removed: [], appendCommits: [] };
13338
14539
  }
@@ -13350,6 +14551,10 @@ export class SharedLog<
13350
14551
  const { appendOptions, minReplicasValue } =
13351
14552
  this.createLogAppendOptions(options);
13352
14553
  appendOptions.__peerbitCanAppendAlreadyValidated = true;
14554
+ attachTrustedLocalCommitEvidence(
14555
+ appendOptions,
14556
+ properties?.localCommitEvidence,
14557
+ );
13353
14558
  const nativeBackboneBatch =
13354
14559
  await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(
13355
14560
  data,
@@ -13372,6 +14577,11 @@ export class SharedLog<
13372
14577
  payloadDatas: properties?.payloadDatas,
13373
14578
  nexts: properties?.nexts,
13374
14579
  });
14580
+ if (result && properties?.localCommitEvidence) {
14581
+ for (const facts of result.appendFacts) {
14582
+ properties.localCommitEvidence.committedHashes.add(facts.hash);
14583
+ }
14584
+ }
13375
14585
  this.throwIfReplicationOwnershipLifecycleInactive(
13376
14586
  ownershipLifecycleController,
13377
14587
  );
@@ -13501,6 +14711,9 @@ export class SharedLog<
13501
14711
  nativeBackboneDocumentIndexes:
13502
14712
  properties?.nativeBackboneDocumentIndexes,
13503
14713
  retainMaterializationBytes: properties?.retainMaterializationBytes,
14714
+ ...(properties?.localCommitEvidence
14715
+ ? { localCommitEvidence: properties.localCommitEvidence }
14716
+ : undefined),
13504
14717
  },
13505
14718
  );
13506
14719
  }
@@ -13513,9 +14726,15 @@ export class SharedLog<
13513
14726
  removed: ShallowOrFullEntry<T>[];
13514
14727
  }> {
13515
14728
  this.throwIfNativeDurableCommitFailed();
14729
+ const persistedDelivery = this.getPersistedDeliveryOptions(options);
13516
14730
  if (data.length === 0) {
13517
14731
  return { entries: [], removed: [] };
13518
14732
  }
14733
+ if (persistedDelivery) {
14734
+ throw new Error(
14735
+ "persisted delivery is not supported for chained appendMany; use independent document puts",
14736
+ );
14737
+ }
13519
14738
  const ownershipLifecycleController =
13520
14739
  this.captureReplicationOwnershipLifecycle();
13521
14740
  if (this._isAdaptiveReplicating) {
@@ -13592,6 +14811,123 @@ export class SharedLog<
13592
14811
  return result;
13593
14812
  }
13594
14813
 
14814
+ /**
14815
+ * Deliver entries that were already committed locally, then wait for the
14816
+ * requested persisted remote quorum. This is the post-commit seam used by
14817
+ * higher-level transactional/batched writers so a receipt timeout never
14818
+ * rolls back or hides their successful local commit.
14819
+ */
14820
+ private async deliverPersistedPlanningEntries(
14821
+ resolveEntries: () => PersistedDeliveryPlanningEntry<T, R>[],
14822
+ committedHashes: string[],
14823
+ options: SharedAppendOptions<T>,
14824
+ ): Promise<void> {
14825
+ let persistedDeadline: PersistedDeliveryDeadline | undefined;
14826
+ try {
14827
+ const delivery = this.getPersistedDeliveryOptions(options);
14828
+ if (!delivery) {
14829
+ throw new Error(
14830
+ 'deliverPersistedEntries requires reliability="persisted"',
14831
+ );
14832
+ }
14833
+ const ownershipLifecycleController =
14834
+ this.captureReplicationOwnershipLifecycle();
14835
+ const deadline = this.createPersistedDeliveryDeadline(
14836
+ delivery,
14837
+ ownershipLifecycleController,
14838
+ committedHashes.length,
14839
+ );
14840
+ persistedDeadline = deadline;
14841
+ const throwIfDeliveryAborted = () => {
14842
+ if (deadline.signal.aborted) {
14843
+ throw deadline.signal.reason ?? new AbortError();
14844
+ }
14845
+ if (Date.now() >= deadline.deadline) {
14846
+ throw new TimeoutError(
14847
+ `Timed out waiting for ${Math.floor(delivery.minAcks!)} persisted remote replicas.`,
14848
+ );
14849
+ }
14850
+ };
14851
+ const entries = resolveEntries();
14852
+ throwIfDeliveryAborted();
14853
+ if (entries.length === 0) return;
14854
+ const { minReplicasValue } = this.createLogAppendOptions(
14855
+ options,
14856
+ ownershipLifecycleController,
14857
+ );
14858
+ throwIfDeliveryAborted();
14859
+ await this.settlePersistedDelivery(
14860
+ entries,
14861
+ minReplicasValue,
14862
+ delivery,
14863
+ ownershipLifecycleController,
14864
+ deadline,
14865
+ true,
14866
+ );
14867
+ } catch (error) {
14868
+ throw new PersistedDeliveryError(error, committedHashes);
14869
+ } finally {
14870
+ persistedDeadline?.dispose();
14871
+ }
14872
+ }
14873
+
14874
+ private createPersistedDeliveryPlanningEntries(
14875
+ appendCommits: PreparedLocalAppendCommit<R>[],
14876
+ materializeEntries: () => Entry<T>[],
14877
+ ): PersistedDeliveryPlanningEntry<T, R>[] {
14878
+ let materializedEntries: Entry<T>[] | undefined;
14879
+ const requiresFullEntries =
14880
+ this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
14881
+ const getMaterializedEntry = (
14882
+ appendCommit: PreparedLocalAppendCommit<R>,
14883
+ index: number,
14884
+ ) => {
14885
+ materializedEntries ??= materializeEntries();
14886
+ const entry = materializedEntries[index];
14887
+ if (!entry || entry.hash !== appendCommit.hash) {
14888
+ throw new Error(
14889
+ `Persisted delivery materializer did not return committed entry ${appendCommit.hash}`,
14890
+ );
14891
+ }
14892
+ return entry;
14893
+ };
14894
+ return appendCommits.map((appendCommit, index) => {
14895
+ if (!requiresFullEntries && appendCommit.coordinateFields) {
14896
+ return this._coordinates.materializeResidentCoordinateEntry(
14897
+ appendCommit.coordinateFields,
14898
+ );
14899
+ }
14900
+ return getMaterializedEntry(appendCommit, index);
14901
+ });
14902
+ }
14903
+
14904
+ private deliverPersistedAppendCommits(
14905
+ appendCommits: PreparedLocalAppendCommit<R>[],
14906
+ materializeEntries: () => Entry<T>[],
14907
+ options: SharedAppendOptions<T>,
14908
+ ): Promise<void> {
14909
+ return this.deliverPersistedPlanningEntries(
14910
+ () =>
14911
+ this.createPersistedDeliveryPlanningEntries(
14912
+ appendCommits,
14913
+ materializeEntries,
14914
+ ),
14915
+ appendCommits.map((appendCommit) => appendCommit.hash),
14916
+ options,
14917
+ );
14918
+ }
14919
+
14920
+ async deliverPersistedEntries(
14921
+ entries: Entry<T>[],
14922
+ options: SharedAppendOptions<T>,
14923
+ ): Promise<void> {
14924
+ return this.deliverPersistedPlanningEntries(
14925
+ () => entries,
14926
+ entries.map((entry) => entry.hash),
14927
+ options,
14928
+ );
14929
+ }
14930
+
13595
14931
  private canCoalesceLocalAppendMany(
13596
14932
  entries: Entry<T>[],
13597
14933
  options?: SharedAppendOptions<T>,
@@ -14733,6 +16069,9 @@ export class SharedLog<
14733
16069
  this._peerSyncCapabilities = new Map();
14734
16070
  this._peerSyncCapabilitySessions = new Map();
14735
16071
  this._peerSyncCapabilityTimestamps = new Map();
16072
+ this._persistedReceiptStorage = undefined;
16073
+ this._persistedReceiptRequestsInFlight = new Map();
16074
+ this._persistedReceiptRequestsInFlightTotal = 0;
14736
16075
  this._liveRawGossipBatches = new Map();
14737
16076
  this._liveRawGossipFlushScheduled = false;
14738
16077
  this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
@@ -14875,6 +16214,43 @@ export class SharedLog<
14875
16214
 
14876
16215
  const fanoutService = getSharedLogFanoutService(this.node.services);
14877
16216
  const blockProviderNamespace = (cid: string) => `cid:${cid}`;
16217
+ const logProviderNamespace = `shared-log|${this.topic}`;
16218
+ const announceBlockProvider = async (cid: string): Promise<void> => {
16219
+ try {
16220
+ await fanoutService?.announceProvider(blockProviderNamespace(cid), {
16221
+ ttlMs: 120_000,
16222
+ bootstrapMaxPeers: 2,
16223
+ });
16224
+ } catch {
16225
+ // Provider publication is best-effort.
16226
+ }
16227
+ };
16228
+ const announceBlockProviders = async (cids: string[]): Promise<void> => {
16229
+ const batchedAnnounce = fanoutService?.announceProviders;
16230
+ if (typeof batchedAnnounce === "function") {
16231
+ const namespaces = function* () {
16232
+ for (const cid of cids) yield blockProviderNamespace(cid);
16233
+ };
16234
+ await batchedAnnounce.call(fanoutService, namespaces(), {
16235
+ ttlMs: 120_000,
16236
+ bootstrapMaxPeers: 2,
16237
+ });
16238
+ return;
16239
+ }
16240
+
16241
+ // Tolerate a skewed runtime that supplied an older FanoutTree service.
16242
+ let nextIndex = 0;
16243
+ const worker = async () => {
16244
+ for (;;) {
16245
+ const index = nextIndex++;
16246
+ if (index >= cids.length) return;
16247
+ await announceBlockProvider(cids[index]!);
16248
+ }
16249
+ };
16250
+ await Promise.all(
16251
+ Array.from({ length: Math.min(8, cids.length) }, () => worker()),
16252
+ );
16253
+ };
14878
16254
  const [replicationIndex, logIndex] = await Promise.all([
14879
16255
  logScope.scope("replication"),
14880
16256
  logScope.scope("log"),
@@ -14974,17 +16350,23 @@ export class SharedLog<
14974
16350
 
14975
16351
  let directoryProviders: string[] = [];
14976
16352
  try {
14977
- directoryProviders =
14978
- (await fanoutService?.queryProviders(
14979
- blockProviderNamespace(cid),
14980
- {
14981
- want: maxPeers,
14982
- timeoutMs: 2_000,
14983
- queryTimeoutMs: 500,
14984
- bootstrapMaxPeers: 2,
14985
- signal: opts?.signal,
14986
- },
14987
- )) ?? [];
16353
+ const query = (namespace: string) =>
16354
+ fanoutService?.queryProviders(namespace, {
16355
+ want: maxPeers,
16356
+ timeoutMs: 2_000,
16357
+ queryTimeoutMs: 500,
16358
+ bootstrapMaxPeers: 2,
16359
+ signal: opts?.signal,
16360
+ }) ?? Promise.resolve([]);
16361
+ const results = await Promise.allSettled([
16362
+ query(blockProviderNamespace(cid)),
16363
+ query(logProviderNamespace),
16364
+ ]);
16365
+ for (const result of results) {
16366
+ if (result.status === "fulfilled") {
16367
+ directoryProviders.push(...result.value);
16368
+ }
16369
+ }
14988
16370
  } catch {
14989
16371
  // Ignore discovery failures; local evidence remains usable.
14990
16372
  }
@@ -14992,11 +16374,7 @@ export class SharedLog<
14992
16374
  const selected: string[] = [];
14993
16375
  const selectedSet = new Set<string>();
14994
16376
  const add = (peer: string | undefined) => {
14995
- if (
14996
- !peer ||
14997
- selectedSet.has(peer) ||
14998
- selected.length >= maxPeers
14999
- ) {
16377
+ if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
15000
16378
  return;
15001
16379
  }
15002
16380
  selectedSet.add(peer);
@@ -15014,31 +16392,46 @@ export class SharedLog<
15014
16392
  return selected;
15015
16393
  },
15016
16394
  watchProviders: fanoutService
15017
- ? (cid, opts) =>
15018
- fanoutService.watchProviders(blockProviderNamespace(cid), {
15019
- signal: opts.signal,
15020
- want: 8,
15021
- ttlMs: 10_000,
15022
- renewIntervalMs: 5_000,
15023
- bootstrapMaxPeers: 2,
15024
- onProviders: (providers) =>
15025
- opts.onProviders(providers.map((provider) => provider.hash)),
15026
- })
15027
- : undefined,
15028
- onPut: fanoutService
15029
- ? async (cid) => {
15030
- // Best-effort directory announce for "get without remote.from" workflows.
16395
+ ? (cid, opts) => {
16396
+ const watch = (namespace: string) =>
16397
+ fanoutService.watchProviders(namespace, {
16398
+ signal: opts.signal,
16399
+ want: 8,
16400
+ ttlMs: 10_000,
16401
+ renewIntervalMs: 5_000,
16402
+ bootstrapMaxPeers: 2,
16403
+ onProviders: (providers) =>
16404
+ opts.onProviders(providers.map((provider) => provider.hash)),
16405
+ });
16406
+ const cidWatch = watch(blockProviderNamespace(cid));
15031
16407
  try {
15032
- await fanoutService.announceProvider(
15033
- blockProviderNamespace(cid),
15034
- {
15035
- ttlMs: 120_000,
15036
- bootstrapMaxPeers: 2,
16408
+ const logWatch = watch(logProviderNamespace);
16409
+ return {
16410
+ close: () => {
16411
+ cidWatch.close();
16412
+ logWatch.close();
15037
16413
  },
15038
- );
16414
+ };
16415
+ } catch (error) {
16416
+ cidWatch.close();
16417
+ throw error;
16418
+ }
16419
+ }
16420
+ : undefined,
16421
+ onPut: fanoutService ? announceBlockProvider : undefined,
16422
+ onPutMany: fanoutService
16423
+ ? (cids) => {
16424
+ // A renewable log-wide provider lease makes every CID in a
16425
+ // stored batch discoverable to current readers. Retain the
16426
+ // per-CID directory publications for released readers that only
16427
+ // know the legacy namespace; the bounded workers avoid creating
16428
+ // one in-flight promise per block.
16429
+ try {
16430
+ this.ensureLogProviderHandle(fanoutService);
15039
16431
  } catch {
15040
16432
  // ignore announce failures
15041
16433
  }
16434
+ return announceBlockProviders(cids);
15042
16435
  }
15043
16436
  : undefined,
15044
16437
  });
@@ -15290,6 +16683,7 @@ export class SharedLog<
15290
16683
  },
15291
16684
  indexer: logIndex,
15292
16685
  });
16686
+ this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
15293
16687
  try {
15294
16688
  const recovered =
15295
16689
  await this.recoverNativeStrictDurableTransactionIntent();
@@ -16088,13 +17482,12 @@ export class SharedLog<
16088
17482
  this.topic,
16089
17483
  );
16090
17484
  // We do this here, because these calls requires this.closed == false
16091
- void this.pruneOfflineReplicators()
16092
- .catch((error) => {
16093
- if (isNotStartedError(error as Error)) {
16094
- return;
16095
- }
16096
- logger.error(error);
16097
- });
17485
+ void this.pruneOfflineReplicators().catch((error) => {
17486
+ if (isNotStartedError(error as Error)) {
17487
+ return;
17488
+ }
17489
+ logger.error(error);
17490
+ });
16098
17491
 
16099
17492
  this._liveness.startReplicatorLivenessSweep();
16100
17493
 
@@ -16413,7 +17806,8 @@ export class SharedLog<
16413
17806
  for (const peer of reachable) {
16414
17807
  if (confirmed?.has(peer)) liveEvidenceTiers[0].push(peer);
16415
17808
  else if (contacted?.has(peer)) liveEvidenceTiers[1].push(peer);
16416
- else if (this.uniqueReplicators.has(peer)) liveEvidenceTiers[2].push(peer);
17809
+ else if (this.uniqueReplicators.has(peer))
17810
+ liveEvidenceTiers[2].push(peer);
16417
17811
  }
16418
17812
  const liveKnown: string[] = [];
16419
17813
  for (let tier = 0; tier < liveEvidenceTiers.length; tier++) {
@@ -16983,8 +18377,7 @@ export class SharedLog<
16983
18377
  ): Promise<void> {
16984
18378
  if (!gids) return;
16985
18379
  const candidates = [...new Set(gids)].filter(
16986
- (gid) =>
16987
- history === this._gidPeersHistory && !!gid && history.has(gid),
18380
+ (gid) => history === this._gidPeersHistory && !!gid && history.has(gid),
16988
18381
  );
16989
18382
  if (candidates.length === 0) return;
16990
18383
  const hasHeads = await this.hasAnyHeadForGidSets(
@@ -17018,9 +18411,7 @@ export class SharedLog<
17018
18411
  for (const gid of gids) {
17019
18412
  if (!gid || !history.has(gid)) continue;
17020
18413
  if (state.pending.has(gid)) continue;
17021
- if (
17022
- state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY
17023
- ) {
18414
+ if (state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY) {
17024
18415
  // History is only a suppression memo. Under sustained synchronous
17025
18416
  // production, forgetting an overflow row is the bounded, safe fallback:
17026
18417
  // it can cause redundant delivery but cannot affect ownership or data.
@@ -17819,6 +19210,9 @@ export class SharedLog<
17819
19210
  this._peerSyncCapabilities?.clear();
17820
19211
  this._peerSyncCapabilitySessions?.clear();
17821
19212
  this._peerSyncCapabilityTimestamps?.clear();
19213
+ this._persistedReceiptStorage = undefined;
19214
+ this._persistedReceiptRequestsInFlight?.clear();
19215
+ this._persistedReceiptRequestsInFlightTotal = 0;
17822
19216
  this._v2Receive?.clearForClose();
17823
19217
  this._v2Send?.clearForClose();
17824
19218
  this._liveRawGossipBatches?.clear();
@@ -18591,7 +19985,11 @@ export class SharedLog<
18591
19985
  async onMessage(
18592
19986
  msg: TransportMessage,
18593
19987
  context: RequestContext,
18594
- ): Promise<void> {
19988
+ ): Promise<void>;
19989
+ async onMessage(
19990
+ msg: TransportMessage,
19991
+ context: RequestContext,
19992
+ ): Promise<any> {
18595
19993
  const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
18596
19994
  ? msg
18597
19995
  : undefined;
@@ -18667,6 +20065,7 @@ export class SharedLog<
18667
20065
  if (
18668
20066
  !context.from.equals(this.node.identity.publicKey) &&
18669
20067
  !(msg instanceof RequestReplicationInfoV2Message) &&
20068
+ !(msg instanceof RequestPersistedEntriesV1) &&
18670
20069
  !isReplicationInfoV2Message(msg)
18671
20070
  ) {
18672
20071
  this._liveness.markReplicatorActivity(receiveFromHash);
@@ -20113,24 +21512,30 @@ export class SharedLog<
20113
21512
  );
20114
21513
  }
20115
21514
  if (syncProfile) {
20116
- emitSyncProfileDuration(syncProfile, coordinatePersistStartedAt, {
20117
- name: "sharedLog.receive.coordinatePersist",
20118
- component: "shared-log",
20119
- entries: entriesToPersist.length,
20120
- messages: 1,
20121
- details: {
20122
- reusedLeaderPlans: reusableCoordinatePersistItemCount,
20123
- nativeBackboneOnly:
20124
- nativeBackboneOnlyPersistedHashes?.size ?? 0,
21515
+ emitSyncProfileDuration(
21516
+ syncProfile,
21517
+ coordinatePersistStartedAt,
21518
+ {
21519
+ name: "sharedLog.receive.coordinatePersist",
21520
+ component: "shared-log",
21521
+ entries: entriesToPersist.length,
21522
+ messages: 1,
21523
+ details: {
21524
+ reusedLeaderPlans: reusableCoordinatePersistItemCount,
21525
+ nativeBackboneOnly:
21526
+ nativeBackboneOnlyPersistedHashes?.size ?? 0,
21527
+ },
20125
21528
  },
20126
- });
21529
+ );
20127
21530
  }
20128
21531
  for (const hash of admittedHashes) {
20129
21532
  confirmedHashes.add(hash);
20130
21533
  }
20131
21534
  const checkedPruneStartedAt = syncProfileStart(syncProfile);
20132
21535
  const ownershipChangedDuringReceive =
20133
- !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
21536
+ !this.isReceiveOwnershipSnapshotStable(
21537
+ receiveOwnershipRevision,
21538
+ );
20134
21539
  if (ownershipChangedDuringReceive) {
20135
21540
  const freshAuditRevision =
20136
21541
  this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
@@ -20307,7 +21712,13 @@ export class SharedLog<
20307
21712
  };
20308
21713
  // The prelude already threw when `context.from` was missing.
20309
21714
  const laneRequestContext = context as ReceiveRequestContext;
20310
- if (msg instanceof RequestIPruneV2) {
21715
+ if (msg instanceof RequestPersistedEntriesV1) {
21716
+ return await this.handleRequestPersistedEntriesV1(
21717
+ msg,
21718
+ laneRequestContext,
21719
+ lane,
21720
+ );
21721
+ } else if (msg instanceof RequestIPruneV2) {
20311
21722
  await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
20312
21723
  } else if (msg instanceof ResponseIPruneV2) {
20313
21724
  await this.handleResponseIPruneV2(msg, laneRequestContext, lane);
@@ -20514,6 +21925,322 @@ export class SharedLog<
20514
21925
  // wire-stash release, lease release, durable-poison recheck).
20515
21926
  // -----------------------------------------------------------------
20516
21927
 
21928
+ private isPersistedReceiptRequestSessionCurrent(
21929
+ request: RequestPersistedEntriesV1,
21930
+ context: ReceiveRequestContext,
21931
+ lane: ReceiveLaneContext,
21932
+ ): boolean {
21933
+ const session = lane.session;
21934
+ return (
21935
+ !!this._persistedReceiptStorage &&
21936
+ !context.from.equals(this.node.identity.publicKey) &&
21937
+ request.expectedReceiverSession === this.ownTransportSession() &&
21938
+ session !== null &&
21939
+ session.phase === "open" &&
21940
+ this._peerSessions.isCurrent(lane.fromHash, session) &&
21941
+ !this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
21942
+ this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
21943
+ this._peerSyncCapabilitySessions.get(lane.fromHash) ===
21944
+ context.message.header.session &&
21945
+ this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
21946
+ this.isRepairLifecycleActive(lane.ownershipLifecycleController)
21947
+ );
21948
+ }
21949
+
21950
+ private admitPersistedReceiptIngress(
21951
+ peer: string,
21952
+ transportSession: bigint,
21953
+ hashCount: number,
21954
+ now = Date.now(),
21955
+ ): boolean {
21956
+ // Charge malformed empty/oversized vectors too. The request was already
21957
+ // decoded by the transport, so letting an invalid shape bypass this bucket
21958
+ // would leave an authenticated peer with an unmetered validation/logging
21959
+ // path. Clamp only the accounting cost; shape validation still rejects the
21960
+ // request below.
21961
+ const hashCost = Math.max(
21962
+ 1,
21963
+ Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount),
21964
+ );
21965
+ let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
21966
+ if (!nodeBudget) {
21967
+ nodeBudget = {
21968
+ requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
21969
+ hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
21970
+ refilledAt: now,
21971
+ peerSessions: new Map(),
21972
+ };
21973
+ persistedReceiptIngressBudgets.set(this.node, nodeBudget);
21974
+ }
21975
+ refillPersistedReceiptIngressBucket(
21976
+ nodeBudget,
21977
+ now,
21978
+ PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
21979
+ PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
21980
+ PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND,
21981
+ PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND,
21982
+ );
21983
+
21984
+ const peerSessionKey = `${peer}\0${transportSession}`;
21985
+ let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
21986
+ if (!peerBudget) {
21987
+ while (
21988
+ nodeBudget.peerSessions.size >=
21989
+ MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
21990
+ ) {
21991
+ const oldest = nodeBudget.peerSessions.keys().next().value;
21992
+ if (oldest === undefined) break;
21993
+ nodeBudget.peerSessions.delete(oldest);
21994
+ }
21995
+ peerBudget = {
21996
+ requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
21997
+ hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
21998
+ refilledAt: now,
21999
+ };
22000
+ nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
22001
+ } else {
22002
+ // Refresh insertion order so bounded eviction prefers inactive sessions.
22003
+ nodeBudget.peerSessions.delete(peerSessionKey);
22004
+ nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
22005
+ }
22006
+ refillPersistedReceiptIngressBucket(
22007
+ peerBudget,
22008
+ now,
22009
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
22010
+ PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
22011
+ PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
22012
+ PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
22013
+ );
22014
+
22015
+ if (
22016
+ nodeBudget.requestTokens < 1 ||
22017
+ nodeBudget.hashTokens < hashCost ||
22018
+ peerBudget.requestTokens < 1 ||
22019
+ peerBudget.hashTokens < hashCost
22020
+ ) {
22021
+ return false;
22022
+ }
22023
+ nodeBudget.requestTokens -= 1;
22024
+ nodeBudget.hashTokens -= hashCost;
22025
+ peerBudget.requestTokens -= 1;
22026
+ peerBudget.hashTokens -= hashCost;
22027
+ return true;
22028
+ }
22029
+
22030
+ private async handleRequestPersistedEntriesV1(
22031
+ request: RequestPersistedEntriesV1,
22032
+ context: ReceiveRequestContext,
22033
+ lane: ReceiveLaneContext,
22034
+ ): Promise<ConfirmEntriesMessage | undefined> {
22035
+ if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
22036
+ return undefined;
22037
+ }
22038
+ if (
22039
+ !this.admitPersistedReceiptIngress(
22040
+ lane.fromHash,
22041
+ context.message.header.session,
22042
+ request.hashes.length,
22043
+ )
22044
+ ) {
22045
+ return undefined;
22046
+ }
22047
+ // Charge the request before shape validation and attacker-controlled CID
22048
+ // parsing. Invalid shapes are rejected quietly here so they cannot turn the
22049
+ // outer receive error logger into a post-budget work amplifier.
22050
+ try {
22051
+ this.validatePersistedReceiptRequestShape(request);
22052
+ } catch {
22053
+ return undefined;
22054
+ }
22055
+ if (!this.hasValidPersistedReceiptHashes(request)) {
22056
+ return undefined;
22057
+ }
22058
+ const peerInFlight =
22059
+ this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
22060
+ if (
22061
+ peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
22062
+ this._persistedReceiptRequestsInFlightTotal >=
22063
+ MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL
22064
+ ) {
22065
+ return undefined;
22066
+ }
22067
+ this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
22068
+ this._persistedReceiptRequestsInFlightTotal++;
22069
+
22070
+ try {
22071
+ // A positive receipt requires the block itself. Reject entirely absent
22072
+ // batches before the serialized mutation lane and, critically, before a
22073
+ // request can force unrelated pending index/coordinate journals durable.
22074
+ // A concurrent put can yield a harmless false negative: the sender retries.
22075
+ const blockPresence = await this.remoteBlocks.localStore.hasMany(
22076
+ request.hashes,
22077
+ );
22078
+ const presentBlockHashes = request.hashes.filter(
22079
+ (_hash, index) => blockPresence[index] === true,
22080
+ );
22081
+ if (
22082
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22083
+ ) {
22084
+ return undefined;
22085
+ }
22086
+ if (presentBlockHashes.length === 0) {
22087
+ return new ConfirmEntriesMessage({ hashes: [] });
22088
+ }
22089
+ this._liveness.markReplicatorActivity(lane.fromHash);
22090
+ return await this.withReplicationRangeMutationQueue(async () => {
22091
+ if (
22092
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22093
+ ) {
22094
+ return undefined;
22095
+ }
22096
+ this.throwIfNativeDurableCommitFailed();
22097
+ const storage = this.resolvePersistedReceiptStorage();
22098
+ if (!storage) {
22099
+ return undefined;
22100
+ }
22101
+
22102
+ await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
22103
+ await this._coordinates.flushNativeBackboneCoordinateJournal();
22104
+ const candidates = new Map(
22105
+ await Promise.all(
22106
+ presentBlockHashes.map(async (hash) => {
22107
+ const [blockPresent, lowerRow, coordinate] = await Promise.all([
22108
+ this.remoteBlocks.localStore.has(hash),
22109
+ this.log.entryIndex.properties.index.get(toId(hash)),
22110
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(
22111
+ hash,
22112
+ ),
22113
+ ]);
22114
+ return [hash, { blockPresent, lowerRow, coordinate }] as const;
22115
+ }),
22116
+ ),
22117
+ );
22118
+ const presentHashes = presentBlockHashes.filter((hash) => {
22119
+ const candidate = candidates.get(hash)!;
22120
+ return (
22121
+ candidate.blockPresent &&
22122
+ candidate.lowerRow != null &&
22123
+ candidate.coordinate != null
22124
+ );
22125
+ });
22126
+ if (presentHashes.length === 0) {
22127
+ return new ConfirmEntriesMessage({ hashes: [] });
22128
+ }
22129
+ await Promise.all(
22130
+ [...new Set([storage.block, storage.lower, storage.coordinate])].map(
22131
+ (store) => store.barrier(),
22132
+ ),
22133
+ );
22134
+ this.throwIfNativeDurableCommitFailed();
22135
+ if (
22136
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22137
+ ) {
22138
+ return undefined;
22139
+ }
22140
+
22141
+ const ownershipRevision =
22142
+ this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
22143
+ if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
22144
+ return new ConfirmEntriesMessage({ hashes: [] });
22145
+ }
22146
+
22147
+ const selfHash = this.node.identity.publicKey.hashcode();
22148
+ const confirmed: string[] = [];
22149
+ for (const hash of presentHashes) {
22150
+ const candidate = candidates.get(hash)!;
22151
+ if (
22152
+ !candidate.blockPresent ||
22153
+ !candidate.lowerRow ||
22154
+ !candidate.coordinate ||
22155
+ this._checkedPrune.hasActiveWork(hash) ||
22156
+ !this.isPersistedReceiptRequestSessionCurrent(
22157
+ request,
22158
+ context,
22159
+ lane,
22160
+ ) ||
22161
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision)
22162
+ ) {
22163
+ continue;
22164
+ }
22165
+
22166
+ const [blockPresent, lowerRow, coordinate] = await Promise.all([
22167
+ this.remoteBlocks.localStore.has(hash),
22168
+ this.log.entryIndex.properties.index.get(toId(hash)),
22169
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
22170
+ ]);
22171
+ if (!blockPresent || !lowerRow || !coordinate) {
22172
+ continue;
22173
+ }
22174
+
22175
+ const replicas = decodeReplicas(coordinate).getValue(this);
22176
+ const leaders = await this.findLeadersFromEntry(
22177
+ coordinate,
22178
+ replicas,
22179
+ { freshLeaderPlan: true },
22180
+ lane.ownershipLifecycleController,
22181
+ );
22182
+ if (!leaders.has(selfHash)) {
22183
+ continue;
22184
+ }
22185
+ if (
22186
+ !this._checkedPrune.hasActiveWork(hash) &&
22187
+ this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
22188
+ this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22189
+ ) {
22190
+ confirmed.push(hash);
22191
+ }
22192
+ }
22193
+
22194
+ if (
22195
+ confirmed.length === 0 ||
22196
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
22197
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22198
+ ) {
22199
+ return new ConfirmEntriesMessage({ hashes: [] });
22200
+ }
22201
+ const finalRows = await Promise.all(
22202
+ confirmed.map(async (hash) => {
22203
+ const [block, lower, coordinate] = await Promise.all([
22204
+ this.remoteBlocks.localStore.has(hash),
22205
+ this.log.entryIndex.properties.index.get(toId(hash)),
22206
+ this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
22207
+ ]);
22208
+ return { hash, block, lower, coordinate };
22209
+ }),
22210
+ );
22211
+ if (
22212
+ !this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
22213
+ !this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
22214
+ ) {
22215
+ return new ConfirmEntriesMessage({ hashes: [] });
22216
+ }
22217
+ return new ConfirmEntriesMessage({
22218
+ hashes: finalRows
22219
+ .filter(
22220
+ (row) =>
22221
+ row.block &&
22222
+ row.lower &&
22223
+ row.coordinate &&
22224
+ !this._checkedPrune.hasActiveWork(row.hash),
22225
+ )
22226
+ .map((row) => row.hash),
22227
+ });
22228
+ }, lane.ownershipLifecycleController);
22229
+ } finally {
22230
+ const remaining =
22231
+ (this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
22232
+ if (remaining <= 0) {
22233
+ this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
22234
+ } else {
22235
+ this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
22236
+ }
22237
+ this._persistedReceiptRequestsInFlightTotal = Math.max(
22238
+ 0,
22239
+ this._persistedReceiptRequestsInFlightTotal - 1,
22240
+ );
22241
+ }
22242
+ }
22243
+
20517
22244
  private async handleRequestIPruneV2(
20518
22245
  msg: RequestIPruneV2,
20519
22246
  context: ReceiveRequestContext,