@peerbit/shared-log 16.0.14 → 16.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/coordinate-persistence.d.ts +10 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +18 -0
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/errors.d.ts +11 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +23 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +6 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +6 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +60 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1315 -141
- package/dist/src/index.js.map +1 -1
- package/dist/src/native-write-through-block-store.d.ts +4 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -1
- package/dist/src/native-write-through-block-store.js +13 -0
- package/dist/src/native-write-through-block-store.js.map +1 -1
- package/dist/src/sync/simple.d.ts +15 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +44 -0
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +19 -19
- package/src/coordinate-persistence.ts +21 -0
- package/src/errors.ts +35 -0
- package/src/exchange-heads.ts +7 -0
- package/src/index.ts +1988 -261
- package/src/native-write-through-block-store.ts +14 -0
- package/src/sync/simple.ts +22 -0
package/dist/src/index.js
CHANGED
|
@@ -45,7 +45,7 @@ import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
|
|
|
45
45
|
import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
|
|
46
46
|
import { RPC } from "@peerbit/rpc";
|
|
47
47
|
import { AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
|
|
48
|
-
import { AbortError, TimeoutError, debounceFixedInterval, waitFor, } from "@peerbit/time";
|
|
48
|
+
import { AbortError, TimeoutError, debounceFixedInterval, delay, waitFor, } from "@peerbit/time";
|
|
49
49
|
import pDefer, {} from "p-defer";
|
|
50
50
|
import PQueue from "p-queue";
|
|
51
51
|
import { concat, fromString } from "uint8arrays";
|
|
@@ -54,8 +54,8 @@ import { CheckedPruneCoordinator, } from "./checked-prune.js";
|
|
|
54
54
|
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
55
55
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
56
56
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
57
|
-
import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
|
|
58
|
-
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
57
|
+
import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, isNotStartedError, } from "./errors.js";
|
|
58
|
+
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
59
59
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
60
60
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
61
61
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
@@ -78,7 +78,7 @@ import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmen
|
|
|
78
78
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
79
79
|
import { createSyncronizer } from "./sync/factory.js";
|
|
80
80
|
import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./sync/profile.js";
|
|
81
|
-
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
81
|
+
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, RequestPersistedEntriesV1, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
82
82
|
import { groupByGid, tryGroupByGidSync } from "./utils.js";
|
|
83
83
|
const getSharedLogFanoutService = (services) => services.fanout;
|
|
84
84
|
const createOneShotPeerReceiveLease = (releaseFn) => {
|
|
@@ -127,7 +127,7 @@ export { createReplicationDomainHash, createReplicationDomainTime, };
|
|
|
127
127
|
export { CPUUsageIntervalLag };
|
|
128
128
|
export * from "./replication.js";
|
|
129
129
|
export * from "./replication-status.js";
|
|
130
|
-
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, };
|
|
130
|
+
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, };
|
|
131
131
|
export { MAX_U32, MAX_U64 };
|
|
132
132
|
export { ExchangeHeadsMessage, RawExchangeHeadsMessage, StashBackedRawExchangeHeadsMessage, };
|
|
133
133
|
export const logger = loggerFn("peerbit:shared-log");
|
|
@@ -565,6 +565,63 @@ const checkMinReplicasLimit = (minReplicas) => {
|
|
|
565
565
|
throw new Error("Higher replication degree than 100 is not recommended for performance reasons");
|
|
566
566
|
}
|
|
567
567
|
};
|
|
568
|
+
const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
|
|
569
|
+
const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
|
|
570
|
+
const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
|
|
571
|
+
const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
|
|
572
|
+
const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
|
|
573
|
+
const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
|
|
574
|
+
const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
575
|
+
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
576
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
577
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
578
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
579
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
580
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
581
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
|
|
582
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
|
|
583
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
|
|
584
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
|
|
585
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
|
|
586
|
+
const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
|
|
587
|
+
const persistedReceiptIngressBudgets = new WeakMap();
|
|
588
|
+
// Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
|
|
589
|
+
// on the Peerbit node (rather than one SharedLog) prevents independent programs
|
|
590
|
+
// from silently overrunning the same remote receiver together.
|
|
591
|
+
const persistedReceiptEgressBudgets = new WeakMap();
|
|
592
|
+
const refillPersistedReceiptIngressBucket = (bucket, now, requestCapacity, hashCapacity, requestsPerSecond, hashesPerSecond) => {
|
|
593
|
+
// Never move the refill watermark backwards. Date.now() can jump after a
|
|
594
|
+
// clock correction; accepting that earlier timestamp would grant the same
|
|
595
|
+
// elapsed interval again on the next request.
|
|
596
|
+
if (now <= bucket.refilledAt) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
|
|
600
|
+
if (elapsedSeconds > 0) {
|
|
601
|
+
bucket.requestTokens = Math.min(requestCapacity, bucket.requestTokens + elapsedSeconds * requestsPerSecond);
|
|
602
|
+
bucket.hashTokens = Math.min(hashCapacity, bucket.hashTokens + elapsedSeconds * hashesPerSecond);
|
|
603
|
+
}
|
|
604
|
+
bucket.refilledAt = now;
|
|
605
|
+
};
|
|
606
|
+
const persistedReceiptPacingFloorMs = (hashCount) => {
|
|
607
|
+
const boundedHashCount = Math.max(0, Math.floor(hashCount));
|
|
608
|
+
const requestCount = Math.ceil(boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE);
|
|
609
|
+
return Math.max(0, ((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
|
|
610
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
611
|
+
1_000, ((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
|
|
612
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
613
|
+
1_000);
|
|
614
|
+
};
|
|
615
|
+
const persistedTransferAdmissionBudgetMs = (hashCount) => Math.ceil(Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
616
|
+
const attachTrustedLocalCommitEvidence = (options, evidence) => {
|
|
617
|
+
if (!evidence)
|
|
618
|
+
return;
|
|
619
|
+
options.__peerbitOnLocalCommit = (hashes) => {
|
|
620
|
+
for (const hash of hashes) {
|
|
621
|
+
evidence.committedHashes.add(hash);
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
};
|
|
568
625
|
const asTrustedLowerLog = (log) => log;
|
|
569
626
|
class ReplicationStatusSnapshotChangedError extends Error {
|
|
570
627
|
constructor() {
|
|
@@ -1410,6 +1467,34 @@ let SharedLog = (() => {
|
|
|
1410
1467
|
handle.release();
|
|
1411
1468
|
}
|
|
1412
1469
|
}
|
|
1470
|
+
async finishCommittedNativeStrictDurableTransaction(handle, finish, shouldWarnOnRetirementFailure = () => true) {
|
|
1471
|
+
let finishResult;
|
|
1472
|
+
let finishError;
|
|
1473
|
+
let finishFailed = false;
|
|
1474
|
+
try {
|
|
1475
|
+
finishResult = await finish();
|
|
1476
|
+
}
|
|
1477
|
+
catch (error) {
|
|
1478
|
+
finishError = error;
|
|
1479
|
+
finishFailed = true;
|
|
1480
|
+
}
|
|
1481
|
+
try {
|
|
1482
|
+
await this.completeNativeStrictDurableTransaction(handle);
|
|
1483
|
+
}
|
|
1484
|
+
catch (error) {
|
|
1485
|
+
// The acknowledged lower commit is already final. Completion poisons and
|
|
1486
|
+
// releases the in-memory transaction before throwing, so preserve an
|
|
1487
|
+
// earlier post-commit failure while leaving recovery to retire the intent.
|
|
1488
|
+
if (!finishFailed && !shouldWarnOnRetirementFailure()) {
|
|
1489
|
+
throw error;
|
|
1490
|
+
}
|
|
1491
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
1492
|
+
}
|
|
1493
|
+
if (finishFailed) {
|
|
1494
|
+
throw finishError;
|
|
1495
|
+
}
|
|
1496
|
+
return finishResult;
|
|
1497
|
+
}
|
|
1413
1498
|
releaseNativeStrictDurableTransaction(handle, cause = new Error("Native durable transaction intent was retained for recovery")) {
|
|
1414
1499
|
if (!handle || handle.released) {
|
|
1415
1500
|
return;
|
|
@@ -1762,6 +1847,9 @@ let SharedLog = (() => {
|
|
|
1762
1847
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
1763
1848
|
_peerSyncCapabilitySessions;
|
|
1764
1849
|
_peerSyncCapabilityTimestamps;
|
|
1850
|
+
_persistedReceiptStorage;
|
|
1851
|
+
_persistedReceiptRequestsInFlight;
|
|
1852
|
+
_persistedReceiptRequestsInFlightTotal;
|
|
1765
1853
|
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
1766
1854
|
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
1767
1855
|
// is hit). Only used when every recipient advertised raw capability.
|
|
@@ -1819,7 +1907,7 @@ let SharedLog = (() => {
|
|
|
1819
1907
|
return new ReplicationInfoV2SendCoordinator({
|
|
1820
1908
|
getRpc: () => this.rpc,
|
|
1821
1909
|
getSelfKey: () => this.node.identity.publicKey,
|
|
1822
|
-
getSenderTransportSession: () =>
|
|
1910
|
+
getSenderTransportSession: () => this.ownTransportSession(),
|
|
1823
1911
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
1824
1912
|
validatePersistedReplicationRangeSnapshot: (ranges) => this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
1825
1913
|
isClosed: () => this.closed,
|
|
@@ -1837,18 +1925,37 @@ let SharedLog = (() => {
|
|
|
1837
1925
|
0,
|
|
1838
1926
|
});
|
|
1839
1927
|
}
|
|
1928
|
+
resolvePersistedReceiptStorage() {
|
|
1929
|
+
if (this.log.appendDurability !== "strict") {
|
|
1930
|
+
return undefined;
|
|
1931
|
+
}
|
|
1932
|
+
const block = this.remoteBlocks.crashSafeDurability;
|
|
1933
|
+
const lower = this.log.entryIndex.properties.index.crashSafeDurability;
|
|
1934
|
+
const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
|
|
1935
|
+
if (!block || !lower || !coordinate) {
|
|
1936
|
+
return undefined;
|
|
1937
|
+
}
|
|
1938
|
+
return { block, lower, coordinate };
|
|
1939
|
+
}
|
|
1940
|
+
ownTransportSession() {
|
|
1941
|
+
return BigInt(this.node.services.pubsub
|
|
1942
|
+
.session);
|
|
1943
|
+
}
|
|
1840
1944
|
replicationInfoV2ReceiveCapabilities() {
|
|
1841
1945
|
return (SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
1842
1946
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
1843
1947
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
1844
1948
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
|
|
1949
|
+
(this._persistedReceiptStorage
|
|
1950
|
+
? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
|
|
1951
|
+
: 0) |
|
|
1845
1952
|
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
1846
1953
|
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
1847
1954
|
: 0));
|
|
1848
1955
|
}
|
|
1849
1956
|
async advertiseReplicationInfoV2ReceiveCapability(properties) {
|
|
1850
1957
|
const peerHash = properties.target.hashcode();
|
|
1851
|
-
const receiverTransportSession =
|
|
1958
|
+
const receiverTransportSession = this.ownTransportSession();
|
|
1852
1959
|
await this.rpc.send(new SyncCapabilitiesMessage({
|
|
1853
1960
|
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
1854
1961
|
}), {
|
|
@@ -1867,7 +1974,7 @@ let SharedLog = (() => {
|
|
|
1867
1974
|
!this._peerSessions.isReceiveEpochCurrent(peerHash, properties.receiveEpoch) ||
|
|
1868
1975
|
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
1869
1976
|
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
1870
|
-
|
|
1977
|
+
this.ownTransportSession() !== receiverTransportSession) {
|
|
1871
1978
|
return undefined;
|
|
1872
1979
|
}
|
|
1873
1980
|
return {
|
|
@@ -1878,7 +1985,7 @@ let SharedLog = (() => {
|
|
|
1878
1985
|
createReplicationInfoV2ReceiveCoordinator() {
|
|
1879
1986
|
return new ReplicationInfoV2ReceiveCoordinator({
|
|
1880
1987
|
getSelfKey: () => this.node.identity.publicKey,
|
|
1881
|
-
getReceiverTransportSession: () =>
|
|
1988
|
+
getReceiverTransportSession: () => this.ownTransportSession(),
|
|
1882
1989
|
isClosed: () => this.closed,
|
|
1883
1990
|
isPeerSessionCurrent: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1884
1991
|
peerSession.phase === "open" &&
|
|
@@ -2014,6 +2121,9 @@ let SharedLog = (() => {
|
|
|
2014
2121
|
this._peerSyncCapabilities = new Map();
|
|
2015
2122
|
this._peerSyncCapabilitySessions = new Map();
|
|
2016
2123
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
2124
|
+
this._persistedReceiptStorage = undefined;
|
|
2125
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
2126
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
2017
2127
|
this._liveRawGossipBatches = new Map();
|
|
2018
2128
|
this._liveRawGossipFlushScheduled = false;
|
|
2019
2129
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -2152,6 +2262,14 @@ let SharedLog = (() => {
|
|
|
2152
2262
|
}
|
|
2153
2263
|
this._fanoutChannel = undefined;
|
|
2154
2264
|
}
|
|
2265
|
+
ensureLogProviderHandle(fanoutService) {
|
|
2266
|
+
if (this._providerHandle || this._closeController.signal.aborted)
|
|
2267
|
+
return;
|
|
2268
|
+
this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
|
|
2269
|
+
ttlMs: 120_000,
|
|
2270
|
+
announceIntervalMs: 60_000,
|
|
2271
|
+
});
|
|
2272
|
+
}
|
|
2155
2273
|
async _onFanoutData(detail) {
|
|
2156
2274
|
let envelope;
|
|
2157
2275
|
try {
|
|
@@ -2250,10 +2368,17 @@ let SharedLog = (() => {
|
|
|
2250
2368
|
const reliability = delivery.reliability ?? "ack";
|
|
2251
2369
|
const deliveryTimeout = delivery.timeout;
|
|
2252
2370
|
const deliverySignal = delivery.signal;
|
|
2253
|
-
const requireRecipients = delivery.requireRecipients === true;
|
|
2371
|
+
const requireRecipients = reliability === "persisted" || delivery.requireRecipients === true;
|
|
2254
2372
|
const minAcks = delivery.minAcks != null && Number.isFinite(delivery.minAcks)
|
|
2255
2373
|
? Math.max(0, Math.floor(delivery.minAcks))
|
|
2256
2374
|
: undefined;
|
|
2375
|
+
if (reliability === "persisted") {
|
|
2376
|
+
if (delivery.minAcks == null ||
|
|
2377
|
+
!Number.isSafeInteger(delivery.minAcks) ||
|
|
2378
|
+
delivery.minAcks <= 0) {
|
|
2379
|
+
throw new Error('persisted delivery requires a positive explicit "minAcks"');
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2257
2382
|
const wrap = deliveryTimeout == null && deliverySignal == null
|
|
2258
2383
|
? undefined
|
|
2259
2384
|
: (promise) => new Promise((resolve, reject) => {
|
|
@@ -2319,6 +2444,62 @@ let SharedLog = (() => {
|
|
|
2319
2444
|
wrap,
|
|
2320
2445
|
};
|
|
2321
2446
|
}
|
|
2447
|
+
validatePersistedReceiptRequestShape(request) {
|
|
2448
|
+
if (request.hashes.length === 0 ||
|
|
2449
|
+
request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES) {
|
|
2450
|
+
throw new Error(`Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`);
|
|
2451
|
+
}
|
|
2452
|
+
let bytes = 0;
|
|
2453
|
+
const encoder = new TextEncoder();
|
|
2454
|
+
for (const hash of request.hashes) {
|
|
2455
|
+
if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
2456
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
2457
|
+
}
|
|
2458
|
+
bytes += encoder.encode(hash).byteLength;
|
|
2459
|
+
if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
2460
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
hasValidPersistedReceiptHashes(request) {
|
|
2465
|
+
const seen = new Set();
|
|
2466
|
+
try {
|
|
2467
|
+
for (const hash of request.hashes) {
|
|
2468
|
+
if (seen.has(hash))
|
|
2469
|
+
return false;
|
|
2470
|
+
cidifyString(hash);
|
|
2471
|
+
seen.add(hash);
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
catch {
|
|
2475
|
+
return false;
|
|
2476
|
+
}
|
|
2477
|
+
return true;
|
|
2478
|
+
}
|
|
2479
|
+
getPersistedDeliveryOptions(options) {
|
|
2480
|
+
const delivery = options?.delivery;
|
|
2481
|
+
if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
|
|
2482
|
+
return undefined;
|
|
2483
|
+
}
|
|
2484
|
+
const parsed = this._parseDeliveryOptions(options?.delivery);
|
|
2485
|
+
const target = options?.target;
|
|
2486
|
+
if (target !== undefined && target !== "replicators") {
|
|
2487
|
+
throw new Error('persisted delivery requires target="replicators" (or an omitted target)');
|
|
2488
|
+
}
|
|
2489
|
+
if (parsed.delivery?.timeout != null &&
|
|
2490
|
+
(!Number.isFinite(parsed.delivery.timeout) ||
|
|
2491
|
+
parsed.delivery.timeout <= 0 ||
|
|
2492
|
+
parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)) {
|
|
2493
|
+
throw new Error(`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`);
|
|
2494
|
+
}
|
|
2495
|
+
if (parsed.delivery?.signal?.aborted) {
|
|
2496
|
+
throw parsed.delivery.signal.reason ?? new AbortError();
|
|
2497
|
+
}
|
|
2498
|
+
return parsed.delivery;
|
|
2499
|
+
}
|
|
2500
|
+
assertPersistedDeliveryOptions(options) {
|
|
2501
|
+
this.getPersistedDeliveryOptions(options);
|
|
2502
|
+
}
|
|
2322
2503
|
async _getSortedRouteHints(targetHash) {
|
|
2323
2504
|
const pubsub = this.node.services.pubsub;
|
|
2324
2505
|
const maybeHints = await pubsub?.getUnifiedRouteHints?.(this.topic, targetHash);
|
|
@@ -2707,7 +2888,9 @@ let SharedLog = (() => {
|
|
|
2707
2888
|
}
|
|
2708
2889
|
attemptedMessages += 1;
|
|
2709
2890
|
await pubsub.publishPreEncodedData(item.payload, { topics: [topic] }, {
|
|
2710
|
-
mode:
|
|
2891
|
+
mode: options?.acknowledge
|
|
2892
|
+
? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
|
|
2893
|
+
: new SilentDelivery({ redundancy: 1, to: [...to] }),
|
|
2711
2894
|
priority: options?.priority,
|
|
2712
2895
|
signal: options?.signal,
|
|
2713
2896
|
});
|
|
@@ -2754,6 +2937,537 @@ let SharedLog = (() => {
|
|
|
2754
2937
|
}
|
|
2755
2938
|
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
2756
2939
|
}
|
|
2940
|
+
persistedReceiptPeerSession(peerHash) {
|
|
2941
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2942
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
2943
|
+
if (capabilitySession == null ||
|
|
2944
|
+
!peerSession ||
|
|
2945
|
+
peerSession.phase !== "open" ||
|
|
2946
|
+
!this._peerSessions.isCurrent(peerHash, peerSession) ||
|
|
2947
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
2948
|
+
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
2949
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
|
|
2950
|
+
0) {
|
|
2951
|
+
return undefined;
|
|
2952
|
+
}
|
|
2953
|
+
return { capabilitySession, peerSession };
|
|
2954
|
+
}
|
|
2955
|
+
async waitPersistedReceiptRetry(signal, ms) {
|
|
2956
|
+
try {
|
|
2957
|
+
await delay(ms, { signal });
|
|
2958
|
+
}
|
|
2959
|
+
catch (error) {
|
|
2960
|
+
throw signal.aborted ? (signal.reason ?? error) : error;
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
reservePersistedReceiptEgress(peer, capabilitySession, hashCount, now = Date.now()) {
|
|
2964
|
+
let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
|
|
2965
|
+
if (!nodeBudget) {
|
|
2966
|
+
nodeBudget = { peerSessions: new Map() };
|
|
2967
|
+
persistedReceiptEgressBudgets.set(this.node, nodeBudget);
|
|
2968
|
+
}
|
|
2969
|
+
const peerSessionKey = `${peer}\0${capabilitySession}`;
|
|
2970
|
+
let bucket = nodeBudget.peerSessions.get(peerSessionKey);
|
|
2971
|
+
if (!bucket) {
|
|
2972
|
+
while (nodeBudget.peerSessions.size >=
|
|
2973
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS) {
|
|
2974
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
2975
|
+
if (oldest === undefined)
|
|
2976
|
+
break;
|
|
2977
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
2978
|
+
}
|
|
2979
|
+
bucket = {
|
|
2980
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
2981
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
2982
|
+
refilledAt: now,
|
|
2983
|
+
};
|
|
2984
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
2985
|
+
}
|
|
2986
|
+
else {
|
|
2987
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
2988
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
2989
|
+
}
|
|
2990
|
+
refillPersistedReceiptIngressBucket(bucket, now, PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND);
|
|
2991
|
+
if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
|
|
2992
|
+
bucket.requestTokens -= 1;
|
|
2993
|
+
bucket.hashTokens -= hashCount;
|
|
2994
|
+
return 0;
|
|
2995
|
+
}
|
|
2996
|
+
return Math.max(1, Math.ceil(Math.max(((1 - bucket.requestTokens) /
|
|
2997
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
2998
|
+
1_000, ((hashCount - bucket.hashTokens) /
|
|
2999
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
3000
|
+
1_000)));
|
|
3001
|
+
}
|
|
3002
|
+
async waitForPersistedReceiptEgressAdmission(peer, capabilitySession, hashCount, signal) {
|
|
3003
|
+
while (true) {
|
|
3004
|
+
if (signal.aborted) {
|
|
3005
|
+
throw signal.reason ?? new AbortError();
|
|
3006
|
+
}
|
|
3007
|
+
const waitMs = this.reservePersistedReceiptEgress(peer, capabilitySession, hashCount);
|
|
3008
|
+
if (waitMs === 0)
|
|
3009
|
+
return;
|
|
3010
|
+
await this.waitPersistedReceiptRetry(signal, waitMs);
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
persistedDeliveryTimeoutMs(delivery, hashCount) {
|
|
3014
|
+
return (delivery.timeout ??
|
|
3015
|
+
Math.min(MAX_PERSISTED_DELIVERY_TIMEOUT_MS, DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
|
|
3016
|
+
persistedTransferAdmissionBudgetMs(hashCount) +
|
|
3017
|
+
Math.ceil(persistedReceiptPacingFloorMs(hashCount))));
|
|
3018
|
+
}
|
|
3019
|
+
async waitForPersistedTransferAdmission(peer, hashes, captured, signal, isStillCurrent) {
|
|
3020
|
+
const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
3021
|
+
while (!signal.aborted && isStillCurrent()) {
|
|
3022
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3023
|
+
if (!current ||
|
|
3024
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3025
|
+
current.peerSession !== captured.peerSession) {
|
|
3026
|
+
return false;
|
|
3027
|
+
}
|
|
3028
|
+
if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
|
|
3029
|
+
return true;
|
|
3030
|
+
}
|
|
3031
|
+
const remaining = expiresAt - Date.now();
|
|
3032
|
+
if (remaining <= 0)
|
|
3033
|
+
return false;
|
|
3034
|
+
await this.waitPersistedReceiptRetry(signal, Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining));
|
|
3035
|
+
}
|
|
3036
|
+
if (signal.aborted)
|
|
3037
|
+
throw signal.reason ?? new AbortError();
|
|
3038
|
+
return false;
|
|
3039
|
+
}
|
|
3040
|
+
createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, hashCount = 1) {
|
|
3041
|
+
const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
|
|
3042
|
+
if (!Number.isFinite(timeoutMs) ||
|
|
3043
|
+
timeoutMs <= 0 ||
|
|
3044
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS) {
|
|
3045
|
+
throw new Error(`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`);
|
|
3046
|
+
}
|
|
3047
|
+
const minAcks = Math.floor(delivery.minAcks);
|
|
3048
|
+
const deadlineController = new AbortController();
|
|
3049
|
+
const timeout = setTimeout(() => deadlineController.abort(new TimeoutError(`Timed out waiting for ${minAcks} persisted remote replicas.`)), timeoutMs);
|
|
3050
|
+
timeout.unref?.();
|
|
3051
|
+
return {
|
|
3052
|
+
deadline: Date.now() + timeoutMs,
|
|
3053
|
+
signal: AbortSignal.any([
|
|
3054
|
+
delivery.signal,
|
|
3055
|
+
this._closeController.signal,
|
|
3056
|
+
ownershipLifecycleController.signal,
|
|
3057
|
+
deadlineController.signal,
|
|
3058
|
+
].filter((value) => !!value)),
|
|
3059
|
+
dispose: () => clearTimeout(timeout),
|
|
3060
|
+
};
|
|
3061
|
+
}
|
|
3062
|
+
async planPersistedDeliveryLeaders(entries, replicas, ownershipLifecycleController) {
|
|
3063
|
+
if (this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry) {
|
|
3064
|
+
const leaders = [];
|
|
3065
|
+
for (const entry of entries) {
|
|
3066
|
+
leaders.push(await this.findLeadersFromEntry(entry, replicas, { freshLeaderPlan: true }, ownershipLifecycleController));
|
|
3067
|
+
}
|
|
3068
|
+
return leaders;
|
|
3069
|
+
}
|
|
3070
|
+
const items = entries.map((entry) => ({
|
|
3071
|
+
entry,
|
|
3072
|
+
replicas,
|
|
3073
|
+
options: { freshLeaderPlan: true, persist: false },
|
|
3074
|
+
}));
|
|
3075
|
+
const nativeRoutingPlanner = this._nativeRangePlanner ?? this._nativeBackbone;
|
|
3076
|
+
if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
|
|
3077
|
+
const options = items[0].options;
|
|
3078
|
+
const context = await this.createLeaderSelectionContext(options, ownershipLifecycleController);
|
|
3079
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3080
|
+
const nativeOptions = this.createNativeLeaderOptions(context, options);
|
|
3081
|
+
const fullReplicaLeaders = nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(replicas, nativeOptions);
|
|
3082
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3083
|
+
if (fullReplicaLeaders) {
|
|
3084
|
+
// Receipt settlement treats leader maps as read-only. Reusing the one
|
|
3085
|
+
// gid-independent routing result avoids one Map allocation per entry.
|
|
3086
|
+
return new Array(items.length).fill(fullReplicaLeaders);
|
|
3087
|
+
}
|
|
3088
|
+
if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
|
|
3089
|
+
const nativeLeaders = nativeRoutingPlanner.planLeaderSamplesForGidsBatch(items.map((item) => ({
|
|
3090
|
+
gid: this.getEntryGid(item.entry),
|
|
3091
|
+
replicas: item.replicas,
|
|
3092
|
+
})), nativeOptions);
|
|
3093
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3094
|
+
if (nativeLeaders?.length === items.length) {
|
|
3095
|
+
return nativeLeaders;
|
|
3096
|
+
}
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
return (await this.planEntryLeaderBatch(items, ownershipLifecycleController)).map((plan) => plan.leaders);
|
|
3100
|
+
}
|
|
3101
|
+
async settlePersistedDelivery(input, replicas, delivery, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), persistedDeadline, transferOnFirstRound = false) {
|
|
3102
|
+
const minAcks = Math.floor(delivery.minAcks);
|
|
3103
|
+
const entries = new Map(input.map((entry) => [entry.hash, entry]));
|
|
3104
|
+
if (entries.size === 0)
|
|
3105
|
+
return;
|
|
3106
|
+
const committedHashes = [...entries.keys()];
|
|
3107
|
+
const ownedDeadline = !persistedDeadline;
|
|
3108
|
+
const deadline = persistedDeadline ??
|
|
3109
|
+
this.createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, entries.size);
|
|
3110
|
+
const signal = deadline.signal;
|
|
3111
|
+
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
3112
|
+
let initialTransferPending = transferOnFirstRound;
|
|
3113
|
+
let needsInitialLeaderCheck = true;
|
|
3114
|
+
const carriedAcknowledgements = new Map(committedHashes.map((hash) => [hash, new Map()]));
|
|
3115
|
+
const repairsByPeer = new Map();
|
|
3116
|
+
let acknowledgementOwnershipRevision;
|
|
3117
|
+
const purgePeerDeliveryState = (peer) => {
|
|
3118
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3119
|
+
acknowledgements.delete(peer);
|
|
3120
|
+
}
|
|
3121
|
+
repairsByPeer.delete(peer);
|
|
3122
|
+
};
|
|
3123
|
+
try {
|
|
3124
|
+
while (true) {
|
|
3125
|
+
if (signal.aborted) {
|
|
3126
|
+
throw signal.reason ?? new AbortError();
|
|
3127
|
+
}
|
|
3128
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3129
|
+
const ownershipRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
3130
|
+
const isRoundOwnershipCurrent = () => this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
3131
|
+
if (acknowledgementOwnershipRevision !== ownershipRevision) {
|
|
3132
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3133
|
+
acknowledgements.clear();
|
|
3134
|
+
}
|
|
3135
|
+
repairsByPeer.clear();
|
|
3136
|
+
acknowledgementOwnershipRevision = ownershipRevision;
|
|
3137
|
+
}
|
|
3138
|
+
else {
|
|
3139
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3140
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3141
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3142
|
+
if (!current ||
|
|
3143
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3144
|
+
current.peerSession !== captured.peerSession) {
|
|
3145
|
+
purgePeerDeliveryState(peer);
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
if (!isRoundOwnershipCurrent()) {
|
|
3151
|
+
await this.waitPersistedReceiptRetry(signal, PERSISTED_RECEIPT_RETRY_MS);
|
|
3152
|
+
continue;
|
|
3153
|
+
}
|
|
3154
|
+
// Carry receipts only across retries in the exact same ownership and
|
|
3155
|
+
// transport epoch. A revision/session change purges them before they can
|
|
3156
|
+
// survive an away-and-back leader transition or combine with a later peer.
|
|
3157
|
+
const hashesByPeer = new Map();
|
|
3158
|
+
const entryArray = [...entries.values()];
|
|
3159
|
+
const leadersByEntry = await this.planPersistedDeliveryLeaders(entryArray, replicas, ownershipLifecycleController);
|
|
3160
|
+
if (!isRoundOwnershipCurrent())
|
|
3161
|
+
continue;
|
|
3162
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3163
|
+
if (needsInitialLeaderCheck) {
|
|
3164
|
+
needsInitialLeaderCheck = false;
|
|
3165
|
+
if (leadersByEntry.some((leaders) => leaders.size === 0 ||
|
|
3166
|
+
(leaders.size === 1 && leaders.has(selfHash)))) {
|
|
3167
|
+
throw new NoPeersError(this.rpc.topic);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
3171
|
+
const hash = entryArray[index].hash;
|
|
3172
|
+
const leaders = leadersByEntry[index];
|
|
3173
|
+
if (signal.aborted) {
|
|
3174
|
+
throw signal.reason ?? new AbortError();
|
|
3175
|
+
}
|
|
3176
|
+
if (!isRoundOwnershipCurrent())
|
|
3177
|
+
break;
|
|
3178
|
+
const acknowledgements = carriedAcknowledgements.get(hash);
|
|
3179
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3180
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3181
|
+
if (!leaders.has(peer) ||
|
|
3182
|
+
!current ||
|
|
3183
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3184
|
+
current.peerSession !== captured.peerSession) {
|
|
3185
|
+
acknowledgements.delete(peer);
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
if (acknowledgements.size >= minAcks)
|
|
3189
|
+
continue;
|
|
3190
|
+
for (const peer of leaders.keys()) {
|
|
3191
|
+
if (peer === selfHash)
|
|
3192
|
+
continue;
|
|
3193
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3194
|
+
if (!current)
|
|
3195
|
+
continue;
|
|
3196
|
+
if (acknowledgements.has(peer))
|
|
3197
|
+
continue;
|
|
3198
|
+
const hashes = hashesByPeer.get(peer) ?? [];
|
|
3199
|
+
hashes.push(hash);
|
|
3200
|
+
hashesByPeer.set(peer, hashes);
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3203
|
+
if (!isRoundOwnershipCurrent())
|
|
3204
|
+
continue;
|
|
3205
|
+
const operationQueue = new PQueue({
|
|
3206
|
+
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
3207
|
+
});
|
|
3208
|
+
const candidateWaves = Math.max(1, Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL));
|
|
3209
|
+
const roundController = new AbortController();
|
|
3210
|
+
const roundSignal = AbortSignal.any([signal, roundController.signal]);
|
|
3211
|
+
const getAttemptTimeout = () => Math.max(1, Math.min(maxAttemptMs, Math.floor((deadline.deadline - Date.now()) / candidateWaves)));
|
|
3212
|
+
const requests = new Set();
|
|
3213
|
+
const transferAllOnRound = initialTransferPending;
|
|
3214
|
+
try {
|
|
3215
|
+
for (const [peer, hashes] of hashesByPeer) {
|
|
3216
|
+
let request;
|
|
3217
|
+
request = (async () => {
|
|
3218
|
+
const captured = this.persistedReceiptPeerSession(peer);
|
|
3219
|
+
if (!captured)
|
|
3220
|
+
return;
|
|
3221
|
+
const previousRepair = repairsByPeer.get(peer);
|
|
3222
|
+
if (previousRepair &&
|
|
3223
|
+
(previousRepair.capabilitySession !==
|
|
3224
|
+
captured.capabilitySession ||
|
|
3225
|
+
previousRepair.peerSession !== captured.peerSession)) {
|
|
3226
|
+
repairsByPeer.delete(peer);
|
|
3227
|
+
}
|
|
3228
|
+
const ensureRepairState = () => {
|
|
3229
|
+
let state = repairsByPeer.get(peer);
|
|
3230
|
+
if (!state) {
|
|
3231
|
+
state = {
|
|
3232
|
+
capabilitySession: captured.capabilitySession,
|
|
3233
|
+
peerSession: captured.peerSession,
|
|
3234
|
+
hashes: new Set(),
|
|
3235
|
+
};
|
|
3236
|
+
repairsByPeer.set(peer, state);
|
|
3237
|
+
}
|
|
3238
|
+
return state;
|
|
3239
|
+
};
|
|
3240
|
+
const isPeerRoundCurrent = () => {
|
|
3241
|
+
if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
|
|
3242
|
+
return false;
|
|
3243
|
+
}
|
|
3244
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3245
|
+
return (!!current &&
|
|
3246
|
+
current.capabilitySession === captured.capabilitySession &&
|
|
3247
|
+
current.peerSession === captured.peerSession);
|
|
3248
|
+
};
|
|
3249
|
+
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
3250
|
+
const transferHashes = transferAllOnRound
|
|
3251
|
+
? hashes
|
|
3252
|
+
: repairs
|
|
3253
|
+
? hashes.filter((hash) => repairs.has(hash))
|
|
3254
|
+
: [];
|
|
3255
|
+
let attemptedHashCount = 0;
|
|
3256
|
+
if (transferHashes.length > 0) {
|
|
3257
|
+
try {
|
|
3258
|
+
await this.pushEntryHashes(peer, transferHashes, {
|
|
3259
|
+
acknowledge: false,
|
|
3260
|
+
chunkTimeout: getAttemptTimeout,
|
|
3261
|
+
chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
3262
|
+
isStillCurrent: isPeerRoundCurrent,
|
|
3263
|
+
onChunkAttempted: (chunk) => {
|
|
3264
|
+
attemptedHashCount += chunk.length;
|
|
3265
|
+
if (!transferAllOnRound) {
|
|
3266
|
+
const state = repairsByPeer.get(peer);
|
|
3267
|
+
for (const hash of chunk)
|
|
3268
|
+
state?.hashes.delete(hash);
|
|
3269
|
+
}
|
|
3270
|
+
},
|
|
3271
|
+
onChunkSent: async (chunk) => {
|
|
3272
|
+
return this.waitForPersistedTransferAdmission(peer, chunk, captured, roundSignal, isPeerRoundCurrent);
|
|
3273
|
+
},
|
|
3274
|
+
operationQueue,
|
|
3275
|
+
priority: delivery.priority,
|
|
3276
|
+
repairHint: !transferAllOnRound,
|
|
3277
|
+
signal: roundSignal,
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
catch {
|
|
3281
|
+
// A send can fail after remote admission. The attempted prefix
|
|
3282
|
+
// includes that uncertain chunk for authoritative receipt probing.
|
|
3283
|
+
}
|
|
3284
|
+
if (transferAllOnRound &&
|
|
3285
|
+
attemptedHashCount < transferHashes.length) {
|
|
3286
|
+
const state = ensureRepairState();
|
|
3287
|
+
for (const hash of transferHashes.slice(attemptedHashCount)) {
|
|
3288
|
+
state.hashes.add(hash);
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
const receiptHashes = transferAllOnRound
|
|
3293
|
+
? hashes.slice(0, attemptedHashCount)
|
|
3294
|
+
: hashes;
|
|
3295
|
+
if (!isPeerRoundCurrent()) {
|
|
3296
|
+
purgePeerDeliveryState(peer);
|
|
3297
|
+
return;
|
|
3298
|
+
}
|
|
3299
|
+
// Keep receipt chunks sequential per peer. The shared operation
|
|
3300
|
+
// queue bounds only active sends/requests; admission waits never
|
|
3301
|
+
// occupy its slots and cannot starve a later healthy candidate.
|
|
3302
|
+
for (let offset = 0; offset < receiptHashes.length; offset += PERSISTED_RECEIPT_CHUNK_SIZE) {
|
|
3303
|
+
const requestedHashes = receiptHashes.slice(offset, offset + PERSISTED_RECEIPT_CHUNK_SIZE);
|
|
3304
|
+
if (roundSignal.aborted || !isPeerRoundCurrent()) {
|
|
3305
|
+
if (!isRoundOwnershipCurrent())
|
|
3306
|
+
roundController.abort();
|
|
3307
|
+
break;
|
|
3308
|
+
}
|
|
3309
|
+
let responses;
|
|
3310
|
+
try {
|
|
3311
|
+
await this.waitForPersistedReceiptEgressAdmission(peer, captured.capabilitySession, requestedHashes.length, roundSignal);
|
|
3312
|
+
if (!isPeerRoundCurrent())
|
|
3313
|
+
break;
|
|
3314
|
+
const attemptTimeout = getAttemptTimeout();
|
|
3315
|
+
responses =
|
|
3316
|
+
(await operationQueue.add(async () => {
|
|
3317
|
+
if (!isPeerRoundCurrent())
|
|
3318
|
+
return [];
|
|
3319
|
+
return this.rpc.request(new RequestPersistedEntriesV1({
|
|
3320
|
+
expectedReceiverSession: captured.capabilitySession,
|
|
3321
|
+
hashes: requestedHashes,
|
|
3322
|
+
}), {
|
|
3323
|
+
mode: new SilentDelivery({
|
|
3324
|
+
to: [peer],
|
|
3325
|
+
redundancy: 1,
|
|
3326
|
+
}),
|
|
3327
|
+
amount: 1,
|
|
3328
|
+
priority: delivery.priority,
|
|
3329
|
+
timeout: attemptTimeout,
|
|
3330
|
+
signal: roundSignal,
|
|
3331
|
+
});
|
|
3332
|
+
})) ?? [];
|
|
3333
|
+
}
|
|
3334
|
+
catch {
|
|
3335
|
+
if (roundSignal.aborted)
|
|
3336
|
+
break;
|
|
3337
|
+
// A peer can disconnect or miss this retry while the overall
|
|
3338
|
+
// quorum deadline remains active. Replan on the next round.
|
|
3339
|
+
break;
|
|
3340
|
+
}
|
|
3341
|
+
if (!isRoundOwnershipCurrent()) {
|
|
3342
|
+
roundController.abort();
|
|
3343
|
+
break;
|
|
3344
|
+
}
|
|
3345
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3346
|
+
if (!current ||
|
|
3347
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3348
|
+
current.peerSession !== captured.peerSession) {
|
|
3349
|
+
purgePeerDeliveryState(peer);
|
|
3350
|
+
break;
|
|
3351
|
+
}
|
|
3352
|
+
const requested = new Set(requestedHashes);
|
|
3353
|
+
const confirmed = new Set();
|
|
3354
|
+
let receivedValidConfirmation = false;
|
|
3355
|
+
for (const result of responses) {
|
|
3356
|
+
if (!(result.response instanceof ConfirmEntriesMessage) ||
|
|
3357
|
+
result.from?.hashcode() !== peer ||
|
|
3358
|
+
result.message.header.session !== captured.capabilitySession) {
|
|
3359
|
+
continue;
|
|
3360
|
+
}
|
|
3361
|
+
const unique = new Set(result.response.hashes);
|
|
3362
|
+
if (unique.size !== result.response.hashes.length ||
|
|
3363
|
+
[...unique].some((hash) => !requested.has(hash))) {
|
|
3364
|
+
continue;
|
|
3365
|
+
}
|
|
3366
|
+
receivedValidConfirmation = true;
|
|
3367
|
+
for (const hash of unique) {
|
|
3368
|
+
confirmed.add(hash);
|
|
3369
|
+
carriedAcknowledgements.get(hash)?.set(peer, captured);
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
if (receivedValidConfirmation) {
|
|
3373
|
+
const state = ensureRepairState();
|
|
3374
|
+
for (const hash of requested) {
|
|
3375
|
+
if (confirmed.has(hash)) {
|
|
3376
|
+
state.hashes.delete(hash);
|
|
3377
|
+
}
|
|
3378
|
+
else {
|
|
3379
|
+
state.hashes.add(hash);
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3384
|
+
})()
|
|
3385
|
+
.then(() => undefined)
|
|
3386
|
+
.catch(() => undefined)
|
|
3387
|
+
.finally(() => requests.delete(request));
|
|
3388
|
+
requests.add(request);
|
|
3389
|
+
}
|
|
3390
|
+
const roundComplete = async () => {
|
|
3391
|
+
if (signal.aborted) {
|
|
3392
|
+
throw signal.reason ?? new AbortError();
|
|
3393
|
+
}
|
|
3394
|
+
if (!isRoundOwnershipCurrent())
|
|
3395
|
+
return false;
|
|
3396
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3397
|
+
if (acknowledgements.size < minAcks)
|
|
3398
|
+
return false;
|
|
3399
|
+
}
|
|
3400
|
+
const validatedLeaders = await this.planPersistedDeliveryLeaders(entryArray, replicas, ownershipLifecycleController);
|
|
3401
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
3402
|
+
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
3403
|
+
if (signal.aborted) {
|
|
3404
|
+
throw signal.reason ?? new AbortError();
|
|
3405
|
+
}
|
|
3406
|
+
return false;
|
|
3407
|
+
}
|
|
3408
|
+
const hash = entryArray[index].hash;
|
|
3409
|
+
const leaders = validatedLeaders[index];
|
|
3410
|
+
let valid = 0;
|
|
3411
|
+
const acknowledgements = carriedAcknowledgements.get(hash);
|
|
3412
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3413
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3414
|
+
if (!current ||
|
|
3415
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3416
|
+
current.peerSession !== captured.peerSession) {
|
|
3417
|
+
purgePeerDeliveryState(peer);
|
|
3418
|
+
continue;
|
|
3419
|
+
}
|
|
3420
|
+
if (!leaders.has(peer)) {
|
|
3421
|
+
acknowledgements.delete(peer);
|
|
3422
|
+
continue;
|
|
3423
|
+
}
|
|
3424
|
+
if (++valid >= minAcks) {
|
|
3425
|
+
break;
|
|
3426
|
+
}
|
|
3427
|
+
}
|
|
3428
|
+
if (valid < minAcks)
|
|
3429
|
+
return false;
|
|
3430
|
+
}
|
|
3431
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3432
|
+
if (signal.aborted) {
|
|
3433
|
+
throw signal.reason ?? new AbortError();
|
|
3434
|
+
}
|
|
3435
|
+
return isRoundOwnershipCurrent();
|
|
3436
|
+
};
|
|
3437
|
+
while (requests.size > 0) {
|
|
3438
|
+
await Promise.race(requests);
|
|
3439
|
+
if (await roundComplete()) {
|
|
3440
|
+
return;
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
if (await roundComplete()) {
|
|
3444
|
+
return;
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
finally {
|
|
3448
|
+
roundController.abort();
|
|
3449
|
+
void Promise.allSettled([...requests]);
|
|
3450
|
+
}
|
|
3451
|
+
if (isRoundOwnershipCurrent()) {
|
|
3452
|
+
initialTransferPending = false;
|
|
3453
|
+
}
|
|
3454
|
+
// Keep early retries fair and responsive, then let a caller's longer
|
|
3455
|
+
// overall deadline accommodate a genuinely slow durability barrier.
|
|
3456
|
+
maxAttemptMs = Math.min(MAX_PERSISTED_DELIVERY_TIMEOUT_MS, maxAttemptMs * 2);
|
|
3457
|
+
await this.waitPersistedReceiptRetry(signal, Math.max(0, Math.min(PERSISTED_RECEIPT_RETRY_MS, deadline.deadline - Date.now())));
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
catch (error) {
|
|
3461
|
+
if (error instanceof PersistedDeliveryError) {
|
|
3462
|
+
throw error;
|
|
3463
|
+
}
|
|
3464
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
3465
|
+
}
|
|
3466
|
+
finally {
|
|
3467
|
+
if (ownedDeadline)
|
|
3468
|
+
deadline.dispose();
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
2757
3471
|
async _appendDeliverToReplicators(entry, coordinates, minReplicasValue, leaders, selfHash, isLeader, deliveryArg, nativeDeliveryPlan, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
2758
3472
|
const throwIfInactive = () => this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
2759
3473
|
throwIfInactive();
|
|
@@ -4966,10 +5680,7 @@ let SharedLog = (() => {
|
|
|
4966
5680
|
try {
|
|
4967
5681
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
4968
5682
|
if (fanoutService?.provide && !this._providerHandle) {
|
|
4969
|
-
this.
|
|
4970
|
-
ttlMs: 120_000,
|
|
4971
|
-
announceIntervalMs: 60_000,
|
|
4972
|
-
});
|
|
5683
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
4973
5684
|
}
|
|
4974
5685
|
}
|
|
4975
5686
|
catch {
|
|
@@ -5370,47 +6081,93 @@ let SharedLog = (() => {
|
|
|
5370
6081
|
});
|
|
5371
6082
|
}
|
|
5372
6083
|
}
|
|
5373
|
-
async
|
|
5374
|
-
if (!isStillCurrent())
|
|
5375
|
-
return;
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
if (
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
const sentMessages = await this.trySendFusedRawExchangeHeads(
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5387
|
-
|
|
6084
|
+
async pushEntryHashChunk(target, chunk, options, isStillCurrent) {
|
|
6085
|
+
if (!isStillCurrent())
|
|
6086
|
+
return false;
|
|
6087
|
+
const useRaw = this._logProperties?.sync?.rawExchangeHeads === true &&
|
|
6088
|
+
this.peerSupportsRawExchangeHeads(target);
|
|
6089
|
+
if (useRaw) {
|
|
6090
|
+
const reserved = options.repairHint ? new Uint8Array(4) : undefined;
|
|
6091
|
+
if (reserved)
|
|
6092
|
+
reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
6093
|
+
const sentMessages = await this.trySendFusedRawExchangeHeads(chunk, [target], {
|
|
6094
|
+
acknowledge: options.acknowledge,
|
|
6095
|
+
priority: options.priority,
|
|
6096
|
+
reserved,
|
|
6097
|
+
signal: options.signal,
|
|
6098
|
+
});
|
|
6099
|
+
if (!isStillCurrent())
|
|
6100
|
+
return false;
|
|
6101
|
+
if (sentMessages === undefined) {
|
|
6102
|
+
for await (const message of createRawExchangeHeadsMessages(this.log, chunk, this._logProperties?.sync?.profile)) {
|
|
6103
|
+
if (!isStillCurrent())
|
|
6104
|
+
return false;
|
|
6105
|
+
if (options.repairHint) {
|
|
6106
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
6107
|
+
}
|
|
6108
|
+
await this.rpc.send(message, {
|
|
6109
|
+
priority: options.priority,
|
|
6110
|
+
mode: options.acknowledge
|
|
6111
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
6112
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
6113
|
+
signal: options.signal,
|
|
6114
|
+
});
|
|
6115
|
+
}
|
|
5388
6116
|
}
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
6117
|
+
}
|
|
6118
|
+
else {
|
|
6119
|
+
for await (const message of createExchangeHeadsMessages(this.log, chunk)) {
|
|
6120
|
+
if (!isStillCurrent())
|
|
6121
|
+
return false;
|
|
6122
|
+
if (options.repairHint) {
|
|
6123
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5392
6124
|
}
|
|
5393
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5394
6125
|
await this.rpc.send(message, {
|
|
5395
|
-
priority:
|
|
5396
|
-
mode:
|
|
5397
|
-
|
|
6126
|
+
priority: options.priority,
|
|
6127
|
+
mode: options.acknowledge
|
|
6128
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
6129
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
6130
|
+
signal: options.signal,
|
|
5398
6131
|
});
|
|
5399
6132
|
}
|
|
5400
|
-
return;
|
|
5401
6133
|
}
|
|
5402
|
-
|
|
5403
|
-
|
|
6134
|
+
return isStillCurrent();
|
|
6135
|
+
}
|
|
6136
|
+
async pushEntryHashes(target, hashes, options) {
|
|
6137
|
+
const isStillCurrent = options.isStillCurrent ?? (() => true);
|
|
6138
|
+
if (!isStillCurrent())
|
|
6139
|
+
return;
|
|
6140
|
+
const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
|
|
6141
|
+
for (let offset = 0; offset < hashes.length; offset += chunkSize) {
|
|
6142
|
+
const chunk = hashes.slice(offset, offset + chunkSize);
|
|
6143
|
+
const pushChunk = () => {
|
|
6144
|
+
options.onChunkAttempted?.(chunk);
|
|
6145
|
+
const chunkTimeout = options.chunkTimeout?.();
|
|
6146
|
+
const chunkSignal = chunkTimeout == null
|
|
6147
|
+
? options.signal
|
|
6148
|
+
: AbortSignal.any([
|
|
6149
|
+
...(options.signal ? [options.signal] : []),
|
|
6150
|
+
AbortSignal.timeout(Math.max(1, chunkTimeout)),
|
|
6151
|
+
]);
|
|
6152
|
+
return this.pushEntryHashChunk(target, chunk, { ...options, signal: chunkSignal }, isStillCurrent);
|
|
6153
|
+
};
|
|
6154
|
+
const pushed = options.operationQueue
|
|
6155
|
+
? await options.operationQueue.add(pushChunk)
|
|
6156
|
+
: await pushChunk();
|
|
6157
|
+
if (pushed !== true)
|
|
6158
|
+
return;
|
|
6159
|
+
if (options.onChunkSent && !(await options.onChunkSent(chunk)))
|
|
5404
6160
|
return;
|
|
5405
|
-
}
|
|
5406
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5407
|
-
await this.rpc.send(message, {
|
|
5408
|
-
priority: SYNC_MESSAGE_PRIORITY,
|
|
5409
|
-
mode: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
5410
|
-
signal,
|
|
5411
|
-
});
|
|
5412
6161
|
}
|
|
5413
6162
|
}
|
|
6163
|
+
async pushRepairEntries(target, entries, isStillCurrent = () => true, signal) {
|
|
6164
|
+
return this.pushEntryHashes(target, [...entries.keys()], {
|
|
6165
|
+
isStillCurrent,
|
|
6166
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
6167
|
+
repairHint: true,
|
|
6168
|
+
signal,
|
|
6169
|
+
});
|
|
6170
|
+
}
|
|
5414
6171
|
async sendRepairEntriesWithTransport(target, entries, transport, options) {
|
|
5415
6172
|
const isStillCurrent = options?.isStillCurrent ?? (() => true);
|
|
5416
6173
|
if (!isStillCurrent()) {
|
|
@@ -6788,23 +7545,65 @@ let SharedLog = (() => {
|
|
|
6788
7545
|
}
|
|
6789
7546
|
async append(data, options) {
|
|
6790
7547
|
this.throwIfNativeDurableCommitFailed();
|
|
7548
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
6791
7549
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6792
7550
|
if (this._isAdaptiveReplicating) {
|
|
6793
7551
|
this.markLocalAppendActivity();
|
|
6794
7552
|
}
|
|
6795
7553
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options, ownershipLifecycleController);
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
6801
|
-
|
|
6802
|
-
|
|
6803
|
-
|
|
7554
|
+
let committedHashes;
|
|
7555
|
+
if (persistedDelivery) {
|
|
7556
|
+
appendOptions.__peerbitOnLocalCommit = (hashes) => {
|
|
7557
|
+
// The lower log reports the exact hashes immediately after their
|
|
7558
|
+
// irreversible local mutation and before entry initialization, trim,
|
|
7559
|
+
// or change callbacks can reject the append.
|
|
7560
|
+
committedHashes = hashes;
|
|
7561
|
+
};
|
|
7562
|
+
}
|
|
7563
|
+
let persistedDeadline;
|
|
7564
|
+
const throwIfDeliveryAborted = () => {
|
|
7565
|
+
if (persistedDeadline?.signal.aborted) {
|
|
7566
|
+
throw persistedDeadline.signal.reason ?? new AbortError();
|
|
7567
|
+
}
|
|
7568
|
+
};
|
|
7569
|
+
try {
|
|
7570
|
+
const result = await this.log.append(data, appendOptions);
|
|
7571
|
+
committedHashes ??= [result.entry.hash];
|
|
7572
|
+
persistedDeadline = persistedDelivery
|
|
7573
|
+
? this.createPersistedDeliveryDeadline(persistedDelivery, ownershipLifecycleController, 1)
|
|
7574
|
+
: undefined;
|
|
7575
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7576
|
+
throwIfDeliveryAborted();
|
|
7577
|
+
await this.processLocalAppend(result.entry, result.removed, options, {
|
|
7578
|
+
minReplicasValue,
|
|
7579
|
+
ownershipLifecycleController,
|
|
7580
|
+
});
|
|
7581
|
+
throwIfDeliveryAborted();
|
|
7582
|
+
if (persistedDelivery && persistedDeadline) {
|
|
7583
|
+
await this.settlePersistedDelivery([result.entry], minReplicasValue, persistedDelivery, ownershipLifecycleController, persistedDeadline);
|
|
7584
|
+
}
|
|
7585
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7586
|
+
return result;
|
|
7587
|
+
}
|
|
7588
|
+
catch (error) {
|
|
7589
|
+
if (persistedDelivery && committedHashes) {
|
|
7590
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
7591
|
+
}
|
|
7592
|
+
throw error;
|
|
7593
|
+
}
|
|
7594
|
+
finally {
|
|
7595
|
+
persistedDeadline?.dispose();
|
|
7596
|
+
}
|
|
7597
|
+
}
|
|
7598
|
+
rejectPersistedDeliveryOnTrustedLocalAppend(options) {
|
|
7599
|
+
if (this.getPersistedDeliveryOptions(options)) {
|
|
7600
|
+
throw new Error("trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit");
|
|
7601
|
+
}
|
|
6804
7602
|
}
|
|
6805
7603
|
// Trusted local append path for callers that already validated the entry.
|
|
6806
7604
|
async appendLocallyValidated(data, options) {
|
|
6807
7605
|
this.throwIfNativeDurableCommitFailed();
|
|
7606
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6808
7607
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6809
7608
|
if (options?.canAppend || options?.onChange) {
|
|
6810
7609
|
throw new Error("appendLocallyValidated does not accept canAppend or onChange hooks");
|
|
@@ -6827,6 +7626,7 @@ let SharedLog = (() => {
|
|
|
6827
7626
|
// Trusted local append path that lets the shared log own change application.
|
|
6828
7627
|
async appendLocallyPrepared(data, options, properties) {
|
|
6829
7628
|
this.throwIfNativeDurableCommitFailed();
|
|
7629
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6830
7630
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6831
7631
|
if (options?.canAppend || options?.onChange) {
|
|
6832
7632
|
throw new Error("appendLocallyPrepared does not accept canAppend or onChange hooks");
|
|
@@ -6836,11 +7636,15 @@ let SharedLog = (() => {
|
|
|
6836
7636
|
}
|
|
6837
7637
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
6838
7638
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7639
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
6839
7640
|
const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(data, appendOptions, {
|
|
6840
7641
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
6841
7642
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
6842
7643
|
payloadData: properties?.payloadData,
|
|
6843
7644
|
});
|
|
7645
|
+
if (properties?.localCommitEvidence) {
|
|
7646
|
+
properties.localCommitEvidence.committedHashes.add(result.appendFacts.hash);
|
|
7647
|
+
}
|
|
6844
7648
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
6845
7649
|
const nativePreparedCommit = await this.processNativePreparedTargetNoneAppend(result, options, {
|
|
6846
7650
|
minReplicasValue,
|
|
@@ -6989,11 +7793,15 @@ let SharedLog = (() => {
|
|
|
6989
7793
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
6990
7794
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
6991
7795
|
payloadData,
|
|
7796
|
+
...(properties?.localCommitEvidence
|
|
7797
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
7798
|
+
: undefined),
|
|
6992
7799
|
});
|
|
6993
7800
|
}
|
|
6994
7801
|
// Trusted local payload append path that keeps the public Entry lazy.
|
|
6995
7802
|
appendLocallyPreparedPayloadCommitOnly(payloadData, options, properties) {
|
|
6996
7803
|
this.throwIfNativeDurableCommitFailed();
|
|
7804
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6997
7805
|
if (options?.canAppend || options?.onChange) {
|
|
6998
7806
|
throw new Error("appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
6999
7807
|
}
|
|
@@ -7010,6 +7818,7 @@ let SharedLog = (() => {
|
|
|
7010
7818
|
}
|
|
7011
7819
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
7012
7820
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7821
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
7013
7822
|
const deferHeadCoordinatePersistence = this.shouldDeferHeadCoordinatePersistence(options);
|
|
7014
7823
|
const nativeBackboneResult = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(payloadData, appendOptions, options, properties, minReplicasValue, deferHeadCoordinatePersistence, ownershipLifecycleController);
|
|
7015
7824
|
if (nativeBackboneResult) {
|
|
@@ -7026,6 +7835,7 @@ let SharedLog = (() => {
|
|
|
7026
7835
|
// Strict native document path. Never falls back to compatibility append.
|
|
7027
7836
|
appendStrictNativeDocumentPayloadCommitOnly(payloadData, options, properties) {
|
|
7028
7837
|
this.throwIfNativeDurableCommitFailed();
|
|
7838
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
7029
7839
|
if (options?.canAppend || options?.onChange) {
|
|
7030
7840
|
throw new Error("appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
7031
7841
|
}
|
|
@@ -7042,6 +7852,7 @@ let SharedLog = (() => {
|
|
|
7042
7852
|
}
|
|
7043
7853
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
7044
7854
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7855
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
7045
7856
|
const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(payloadData, appendOptions, options, properties, minReplicasValue, this.shouldDeferHeadCoordinatePersistence(options), ownershipLifecycleController);
|
|
7046
7857
|
return mapMaybePromise(result, (commitOnly) => {
|
|
7047
7858
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
@@ -7058,6 +7869,9 @@ let SharedLog = (() => {
|
|
|
7058
7869
|
includeAppendFactsBytes: !deferHeadCoordinatePersistence,
|
|
7059
7870
|
});
|
|
7060
7871
|
return mapMaybePromise(resultMaybe, (result) => {
|
|
7872
|
+
if (result && properties?.localCommitEvidence) {
|
|
7873
|
+
properties.localCommitEvidence.committedHashes.add(result.appendFacts.hash);
|
|
7874
|
+
}
|
|
7061
7875
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7062
7876
|
return this.finishPreparedPayloadCommitOnlyAppend(result, options, minReplicasValue, ownershipLifecycleController);
|
|
7063
7877
|
});
|
|
@@ -7438,17 +8252,16 @@ let SharedLog = (() => {
|
|
|
7438
8252
|
// boundary separates the catch above from this statement and
|
|
7439
8253
|
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
7440
8254
|
// (the retire below only warns), so the token is terminal here.
|
|
7441
|
-
|
|
7442
|
-
|
|
7443
|
-
removedHashes: prepared.removedHashes,
|
|
7444
|
-
removedGids: prepared.removedGids,
|
|
7445
|
-
});
|
|
7446
|
-
try {
|
|
7447
|
-
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7448
|
-
}
|
|
7449
|
-
catch (error) {
|
|
7450
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
8255
|
+
if (properties?.localCommitEvidence) {
|
|
8256
|
+
properties.localCommitEvidence.committedHashes.add(prepared.appendFacts.hash);
|
|
7451
8257
|
}
|
|
8258
|
+
await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
8259
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
8260
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
8261
|
+
removedHashes: prepared.removedHashes,
|
|
8262
|
+
removedGids: prepared.removedGids,
|
|
8263
|
+
});
|
|
8264
|
+
});
|
|
7452
8265
|
return finishResult;
|
|
7453
8266
|
});
|
|
7454
8267
|
}
|
|
@@ -7889,18 +8702,17 @@ let SharedLog = (() => {
|
|
|
7889
8702
|
// Success seam: the finalizer acknowledge above is the last
|
|
7890
8703
|
// await inside the protected try, so `rollback` can no
|
|
7891
8704
|
// longer fire and the retire below only warns.
|
|
7892
|
-
|
|
7893
|
-
|
|
7894
|
-
forgetNativeCoordinates: false,
|
|
7895
|
-
removedHashes: prepared.removedHashes,
|
|
7896
|
-
removedGids: prepared.removedGids,
|
|
7897
|
-
});
|
|
7898
|
-
try {
|
|
7899
|
-
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7900
|
-
}
|
|
7901
|
-
catch (error) {
|
|
7902
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
8705
|
+
if (properties?.localCommitEvidence) {
|
|
8706
|
+
properties.localCommitEvidence.committedHashes.add(prepared.appendFacts.hash);
|
|
7903
8707
|
}
|
|
8708
|
+
await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
8709
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollbackCoordinateEntries);
|
|
8710
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
8711
|
+
forgetNativeCoordinates: false,
|
|
8712
|
+
removedHashes: prepared.removedHashes,
|
|
8713
|
+
removedGids: prepared.removedGids,
|
|
8714
|
+
});
|
|
8715
|
+
});
|
|
7904
8716
|
if (commitBlocksInBackbone &&
|
|
7905
8717
|
!runtimeOnlyCoordinates &&
|
|
7906
8718
|
this.remoteBlocks.hasNotifyStoredHook()) {
|
|
@@ -8569,50 +9381,49 @@ let SharedLog = (() => {
|
|
|
8569
9381
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8570
9382
|
return this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction);
|
|
8571
9383
|
});
|
|
8572
|
-
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8573
9384
|
}
|
|
8574
9385
|
catch (error) {
|
|
8575
9386
|
return rollbackBatch(error);
|
|
8576
9387
|
}
|
|
8577
9388
|
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
8578
9389
|
// above), and everything from here on escapes without any rollback.
|
|
8579
|
-
|
|
8580
|
-
|
|
8581
|
-
|
|
8582
|
-
for (let i = 0; i < coordinateRows.length; i++) {
|
|
8583
|
-
const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
|
|
8584
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
|
|
8585
|
-
forgetNativeCoordinates: false,
|
|
8586
|
-
removedHashes: plannedCoordinateDeleteHashes,
|
|
8587
|
-
removedGids: backboneAppend.trimmedGids ??
|
|
8588
|
-
(backboneAppend.trimmed.length > 0
|
|
8589
|
-
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
8590
|
-
: undefined),
|
|
8591
|
-
});
|
|
8592
|
-
if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
|
|
8593
|
-
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
9390
|
+
if (properties?.localCommitEvidence) {
|
|
9391
|
+
for (const facts of appended.appendFacts) {
|
|
9392
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
8594
9393
|
}
|
|
8595
|
-
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
|
|
8596
|
-
hashNumber: backboneAppend.coordinate.hashNumber,
|
|
8597
|
-
coordinateFields,
|
|
8598
|
-
});
|
|
8599
|
-
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
8600
|
-
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
8601
|
-
appended.documentTrimmedHeadsProcessed?.[i];
|
|
8602
|
-
appendCommit.documentPreviousContext =
|
|
8603
|
-
backboneAppend.documentPreviousContext;
|
|
8604
|
-
appendCommits.push(appendCommit);
|
|
8605
9394
|
}
|
|
8606
|
-
|
|
8607
|
-
|
|
9395
|
+
const appendCommits = await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
9396
|
+
this._coordinates.settleResidentCoordinateSnapshot(batchCoordinateRollback);
|
|
8608
9397
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
8612
|
-
|
|
8613
|
-
|
|
8614
|
-
|
|
8615
|
-
|
|
9398
|
+
const commits = [];
|
|
9399
|
+
for (let i = 0; i < coordinateRows.length; i++) {
|
|
9400
|
+
const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
|
|
9401
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
|
|
9402
|
+
forgetNativeCoordinates: false,
|
|
9403
|
+
removedHashes: plannedCoordinateDeleteHashes,
|
|
9404
|
+
removedGids: backboneAppend.trimmedGids ??
|
|
9405
|
+
(backboneAppend.trimmed.length > 0
|
|
9406
|
+
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
9407
|
+
: undefined),
|
|
9408
|
+
});
|
|
9409
|
+
if (!runtimeOnlyCoordinates &&
|
|
9410
|
+
this.remoteBlocks.hasNotifyStoredHook()) {
|
|
9411
|
+
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
9412
|
+
}
|
|
9413
|
+
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
|
|
9414
|
+
hashNumber: backboneAppend.coordinate
|
|
9415
|
+
.hashNumber,
|
|
9416
|
+
coordinateFields,
|
|
9417
|
+
});
|
|
9418
|
+
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
9419
|
+
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
9420
|
+
appended.documentTrimmedHeadsProcessed?.[i];
|
|
9421
|
+
appendCommit.documentPreviousContext =
|
|
9422
|
+
backboneAppend.documentPreviousContext;
|
|
9423
|
+
commits.push(appendCommit);
|
|
9424
|
+
}
|
|
9425
|
+
return commits;
|
|
9426
|
+
}, () => this.isRepairLifecycleActive(ownershipLifecycleController));
|
|
8616
9427
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8617
9428
|
const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
|
|
8618
9429
|
if (!delayAdaptiveRebalance) {
|
|
@@ -8629,6 +9440,7 @@ let SharedLog = (() => {
|
|
|
8629
9440
|
}
|
|
8630
9441
|
async appendLocallyPreparedManyIndependent(data, options, properties) {
|
|
8631
9442
|
this.throwIfNativeDurableCommitFailed();
|
|
9443
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
8632
9444
|
if (data.length === 0) {
|
|
8633
9445
|
return { entries: [], removed: [], appendCommits: [] };
|
|
8634
9446
|
}
|
|
@@ -8641,6 +9453,7 @@ let SharedLog = (() => {
|
|
|
8641
9453
|
}
|
|
8642
9454
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
8643
9455
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
9456
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
8644
9457
|
const nativeBackboneBatch = await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(data, appendOptions, options, properties, minReplicasValue, ownershipLifecycleController);
|
|
8645
9458
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8646
9459
|
if (nativeBackboneBatch) {
|
|
@@ -8651,6 +9464,11 @@ let SharedLog = (() => {
|
|
|
8651
9464
|
payloadDatas: properties?.payloadDatas,
|
|
8652
9465
|
nexts: properties?.nexts,
|
|
8653
9466
|
});
|
|
9467
|
+
if (result && properties?.localCommitEvidence) {
|
|
9468
|
+
for (const facts of result.appendFacts) {
|
|
9469
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
9470
|
+
}
|
|
9471
|
+
}
|
|
8654
9472
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8655
9473
|
if (!result) {
|
|
8656
9474
|
return undefined;
|
|
@@ -8719,13 +9537,20 @@ let SharedLog = (() => {
|
|
|
8719
9537
|
nexts: properties?.nexts,
|
|
8720
9538
|
nativeBackboneDocumentIndexes: properties?.nativeBackboneDocumentIndexes,
|
|
8721
9539
|
retainMaterializationBytes: properties?.retainMaterializationBytes,
|
|
9540
|
+
...(properties?.localCommitEvidence
|
|
9541
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
9542
|
+
: undefined),
|
|
8722
9543
|
});
|
|
8723
9544
|
}
|
|
8724
9545
|
async appendMany(data, options) {
|
|
8725
9546
|
this.throwIfNativeDurableCommitFailed();
|
|
9547
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
8726
9548
|
if (data.length === 0) {
|
|
8727
9549
|
return { entries: [], removed: [] };
|
|
8728
9550
|
}
|
|
9551
|
+
if (persistedDelivery) {
|
|
9552
|
+
throw new Error("persisted delivery is not supported for chained appendMany; use independent document puts");
|
|
9553
|
+
}
|
|
8729
9554
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
8730
9555
|
if (this._isAdaptiveReplicating) {
|
|
8731
9556
|
this.markLocalAppendActivity();
|
|
@@ -8768,6 +9593,69 @@ let SharedLog = (() => {
|
|
|
8768
9593
|
}
|
|
8769
9594
|
return result;
|
|
8770
9595
|
}
|
|
9596
|
+
/**
|
|
9597
|
+
* Deliver entries that were already committed locally, then wait for the
|
|
9598
|
+
* requested persisted remote quorum. This is the post-commit seam used by
|
|
9599
|
+
* higher-level transactional/batched writers so a receipt timeout never
|
|
9600
|
+
* rolls back or hides their successful local commit.
|
|
9601
|
+
*/
|
|
9602
|
+
async deliverPersistedPlanningEntries(resolveEntries, committedHashes, options) {
|
|
9603
|
+
let persistedDeadline;
|
|
9604
|
+
try {
|
|
9605
|
+
const delivery = this.getPersistedDeliveryOptions(options);
|
|
9606
|
+
if (!delivery) {
|
|
9607
|
+
throw new Error('deliverPersistedEntries requires reliability="persisted"');
|
|
9608
|
+
}
|
|
9609
|
+
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
9610
|
+
const deadline = this.createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, committedHashes.length);
|
|
9611
|
+
persistedDeadline = deadline;
|
|
9612
|
+
const throwIfDeliveryAborted = () => {
|
|
9613
|
+
if (deadline.signal.aborted) {
|
|
9614
|
+
throw deadline.signal.reason ?? new AbortError();
|
|
9615
|
+
}
|
|
9616
|
+
if (Date.now() >= deadline.deadline) {
|
|
9617
|
+
throw new TimeoutError(`Timed out waiting for ${Math.floor(delivery.minAcks)} persisted remote replicas.`);
|
|
9618
|
+
}
|
|
9619
|
+
};
|
|
9620
|
+
const entries = resolveEntries();
|
|
9621
|
+
throwIfDeliveryAborted();
|
|
9622
|
+
if (entries.length === 0)
|
|
9623
|
+
return;
|
|
9624
|
+
const { minReplicasValue } = this.createLogAppendOptions(options, ownershipLifecycleController);
|
|
9625
|
+
throwIfDeliveryAborted();
|
|
9626
|
+
await this.settlePersistedDelivery(entries, minReplicasValue, delivery, ownershipLifecycleController, deadline, true);
|
|
9627
|
+
}
|
|
9628
|
+
catch (error) {
|
|
9629
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
9630
|
+
}
|
|
9631
|
+
finally {
|
|
9632
|
+
persistedDeadline?.dispose();
|
|
9633
|
+
}
|
|
9634
|
+
}
|
|
9635
|
+
createPersistedDeliveryPlanningEntries(appendCommits, materializeEntries) {
|
|
9636
|
+
let materializedEntries;
|
|
9637
|
+
const requiresFullEntries = this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
|
|
9638
|
+
const getMaterializedEntry = (appendCommit, index) => {
|
|
9639
|
+
materializedEntries ??= materializeEntries();
|
|
9640
|
+
const entry = materializedEntries[index];
|
|
9641
|
+
if (!entry || entry.hash !== appendCommit.hash) {
|
|
9642
|
+
throw new Error(`Persisted delivery materializer did not return committed entry ${appendCommit.hash}`);
|
|
9643
|
+
}
|
|
9644
|
+
return entry;
|
|
9645
|
+
};
|
|
9646
|
+
return appendCommits.map((appendCommit, index) => {
|
|
9647
|
+
if (!requiresFullEntries && appendCommit.coordinateFields) {
|
|
9648
|
+
return this._coordinates.materializeResidentCoordinateEntry(appendCommit.coordinateFields);
|
|
9649
|
+
}
|
|
9650
|
+
return getMaterializedEntry(appendCommit, index);
|
|
9651
|
+
});
|
|
9652
|
+
}
|
|
9653
|
+
deliverPersistedAppendCommits(appendCommits, materializeEntries, options) {
|
|
9654
|
+
return this.deliverPersistedPlanningEntries(() => this.createPersistedDeliveryPlanningEntries(appendCommits, materializeEntries), appendCommits.map((appendCommit) => appendCommit.hash), options);
|
|
9655
|
+
}
|
|
9656
|
+
async deliverPersistedEntries(entries, options) {
|
|
9657
|
+
return this.deliverPersistedPlanningEntries(() => entries, entries.map((entry) => entry.hash), options);
|
|
9658
|
+
}
|
|
8771
9659
|
canCoalesceLocalAppendMany(entries, options) {
|
|
8772
9660
|
if (entries.length <= 1 ||
|
|
8773
9661
|
options?.target === "all" ||
|
|
@@ -9532,6 +10420,9 @@ let SharedLog = (() => {
|
|
|
9532
10420
|
this._peerSyncCapabilities = new Map();
|
|
9533
10421
|
this._peerSyncCapabilitySessions = new Map();
|
|
9534
10422
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
10423
|
+
this._persistedReceiptStorage = undefined;
|
|
10424
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
10425
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
9535
10426
|
this._liveRawGossipBatches = new Map();
|
|
9536
10427
|
this._liveRawGossipFlushScheduled = false;
|
|
9537
10428
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -9618,6 +10509,43 @@ let SharedLog = (() => {
|
|
|
9618
10509
|
]);
|
|
9619
10510
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
9620
10511
|
const blockProviderNamespace = (cid) => `cid:${cid}`;
|
|
10512
|
+
const logProviderNamespace = `shared-log|${this.topic}`;
|
|
10513
|
+
const announceBlockProvider = async (cid) => {
|
|
10514
|
+
try {
|
|
10515
|
+
await fanoutService?.announceProvider(blockProviderNamespace(cid), {
|
|
10516
|
+
ttlMs: 120_000,
|
|
10517
|
+
bootstrapMaxPeers: 2,
|
|
10518
|
+
});
|
|
10519
|
+
}
|
|
10520
|
+
catch {
|
|
10521
|
+
// Provider publication is best-effort.
|
|
10522
|
+
}
|
|
10523
|
+
};
|
|
10524
|
+
const announceBlockProviders = async (cids) => {
|
|
10525
|
+
const batchedAnnounce = fanoutService?.announceProviders;
|
|
10526
|
+
if (typeof batchedAnnounce === "function") {
|
|
10527
|
+
const namespaces = function* () {
|
|
10528
|
+
for (const cid of cids)
|
|
10529
|
+
yield blockProviderNamespace(cid);
|
|
10530
|
+
};
|
|
10531
|
+
await batchedAnnounce.call(fanoutService, namespaces(), {
|
|
10532
|
+
ttlMs: 120_000,
|
|
10533
|
+
bootstrapMaxPeers: 2,
|
|
10534
|
+
});
|
|
10535
|
+
return;
|
|
10536
|
+
}
|
|
10537
|
+
// Tolerate a skewed runtime that supplied an older FanoutTree service.
|
|
10538
|
+
let nextIndex = 0;
|
|
10539
|
+
const worker = async () => {
|
|
10540
|
+
for (;;) {
|
|
10541
|
+
const index = nextIndex++;
|
|
10542
|
+
if (index >= cids.length)
|
|
10543
|
+
return;
|
|
10544
|
+
await announceBlockProvider(cids[index]);
|
|
10545
|
+
}
|
|
10546
|
+
};
|
|
10547
|
+
await Promise.all(Array.from({ length: Math.min(8, cids.length) }, () => worker()));
|
|
10548
|
+
};
|
|
9621
10549
|
const [replicationIndex, logIndex] = await Promise.all([
|
|
9622
10550
|
logScope.scope("replication"),
|
|
9623
10551
|
logScope.scope("log"),
|
|
@@ -9699,14 +10627,22 @@ let SharedLog = (() => {
|
|
|
9699
10627
|
}
|
|
9700
10628
|
let directoryProviders = [];
|
|
9701
10629
|
try {
|
|
9702
|
-
|
|
9703
|
-
|
|
9704
|
-
|
|
9705
|
-
|
|
9706
|
-
|
|
9707
|
-
|
|
9708
|
-
|
|
9709
|
-
|
|
10630
|
+
const query = (namespace) => fanoutService?.queryProviders(namespace, {
|
|
10631
|
+
want: maxPeers,
|
|
10632
|
+
timeoutMs: 2_000,
|
|
10633
|
+
queryTimeoutMs: 500,
|
|
10634
|
+
bootstrapMaxPeers: 2,
|
|
10635
|
+
signal: opts?.signal,
|
|
10636
|
+
}) ?? Promise.resolve([]);
|
|
10637
|
+
const results = await Promise.allSettled([
|
|
10638
|
+
query(blockProviderNamespace(cid)),
|
|
10639
|
+
query(logProviderNamespace),
|
|
10640
|
+
]);
|
|
10641
|
+
for (const result of results) {
|
|
10642
|
+
if (result.status === "fulfilled") {
|
|
10643
|
+
directoryProviders.push(...result.value);
|
|
10644
|
+
}
|
|
10645
|
+
}
|
|
9710
10646
|
}
|
|
9711
10647
|
catch {
|
|
9712
10648
|
// Ignore discovery failures; local evidence remains usable.
|
|
@@ -9714,9 +10650,7 @@ let SharedLog = (() => {
|
|
|
9714
10650
|
const selected = [];
|
|
9715
10651
|
const selectedSet = new Set();
|
|
9716
10652
|
const add = (peer) => {
|
|
9717
|
-
if (!peer ||
|
|
9718
|
-
selectedSet.has(peer) ||
|
|
9719
|
-
selected.length >= maxPeers) {
|
|
10653
|
+
if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
|
|
9720
10654
|
return;
|
|
9721
10655
|
}
|
|
9722
10656
|
selectedSet.add(peer);
|
|
@@ -9730,27 +10664,46 @@ let SharedLog = (() => {
|
|
|
9730
10664
|
return selected;
|
|
9731
10665
|
},
|
|
9732
10666
|
watchProviders: fanoutService
|
|
9733
|
-
? (cid, opts) =>
|
|
9734
|
-
|
|
9735
|
-
|
|
9736
|
-
|
|
9737
|
-
|
|
9738
|
-
|
|
9739
|
-
|
|
9740
|
-
|
|
10667
|
+
? (cid, opts) => {
|
|
10668
|
+
const watch = (namespace) => fanoutService.watchProviders(namespace, {
|
|
10669
|
+
signal: opts.signal,
|
|
10670
|
+
want: 8,
|
|
10671
|
+
ttlMs: 10_000,
|
|
10672
|
+
renewIntervalMs: 5_000,
|
|
10673
|
+
bootstrapMaxPeers: 2,
|
|
10674
|
+
onProviders: (providers) => opts.onProviders(providers.map((provider) => provider.hash)),
|
|
10675
|
+
});
|
|
10676
|
+
const cidWatch = watch(blockProviderNamespace(cid));
|
|
10677
|
+
try {
|
|
10678
|
+
const logWatch = watch(logProviderNamespace);
|
|
10679
|
+
return {
|
|
10680
|
+
close: () => {
|
|
10681
|
+
cidWatch.close();
|
|
10682
|
+
logWatch.close();
|
|
10683
|
+
},
|
|
10684
|
+
};
|
|
10685
|
+
}
|
|
10686
|
+
catch (error) {
|
|
10687
|
+
cidWatch.close();
|
|
10688
|
+
throw error;
|
|
10689
|
+
}
|
|
10690
|
+
}
|
|
9741
10691
|
: undefined,
|
|
9742
|
-
onPut: fanoutService
|
|
9743
|
-
|
|
9744
|
-
|
|
10692
|
+
onPut: fanoutService ? announceBlockProvider : undefined,
|
|
10693
|
+
onPutMany: fanoutService
|
|
10694
|
+
? (cids) => {
|
|
10695
|
+
// A renewable log-wide provider lease makes every CID in a
|
|
10696
|
+
// stored batch discoverable to current readers. Retain the
|
|
10697
|
+
// per-CID directory publications for released readers that only
|
|
10698
|
+
// know the legacy namespace; the bounded workers avoid creating
|
|
10699
|
+
// one in-flight promise per block.
|
|
9745
10700
|
try {
|
|
9746
|
-
|
|
9747
|
-
ttlMs: 120_000,
|
|
9748
|
-
bootstrapMaxPeers: 2,
|
|
9749
|
-
});
|
|
10701
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
9750
10702
|
}
|
|
9751
10703
|
catch {
|
|
9752
10704
|
// ignore announce failures
|
|
9753
10705
|
}
|
|
10706
|
+
return announceBlockProviders(cids);
|
|
9754
10707
|
}
|
|
9755
10708
|
: undefined,
|
|
9756
10709
|
});
|
|
@@ -9943,6 +10896,7 @@ let SharedLog = (() => {
|
|
|
9943
10896
|
},
|
|
9944
10897
|
indexer: logIndex,
|
|
9945
10898
|
});
|
|
10899
|
+
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
9946
10900
|
try {
|
|
9947
10901
|
const recovered = await this.recoverNativeStrictDurableTransactionIntent();
|
|
9948
10902
|
if (recovered) {
|
|
@@ -10549,8 +11503,7 @@ let SharedLog = (() => {
|
|
|
10549
11503
|
const subscriberDiscoveryPromise = this._getTopicSubscribers(this.topic);
|
|
10550
11504
|
const existingSubscribersPromise = this.node.services.pubsub.getSubscribers(this.topic);
|
|
10551
11505
|
// We do this here, because these calls requires this.closed == false
|
|
10552
|
-
void this.pruneOfflineReplicators()
|
|
10553
|
-
.catch((error) => {
|
|
11506
|
+
void this.pruneOfflineReplicators().catch((error) => {
|
|
10554
11507
|
if (isNotStartedError(error)) {
|
|
10555
11508
|
return;
|
|
10556
11509
|
}
|
|
@@ -11927,6 +12880,9 @@ let SharedLog = (() => {
|
|
|
11927
12880
|
this._peerSyncCapabilities?.clear();
|
|
11928
12881
|
this._peerSyncCapabilitySessions?.clear();
|
|
11929
12882
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
12883
|
+
this._persistedReceiptStorage = undefined;
|
|
12884
|
+
this._persistedReceiptRequestsInFlight?.clear();
|
|
12885
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
11930
12886
|
this._v2Receive?.clearForClose();
|
|
11931
12887
|
this._v2Send?.clearForClose();
|
|
11932
12888
|
this._liveRawGossipBatches?.clear();
|
|
@@ -12582,7 +13538,6 @@ let SharedLog = (() => {
|
|
|
12582
13538
|
preparedSelection: rawPreparedReceiveSelectionValue,
|
|
12583
13539
|
};
|
|
12584
13540
|
}
|
|
12585
|
-
// Callback for receiving a message from the network
|
|
12586
13541
|
async onMessage(msg, context) {
|
|
12587
13542
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
12588
13543
|
? msg
|
|
@@ -12644,6 +13599,7 @@ let SharedLog = (() => {
|
|
|
12644
13599
|
}
|
|
12645
13600
|
if (!context.from.equals(this.node.identity.publicKey) &&
|
|
12646
13601
|
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
13602
|
+
!(msg instanceof RequestPersistedEntriesV1) &&
|
|
12647
13603
|
!isReplicationInfoV2Message(msg)) {
|
|
12648
13604
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
12649
13605
|
}
|
|
@@ -13987,7 +14943,10 @@ let SharedLog = (() => {
|
|
|
13987
14943
|
};
|
|
13988
14944
|
// The prelude already threw when `context.from` was missing.
|
|
13989
14945
|
const laneRequestContext = context;
|
|
13990
|
-
if (msg instanceof
|
|
14946
|
+
if (msg instanceof RequestPersistedEntriesV1) {
|
|
14947
|
+
return await this.handleRequestPersistedEntriesV1(msg, laneRequestContext, lane);
|
|
14948
|
+
}
|
|
14949
|
+
else if (msg instanceof RequestIPruneV2) {
|
|
13991
14950
|
await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
|
|
13992
14951
|
}
|
|
13993
14952
|
else if (msg instanceof ResponseIPruneV2) {
|
|
@@ -14180,6 +15139,221 @@ let SharedLog = (() => {
|
|
|
14180
15139
|
// `onMessage`'s shared catch/finally envelope (error classification,
|
|
14181
15140
|
// wire-stash release, lease release, durable-poison recheck).
|
|
14182
15141
|
// -----------------------------------------------------------------
|
|
15142
|
+
isPersistedReceiptRequestSessionCurrent(request, context, lane) {
|
|
15143
|
+
const session = lane.session;
|
|
15144
|
+
return (!!this._persistedReceiptStorage &&
|
|
15145
|
+
!context.from.equals(this.node.identity.publicKey) &&
|
|
15146
|
+
request.expectedReceiverSession === this.ownTransportSession() &&
|
|
15147
|
+
session !== null &&
|
|
15148
|
+
session.phase === "open" &&
|
|
15149
|
+
this._peerSessions.isCurrent(lane.fromHash, session) &&
|
|
15150
|
+
!this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
|
|
15151
|
+
this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
|
|
15152
|
+
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
15153
|
+
context.message.header.session &&
|
|
15154
|
+
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
15155
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController));
|
|
15156
|
+
}
|
|
15157
|
+
admitPersistedReceiptIngress(peer, transportSession, hashCount, now = Date.now()) {
|
|
15158
|
+
// Charge malformed empty/oversized vectors too. The request was already
|
|
15159
|
+
// decoded by the transport, so letting an invalid shape bypass this bucket
|
|
15160
|
+
// would leave an authenticated peer with an unmetered validation/logging
|
|
15161
|
+
// path. Clamp only the accounting cost; shape validation still rejects the
|
|
15162
|
+
// request below.
|
|
15163
|
+
const hashCost = Math.max(1, Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount));
|
|
15164
|
+
let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
|
|
15165
|
+
if (!nodeBudget) {
|
|
15166
|
+
nodeBudget = {
|
|
15167
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
15168
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
15169
|
+
refilledAt: now,
|
|
15170
|
+
peerSessions: new Map(),
|
|
15171
|
+
};
|
|
15172
|
+
persistedReceiptIngressBudgets.set(this.node, nodeBudget);
|
|
15173
|
+
}
|
|
15174
|
+
refillPersistedReceiptIngressBucket(nodeBudget, now, PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND);
|
|
15175
|
+
const peerSessionKey = `${peer}\0${transportSession}`;
|
|
15176
|
+
let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
|
|
15177
|
+
if (!peerBudget) {
|
|
15178
|
+
while (nodeBudget.peerSessions.size >=
|
|
15179
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS) {
|
|
15180
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
15181
|
+
if (oldest === undefined)
|
|
15182
|
+
break;
|
|
15183
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
15184
|
+
}
|
|
15185
|
+
peerBudget = {
|
|
15186
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
15187
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
15188
|
+
refilledAt: now,
|
|
15189
|
+
};
|
|
15190
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
15191
|
+
}
|
|
15192
|
+
else {
|
|
15193
|
+
// Refresh insertion order so bounded eviction prefers inactive sessions.
|
|
15194
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
15195
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
15196
|
+
}
|
|
15197
|
+
refillPersistedReceiptIngressBucket(peerBudget, now, PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND);
|
|
15198
|
+
if (nodeBudget.requestTokens < 1 ||
|
|
15199
|
+
nodeBudget.hashTokens < hashCost ||
|
|
15200
|
+
peerBudget.requestTokens < 1 ||
|
|
15201
|
+
peerBudget.hashTokens < hashCost) {
|
|
15202
|
+
return false;
|
|
15203
|
+
}
|
|
15204
|
+
nodeBudget.requestTokens -= 1;
|
|
15205
|
+
nodeBudget.hashTokens -= hashCost;
|
|
15206
|
+
peerBudget.requestTokens -= 1;
|
|
15207
|
+
peerBudget.hashTokens -= hashCost;
|
|
15208
|
+
return true;
|
|
15209
|
+
}
|
|
15210
|
+
async handleRequestPersistedEntriesV1(request, context, lane) {
|
|
15211
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15212
|
+
return undefined;
|
|
15213
|
+
}
|
|
15214
|
+
if (!this.admitPersistedReceiptIngress(lane.fromHash, context.message.header.session, request.hashes.length)) {
|
|
15215
|
+
return undefined;
|
|
15216
|
+
}
|
|
15217
|
+
// Charge the request before shape validation and attacker-controlled CID
|
|
15218
|
+
// parsing. Invalid shapes are rejected quietly here so they cannot turn the
|
|
15219
|
+
// outer receive error logger into a post-budget work amplifier.
|
|
15220
|
+
try {
|
|
15221
|
+
this.validatePersistedReceiptRequestShape(request);
|
|
15222
|
+
}
|
|
15223
|
+
catch {
|
|
15224
|
+
return undefined;
|
|
15225
|
+
}
|
|
15226
|
+
if (!this.hasValidPersistedReceiptHashes(request)) {
|
|
15227
|
+
return undefined;
|
|
15228
|
+
}
|
|
15229
|
+
const peerInFlight = this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
|
|
15230
|
+
if (peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
|
|
15231
|
+
this._persistedReceiptRequestsInFlightTotal >=
|
|
15232
|
+
MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL) {
|
|
15233
|
+
return undefined;
|
|
15234
|
+
}
|
|
15235
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
|
|
15236
|
+
this._persistedReceiptRequestsInFlightTotal++;
|
|
15237
|
+
try {
|
|
15238
|
+
// A positive receipt requires the block itself. Reject entirely absent
|
|
15239
|
+
// batches before the serialized mutation lane and, critically, before a
|
|
15240
|
+
// request can force unrelated pending index/coordinate journals durable.
|
|
15241
|
+
// A concurrent put can yield a harmless false negative: the sender retries.
|
|
15242
|
+
const blockPresence = await this.remoteBlocks.localStore.hasMany(request.hashes);
|
|
15243
|
+
const presentBlockHashes = request.hashes.filter((_hash, index) => blockPresence[index] === true);
|
|
15244
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15245
|
+
return undefined;
|
|
15246
|
+
}
|
|
15247
|
+
if (presentBlockHashes.length === 0) {
|
|
15248
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15249
|
+
}
|
|
15250
|
+
this._liveness.markReplicatorActivity(lane.fromHash);
|
|
15251
|
+
return await this.withReplicationRangeMutationQueue(async () => {
|
|
15252
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15253
|
+
return undefined;
|
|
15254
|
+
}
|
|
15255
|
+
this.throwIfNativeDurableCommitFailed();
|
|
15256
|
+
const storage = this.resolvePersistedReceiptStorage();
|
|
15257
|
+
if (!storage) {
|
|
15258
|
+
return undefined;
|
|
15259
|
+
}
|
|
15260
|
+
await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
|
|
15261
|
+
await this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
15262
|
+
const candidates = new Map(await Promise.all(presentBlockHashes.map(async (hash) => {
|
|
15263
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
15264
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15265
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15266
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15267
|
+
]);
|
|
15268
|
+
return [hash, { blockPresent, lowerRow, coordinate }];
|
|
15269
|
+
})));
|
|
15270
|
+
const presentHashes = presentBlockHashes.filter((hash) => {
|
|
15271
|
+
const candidate = candidates.get(hash);
|
|
15272
|
+
return (candidate.blockPresent &&
|
|
15273
|
+
candidate.lowerRow != null &&
|
|
15274
|
+
candidate.coordinate != null);
|
|
15275
|
+
});
|
|
15276
|
+
if (presentHashes.length === 0) {
|
|
15277
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15278
|
+
}
|
|
15279
|
+
await Promise.all([...new Set([storage.block, storage.lower, storage.coordinate])].map((store) => store.barrier()));
|
|
15280
|
+
this.throwIfNativeDurableCommitFailed();
|
|
15281
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15282
|
+
return undefined;
|
|
15283
|
+
}
|
|
15284
|
+
const ownershipRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
15285
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
15286
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15287
|
+
}
|
|
15288
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
15289
|
+
const confirmed = [];
|
|
15290
|
+
for (const hash of presentHashes) {
|
|
15291
|
+
const candidate = candidates.get(hash);
|
|
15292
|
+
if (!candidate.blockPresent ||
|
|
15293
|
+
!candidate.lowerRow ||
|
|
15294
|
+
!candidate.coordinate ||
|
|
15295
|
+
this._checkedPrune.hasActiveWork(hash) ||
|
|
15296
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane) ||
|
|
15297
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
15298
|
+
continue;
|
|
15299
|
+
}
|
|
15300
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
15301
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15302
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15303
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15304
|
+
]);
|
|
15305
|
+
if (!blockPresent || !lowerRow || !coordinate) {
|
|
15306
|
+
continue;
|
|
15307
|
+
}
|
|
15308
|
+
const replicas = decodeReplicas(coordinate).getValue(this);
|
|
15309
|
+
const leaders = await this.findLeadersFromEntry(coordinate, replicas, { freshLeaderPlan: true }, lane.ownershipLifecycleController);
|
|
15310
|
+
if (!leaders.has(selfHash)) {
|
|
15311
|
+
continue;
|
|
15312
|
+
}
|
|
15313
|
+
if (!this._checkedPrune.hasActiveWork(hash) &&
|
|
15314
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
|
|
15315
|
+
this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15316
|
+
confirmed.push(hash);
|
|
15317
|
+
}
|
|
15318
|
+
}
|
|
15319
|
+
if (confirmed.length === 0 ||
|
|
15320
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
15321
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15322
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15323
|
+
}
|
|
15324
|
+
const finalRows = await Promise.all(confirmed.map(async (hash) => {
|
|
15325
|
+
const [block, lower, coordinate] = await Promise.all([
|
|
15326
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15327
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15328
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15329
|
+
]);
|
|
15330
|
+
return { hash, block, lower, coordinate };
|
|
15331
|
+
}));
|
|
15332
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
15333
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15334
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15335
|
+
}
|
|
15336
|
+
return new ConfirmEntriesMessage({
|
|
15337
|
+
hashes: finalRows
|
|
15338
|
+
.filter((row) => row.block &&
|
|
15339
|
+
row.lower &&
|
|
15340
|
+
row.coordinate &&
|
|
15341
|
+
!this._checkedPrune.hasActiveWork(row.hash))
|
|
15342
|
+
.map((row) => row.hash),
|
|
15343
|
+
});
|
|
15344
|
+
}, lane.ownershipLifecycleController);
|
|
15345
|
+
}
|
|
15346
|
+
finally {
|
|
15347
|
+
const remaining = (this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
|
|
15348
|
+
if (remaining <= 0) {
|
|
15349
|
+
this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
|
|
15350
|
+
}
|
|
15351
|
+
else {
|
|
15352
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
|
|
15353
|
+
}
|
|
15354
|
+
this._persistedReceiptRequestsInFlightTotal = Math.max(0, this._persistedReceiptRequestsInFlightTotal - 1);
|
|
15355
|
+
}
|
|
15356
|
+
}
|
|
14183
15357
|
async handleRequestIPruneV2(msg, context, lane) {
|
|
14184
15358
|
const syncProfile = lane.syncProfile;
|
|
14185
15359
|
const receiveReplicationLifecycleController = lane.lifecycleController;
|