@peerbit/shared-log 16.0.13 → 16.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/src/coordinate-persistence.d.ts +10 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +18 -0
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/errors.d.ts +11 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +23 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +6 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +6 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +60 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1325 -144
- package/dist/src/index.js.map +1 -1
- package/dist/src/native-write-through-block-store.d.ts +4 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -1
- package/dist/src/native-write-through-block-store.js +13 -0
- package/dist/src/native-write-through-block-store.js.map +1 -1
- package/dist/src/sync/simple.d.ts +15 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +44 -0
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +18 -18
- package/src/coordinate-persistence.ts +21 -0
- package/src/errors.ts +35 -0
- package/src/exchange-heads.ts +7 -0
- package/src/index.ts +2002 -264
- package/src/native-write-through-block-store.ts +14 -0
- package/src/sync/simple.ts +22 -0
package/dist/src/index.js
CHANGED
|
@@ -45,7 +45,7 @@ import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
|
|
|
45
45
|
import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
|
|
46
46
|
import { RPC } from "@peerbit/rpc";
|
|
47
47
|
import { AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
|
|
48
|
-
import { AbortError, TimeoutError, debounceFixedInterval, waitFor, } from "@peerbit/time";
|
|
48
|
+
import { AbortError, TimeoutError, debounceFixedInterval, delay, waitFor, } from "@peerbit/time";
|
|
49
49
|
import pDefer, {} from "p-defer";
|
|
50
50
|
import PQueue from "p-queue";
|
|
51
51
|
import { concat, fromString } from "uint8arrays";
|
|
@@ -54,8 +54,8 @@ import { CheckedPruneCoordinator, } from "./checked-prune.js";
|
|
|
54
54
|
import { CoordinatePersistenceCoordinator, combineCoordinateDeleteHashes, isPromiseLike, mapMaybePromise, normalizedHashValues, } from "./coordinate-persistence.js";
|
|
55
55
|
import { CPUUsageIntervalLag } from "./cpu.js";
|
|
56
56
|
import { debouncedAccumulatorMap, } from "./debounce.js";
|
|
57
|
-
import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, isNotStartedError, } from "./errors.js";
|
|
58
|
-
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
57
|
+
import { CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, isNotStartedError, } from "./errors.js";
|
|
58
|
+
import { EXCHANGE_HEADS_REPAIR_HINT, EntryWithRefs, ExchangeHeadsMessage, MAX_RAW_EXCHANGE_MESSAGE_SIZE, RawEntryWithRefs, RawExchangeHeadsMessage, RequestIPrune, RequestIPruneV2, ResponseIPrune, ResponseIPruneV2, SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS, SYNC_CAPABILITY_RAW_EXCHANGE_HEADS, SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY, SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM, SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE, SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND, StashBackedRawExchangeHeadsMessage, SyncCapabilitiesMessage, collectRawExchangeHeadSendPlan, createExchangeHeadsMessages, createRawExchangeHeadsMessages, getExchangeHeadHash, getPreparedRawExchangeGid, getPreparedRawExchangeHashNumber, getPreparedRawExchangeHeadAppendFacts, getPreparedRawExchangeHeadGid, getPreparedRawExchangeHeadRequestedReplicas, getPreparedRawExchangeHeadShallowEntry, getPreparedRawExchangeHeadSignatureVerified, getPreparedRawExchangeNext, getPreparedRawExchangeRequestedReplicas, getPreparedRawExchangeTimestamp, getRawExchangeHeadByteLength, getRawExchangeHeadStashIndexes, initExchangeHeadEntry, isPreparedRawEntryWithRefs, isStashBackedRawExchangeHeadsMessage, materializeVerifiedRawExchangeHeadsMessage, } from "./exchange-heads.js";
|
|
59
59
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
60
60
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
61
61
|
import { MAX_U32, MAX_U64, createNumbers, } from "./integers.js";
|
|
@@ -78,7 +78,7 @@ import { AbsoluteReplicas, AddedReplicationInfoV2Message, AddedReplicationSegmen
|
|
|
78
78
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
79
79
|
import { createSyncronizer } from "./sync/factory.js";
|
|
80
80
|
import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./sync/profile.js";
|
|
81
|
-
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
81
|
+
import { ConfirmEntriesMessage, RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS, RequestPersistedEntriesV1, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
|
|
82
82
|
import { groupByGid, tryGroupByGidSync } from "./utils.js";
|
|
83
83
|
const getSharedLogFanoutService = (services) => services.fanout;
|
|
84
84
|
const createOneShotPeerReceiveLease = (releaseFn) => {
|
|
@@ -127,7 +127,7 @@ export { createReplicationDomainHash, createReplicationDomainTime, };
|
|
|
127
127
|
export { CPUUsageIntervalLag };
|
|
128
128
|
export * from "./replication.js";
|
|
129
129
|
export * from "./replication-status.js";
|
|
130
|
-
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, };
|
|
130
|
+
export { ReplicationRangeIndexableU32, ReplicationRangeIndexableU64, EntryReplicatedU32, EntryReplicatedU64, CompatibilityModeRetiredError, NativeDurableCommitError, NoPeersError, PersistedDeliveryError, };
|
|
131
131
|
export { MAX_U32, MAX_U64 };
|
|
132
132
|
export { ExchangeHeadsMessage, RawExchangeHeadsMessage, StashBackedRawExchangeHeadsMessage, };
|
|
133
133
|
export const logger = loggerFn("peerbit:shared-log");
|
|
@@ -151,11 +151,18 @@ const canUseOptionalNativeModuleImports = () => {
|
|
|
151
151
|
* to avoid doubled slashes.
|
|
152
152
|
*/
|
|
153
153
|
const joinNativeCoordinateDirectory = (nodeDirectory, fsSafeLogId) => `${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
|
|
154
|
+
const defaultNativeEntryBlockCompactMaxJournalBytes = 64 * 1024 * 1024;
|
|
154
155
|
const createNativeDurableBlockStore = async (storage) => new AnyBlockStore(await storage.sublevel("blocks", {
|
|
155
|
-
// Strict mirrors remain WAL-backed
|
|
156
|
-
//
|
|
157
|
-
//
|
|
156
|
+
// Strict mirrors remain WAL-backed. On POSIX Node, checkpoint only the
|
|
157
|
+
// historical suffix through the Rust store's fsync + atomic-rename path;
|
|
158
|
+
// browsers/custom backends and Windows retain the prior unbounded WAL until
|
|
159
|
+
// they expose an equally strong directory durability barrier.
|
|
158
160
|
compactOnClose: false,
|
|
161
|
+
...(storage.supportsCrashSafeJournalCheckpoint === true
|
|
162
|
+
? {
|
|
163
|
+
compactMaxJournalBytes: defaultNativeEntryBlockCompactMaxJournalBytes,
|
|
164
|
+
}
|
|
165
|
+
: {}),
|
|
159
166
|
// A native append is acknowledged only after this mirror resolves. The
|
|
160
167
|
// Rust store's normal immutable fast path may resolve before its WAL write;
|
|
161
168
|
// strict mode waits for the journal write and sync, closing the SIGKILL gap.
|
|
@@ -558,6 +565,63 @@ const checkMinReplicasLimit = (minReplicas) => {
|
|
|
558
565
|
throw new Error("Higher replication degree than 100 is not recommended for performance reasons");
|
|
559
566
|
}
|
|
560
567
|
};
|
|
568
|
+
const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
|
|
569
|
+
const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
|
|
570
|
+
const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
|
|
571
|
+
const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
|
|
572
|
+
const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
|
|
573
|
+
const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
|
|
574
|
+
const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
575
|
+
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
576
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
577
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
578
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
579
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
580
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
581
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
|
|
582
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
|
|
583
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
|
|
584
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
|
|
585
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
|
|
586
|
+
const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
|
|
587
|
+
const persistedReceiptIngressBudgets = new WeakMap();
|
|
588
|
+
// Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
|
|
589
|
+
// on the Peerbit node (rather than one SharedLog) prevents independent programs
|
|
590
|
+
// from silently overrunning the same remote receiver together.
|
|
591
|
+
const persistedReceiptEgressBudgets = new WeakMap();
|
|
592
|
+
const refillPersistedReceiptIngressBucket = (bucket, now, requestCapacity, hashCapacity, requestsPerSecond, hashesPerSecond) => {
|
|
593
|
+
// Never move the refill watermark backwards. Date.now() can jump after a
|
|
594
|
+
// clock correction; accepting that earlier timestamp would grant the same
|
|
595
|
+
// elapsed interval again on the next request.
|
|
596
|
+
if (now <= bucket.refilledAt) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
|
|
600
|
+
if (elapsedSeconds > 0) {
|
|
601
|
+
bucket.requestTokens = Math.min(requestCapacity, bucket.requestTokens + elapsedSeconds * requestsPerSecond);
|
|
602
|
+
bucket.hashTokens = Math.min(hashCapacity, bucket.hashTokens + elapsedSeconds * hashesPerSecond);
|
|
603
|
+
}
|
|
604
|
+
bucket.refilledAt = now;
|
|
605
|
+
};
|
|
606
|
+
const persistedReceiptPacingFloorMs = (hashCount) => {
|
|
607
|
+
const boundedHashCount = Math.max(0, Math.floor(hashCount));
|
|
608
|
+
const requestCount = Math.ceil(boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE);
|
|
609
|
+
return Math.max(0, ((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
|
|
610
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
611
|
+
1_000, ((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
|
|
612
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
613
|
+
1_000);
|
|
614
|
+
};
|
|
615
|
+
const persistedTransferAdmissionBudgetMs = (hashCount) => Math.ceil(Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
616
|
+
const attachTrustedLocalCommitEvidence = (options, evidence) => {
|
|
617
|
+
if (!evidence)
|
|
618
|
+
return;
|
|
619
|
+
options.__peerbitOnLocalCommit = (hashes) => {
|
|
620
|
+
for (const hash of hashes) {
|
|
621
|
+
evidence.committedHashes.add(hash);
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
};
|
|
561
625
|
const asTrustedLowerLog = (log) => log;
|
|
562
626
|
class ReplicationStatusSnapshotChangedError extends Error {
|
|
563
627
|
constructor() {
|
|
@@ -1403,6 +1467,34 @@ let SharedLog = (() => {
|
|
|
1403
1467
|
handle.release();
|
|
1404
1468
|
}
|
|
1405
1469
|
}
|
|
1470
|
+
async finishCommittedNativeStrictDurableTransaction(handle, finish, shouldWarnOnRetirementFailure = () => true) {
|
|
1471
|
+
let finishResult;
|
|
1472
|
+
let finishError;
|
|
1473
|
+
let finishFailed = false;
|
|
1474
|
+
try {
|
|
1475
|
+
finishResult = await finish();
|
|
1476
|
+
}
|
|
1477
|
+
catch (error) {
|
|
1478
|
+
finishError = error;
|
|
1479
|
+
finishFailed = true;
|
|
1480
|
+
}
|
|
1481
|
+
try {
|
|
1482
|
+
await this.completeNativeStrictDurableTransaction(handle);
|
|
1483
|
+
}
|
|
1484
|
+
catch (error) {
|
|
1485
|
+
// The acknowledged lower commit is already final. Completion poisons and
|
|
1486
|
+
// releases the in-memory transaction before throwing, so preserve an
|
|
1487
|
+
// earlier post-commit failure while leaving recovery to retire the intent.
|
|
1488
|
+
if (!finishFailed && !shouldWarnOnRetirementFailure()) {
|
|
1489
|
+
throw error;
|
|
1490
|
+
}
|
|
1491
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
1492
|
+
}
|
|
1493
|
+
if (finishFailed) {
|
|
1494
|
+
throw finishError;
|
|
1495
|
+
}
|
|
1496
|
+
return finishResult;
|
|
1497
|
+
}
|
|
1406
1498
|
releaseNativeStrictDurableTransaction(handle, cause = new Error("Native durable transaction intent was retained for recovery")) {
|
|
1407
1499
|
if (!handle || handle.released) {
|
|
1408
1500
|
return;
|
|
@@ -1755,6 +1847,9 @@ let SharedLog = (() => {
|
|
|
1755
1847
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
1756
1848
|
_peerSyncCapabilitySessions;
|
|
1757
1849
|
_peerSyncCapabilityTimestamps;
|
|
1850
|
+
_persistedReceiptStorage;
|
|
1851
|
+
_persistedReceiptRequestsInFlight;
|
|
1852
|
+
_persistedReceiptRequestsInFlightTotal;
|
|
1758
1853
|
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
1759
1854
|
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
1760
1855
|
// is hit). Only used when every recipient advertised raw capability.
|
|
@@ -1812,7 +1907,7 @@ let SharedLog = (() => {
|
|
|
1812
1907
|
return new ReplicationInfoV2SendCoordinator({
|
|
1813
1908
|
getRpc: () => this.rpc,
|
|
1814
1909
|
getSelfKey: () => this.node.identity.publicKey,
|
|
1815
|
-
getSenderTransportSession: () =>
|
|
1910
|
+
getSenderTransportSession: () => this.ownTransportSession(),
|
|
1816
1911
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
1817
1912
|
validatePersistedReplicationRangeSnapshot: (ranges) => this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
1818
1913
|
isClosed: () => this.closed,
|
|
@@ -1830,18 +1925,37 @@ let SharedLog = (() => {
|
|
|
1830
1925
|
0,
|
|
1831
1926
|
});
|
|
1832
1927
|
}
|
|
1928
|
+
resolvePersistedReceiptStorage() {
|
|
1929
|
+
if (this.log.appendDurability !== "strict") {
|
|
1930
|
+
return undefined;
|
|
1931
|
+
}
|
|
1932
|
+
const block = this.remoteBlocks.crashSafeDurability;
|
|
1933
|
+
const lower = this.log.entryIndex.properties.index.crashSafeDurability;
|
|
1934
|
+
const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
|
|
1935
|
+
if (!block || !lower || !coordinate) {
|
|
1936
|
+
return undefined;
|
|
1937
|
+
}
|
|
1938
|
+
return { block, lower, coordinate };
|
|
1939
|
+
}
|
|
1940
|
+
ownTransportSession() {
|
|
1941
|
+
return BigInt(this.node.services.pubsub
|
|
1942
|
+
.session);
|
|
1943
|
+
}
|
|
1833
1944
|
replicationInfoV2ReceiveCapabilities() {
|
|
1834
1945
|
return (SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
1835
1946
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
1836
1947
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
1837
1948
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
|
|
1949
|
+
(this._persistedReceiptStorage
|
|
1950
|
+
? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
|
|
1951
|
+
: 0) |
|
|
1838
1952
|
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
1839
1953
|
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
1840
1954
|
: 0));
|
|
1841
1955
|
}
|
|
1842
1956
|
async advertiseReplicationInfoV2ReceiveCapability(properties) {
|
|
1843
1957
|
const peerHash = properties.target.hashcode();
|
|
1844
|
-
const receiverTransportSession =
|
|
1958
|
+
const receiverTransportSession = this.ownTransportSession();
|
|
1845
1959
|
await this.rpc.send(new SyncCapabilitiesMessage({
|
|
1846
1960
|
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
1847
1961
|
}), {
|
|
@@ -1860,7 +1974,7 @@ let SharedLog = (() => {
|
|
|
1860
1974
|
!this._peerSessions.isReceiveEpochCurrent(peerHash, properties.receiveEpoch) ||
|
|
1861
1975
|
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
1862
1976
|
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
1863
|
-
|
|
1977
|
+
this.ownTransportSession() !== receiverTransportSession) {
|
|
1864
1978
|
return undefined;
|
|
1865
1979
|
}
|
|
1866
1980
|
return {
|
|
@@ -1871,7 +1985,7 @@ let SharedLog = (() => {
|
|
|
1871
1985
|
createReplicationInfoV2ReceiveCoordinator() {
|
|
1872
1986
|
return new ReplicationInfoV2ReceiveCoordinator({
|
|
1873
1987
|
getSelfKey: () => this.node.identity.publicKey,
|
|
1874
|
-
getReceiverTransportSession: () =>
|
|
1988
|
+
getReceiverTransportSession: () => this.ownTransportSession(),
|
|
1875
1989
|
isClosed: () => this.closed,
|
|
1876
1990
|
isPeerSessionCurrent: (peerHash, peerSession) => this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
1877
1991
|
peerSession.phase === "open" &&
|
|
@@ -2007,6 +2121,9 @@ let SharedLog = (() => {
|
|
|
2007
2121
|
this._peerSyncCapabilities = new Map();
|
|
2008
2122
|
this._peerSyncCapabilitySessions = new Map();
|
|
2009
2123
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
2124
|
+
this._persistedReceiptStorage = undefined;
|
|
2125
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
2126
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
2010
2127
|
this._liveRawGossipBatches = new Map();
|
|
2011
2128
|
this._liveRawGossipFlushScheduled = false;
|
|
2012
2129
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -2145,6 +2262,14 @@ let SharedLog = (() => {
|
|
|
2145
2262
|
}
|
|
2146
2263
|
this._fanoutChannel = undefined;
|
|
2147
2264
|
}
|
|
2265
|
+
ensureLogProviderHandle(fanoutService) {
|
|
2266
|
+
if (this._providerHandle || this._closeController.signal.aborted)
|
|
2267
|
+
return;
|
|
2268
|
+
this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
|
|
2269
|
+
ttlMs: 120_000,
|
|
2270
|
+
announceIntervalMs: 60_000,
|
|
2271
|
+
});
|
|
2272
|
+
}
|
|
2148
2273
|
async _onFanoutData(detail) {
|
|
2149
2274
|
let envelope;
|
|
2150
2275
|
try {
|
|
@@ -2243,10 +2368,17 @@ let SharedLog = (() => {
|
|
|
2243
2368
|
const reliability = delivery.reliability ?? "ack";
|
|
2244
2369
|
const deliveryTimeout = delivery.timeout;
|
|
2245
2370
|
const deliverySignal = delivery.signal;
|
|
2246
|
-
const requireRecipients = delivery.requireRecipients === true;
|
|
2371
|
+
const requireRecipients = reliability === "persisted" || delivery.requireRecipients === true;
|
|
2247
2372
|
const minAcks = delivery.minAcks != null && Number.isFinite(delivery.minAcks)
|
|
2248
2373
|
? Math.max(0, Math.floor(delivery.minAcks))
|
|
2249
2374
|
: undefined;
|
|
2375
|
+
if (reliability === "persisted") {
|
|
2376
|
+
if (delivery.minAcks == null ||
|
|
2377
|
+
!Number.isSafeInteger(delivery.minAcks) ||
|
|
2378
|
+
delivery.minAcks <= 0) {
|
|
2379
|
+
throw new Error('persisted delivery requires a positive explicit "minAcks"');
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2250
2382
|
const wrap = deliveryTimeout == null && deliverySignal == null
|
|
2251
2383
|
? undefined
|
|
2252
2384
|
: (promise) => new Promise((resolve, reject) => {
|
|
@@ -2312,6 +2444,62 @@ let SharedLog = (() => {
|
|
|
2312
2444
|
wrap,
|
|
2313
2445
|
};
|
|
2314
2446
|
}
|
|
2447
|
+
validatePersistedReceiptRequestShape(request) {
|
|
2448
|
+
if (request.hashes.length === 0 ||
|
|
2449
|
+
request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES) {
|
|
2450
|
+
throw new Error(`Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`);
|
|
2451
|
+
}
|
|
2452
|
+
let bytes = 0;
|
|
2453
|
+
const encoder = new TextEncoder();
|
|
2454
|
+
for (const hash of request.hashes) {
|
|
2455
|
+
if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
2456
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
2457
|
+
}
|
|
2458
|
+
bytes += encoder.encode(hash).byteLength;
|
|
2459
|
+
if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
2460
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
hasValidPersistedReceiptHashes(request) {
|
|
2465
|
+
const seen = new Set();
|
|
2466
|
+
try {
|
|
2467
|
+
for (const hash of request.hashes) {
|
|
2468
|
+
if (seen.has(hash))
|
|
2469
|
+
return false;
|
|
2470
|
+
cidifyString(hash);
|
|
2471
|
+
seen.add(hash);
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
catch {
|
|
2475
|
+
return false;
|
|
2476
|
+
}
|
|
2477
|
+
return true;
|
|
2478
|
+
}
|
|
2479
|
+
getPersistedDeliveryOptions(options) {
|
|
2480
|
+
const delivery = options?.delivery;
|
|
2481
|
+
if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
|
|
2482
|
+
return undefined;
|
|
2483
|
+
}
|
|
2484
|
+
const parsed = this._parseDeliveryOptions(options?.delivery);
|
|
2485
|
+
const target = options?.target;
|
|
2486
|
+
if (target !== undefined && target !== "replicators") {
|
|
2487
|
+
throw new Error('persisted delivery requires target="replicators" (or an omitted target)');
|
|
2488
|
+
}
|
|
2489
|
+
if (parsed.delivery?.timeout != null &&
|
|
2490
|
+
(!Number.isFinite(parsed.delivery.timeout) ||
|
|
2491
|
+
parsed.delivery.timeout <= 0 ||
|
|
2492
|
+
parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)) {
|
|
2493
|
+
throw new Error(`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`);
|
|
2494
|
+
}
|
|
2495
|
+
if (parsed.delivery?.signal?.aborted) {
|
|
2496
|
+
throw parsed.delivery.signal.reason ?? new AbortError();
|
|
2497
|
+
}
|
|
2498
|
+
return parsed.delivery;
|
|
2499
|
+
}
|
|
2500
|
+
assertPersistedDeliveryOptions(options) {
|
|
2501
|
+
this.getPersistedDeliveryOptions(options);
|
|
2502
|
+
}
|
|
2315
2503
|
async _getSortedRouteHints(targetHash) {
|
|
2316
2504
|
const pubsub = this.node.services.pubsub;
|
|
2317
2505
|
const maybeHints = await pubsub?.getUnifiedRouteHints?.(this.topic, targetHash);
|
|
@@ -2700,7 +2888,9 @@ let SharedLog = (() => {
|
|
|
2700
2888
|
}
|
|
2701
2889
|
attemptedMessages += 1;
|
|
2702
2890
|
await pubsub.publishPreEncodedData(item.payload, { topics: [topic] }, {
|
|
2703
|
-
mode:
|
|
2891
|
+
mode: options?.acknowledge
|
|
2892
|
+
? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
|
|
2893
|
+
: new SilentDelivery({ redundancy: 1, to: [...to] }),
|
|
2704
2894
|
priority: options?.priority,
|
|
2705
2895
|
signal: options?.signal,
|
|
2706
2896
|
});
|
|
@@ -2747,6 +2937,537 @@ let SharedLog = (() => {
|
|
|
2747
2937
|
}
|
|
2748
2938
|
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
2749
2939
|
}
|
|
2940
|
+
persistedReceiptPeerSession(peerHash) {
|
|
2941
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
2942
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
2943
|
+
if (capabilitySession == null ||
|
|
2944
|
+
!peerSession ||
|
|
2945
|
+
peerSession.phase !== "open" ||
|
|
2946
|
+
!this._peerSessions.isCurrent(peerHash, peerSession) ||
|
|
2947
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
2948
|
+
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
2949
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
|
|
2950
|
+
0) {
|
|
2951
|
+
return undefined;
|
|
2952
|
+
}
|
|
2953
|
+
return { capabilitySession, peerSession };
|
|
2954
|
+
}
|
|
2955
|
+
async waitPersistedReceiptRetry(signal, ms) {
|
|
2956
|
+
try {
|
|
2957
|
+
await delay(ms, { signal });
|
|
2958
|
+
}
|
|
2959
|
+
catch (error) {
|
|
2960
|
+
throw signal.aborted ? (signal.reason ?? error) : error;
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
reservePersistedReceiptEgress(peer, capabilitySession, hashCount, now = Date.now()) {
|
|
2964
|
+
let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
|
|
2965
|
+
if (!nodeBudget) {
|
|
2966
|
+
nodeBudget = { peerSessions: new Map() };
|
|
2967
|
+
persistedReceiptEgressBudgets.set(this.node, nodeBudget);
|
|
2968
|
+
}
|
|
2969
|
+
const peerSessionKey = `${peer}\0${capabilitySession}`;
|
|
2970
|
+
let bucket = nodeBudget.peerSessions.get(peerSessionKey);
|
|
2971
|
+
if (!bucket) {
|
|
2972
|
+
while (nodeBudget.peerSessions.size >=
|
|
2973
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS) {
|
|
2974
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
2975
|
+
if (oldest === undefined)
|
|
2976
|
+
break;
|
|
2977
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
2978
|
+
}
|
|
2979
|
+
bucket = {
|
|
2980
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
2981
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
2982
|
+
refilledAt: now,
|
|
2983
|
+
};
|
|
2984
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
2985
|
+
}
|
|
2986
|
+
else {
|
|
2987
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
2988
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
2989
|
+
}
|
|
2990
|
+
refillPersistedReceiptIngressBucket(bucket, now, PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND);
|
|
2991
|
+
if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
|
|
2992
|
+
bucket.requestTokens -= 1;
|
|
2993
|
+
bucket.hashTokens -= hashCount;
|
|
2994
|
+
return 0;
|
|
2995
|
+
}
|
|
2996
|
+
return Math.max(1, Math.ceil(Math.max(((1 - bucket.requestTokens) /
|
|
2997
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
2998
|
+
1_000, ((hashCount - bucket.hashTokens) /
|
|
2999
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
3000
|
+
1_000)));
|
|
3001
|
+
}
|
|
3002
|
+
async waitForPersistedReceiptEgressAdmission(peer, capabilitySession, hashCount, signal) {
|
|
3003
|
+
while (true) {
|
|
3004
|
+
if (signal.aborted) {
|
|
3005
|
+
throw signal.reason ?? new AbortError();
|
|
3006
|
+
}
|
|
3007
|
+
const waitMs = this.reservePersistedReceiptEgress(peer, capabilitySession, hashCount);
|
|
3008
|
+
if (waitMs === 0)
|
|
3009
|
+
return;
|
|
3010
|
+
await this.waitPersistedReceiptRetry(signal, waitMs);
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
persistedDeliveryTimeoutMs(delivery, hashCount) {
|
|
3014
|
+
return (delivery.timeout ??
|
|
3015
|
+
Math.min(MAX_PERSISTED_DELIVERY_TIMEOUT_MS, DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
|
|
3016
|
+
persistedTransferAdmissionBudgetMs(hashCount) +
|
|
3017
|
+
Math.ceil(persistedReceiptPacingFloorMs(hashCount))));
|
|
3018
|
+
}
|
|
3019
|
+
async waitForPersistedTransferAdmission(peer, hashes, captured, signal, isStillCurrent) {
|
|
3020
|
+
const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
3021
|
+
while (!signal.aborted && isStillCurrent()) {
|
|
3022
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3023
|
+
if (!current ||
|
|
3024
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3025
|
+
current.peerSession !== captured.peerSession) {
|
|
3026
|
+
return false;
|
|
3027
|
+
}
|
|
3028
|
+
if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
|
|
3029
|
+
return true;
|
|
3030
|
+
}
|
|
3031
|
+
const remaining = expiresAt - Date.now();
|
|
3032
|
+
if (remaining <= 0)
|
|
3033
|
+
return false;
|
|
3034
|
+
await this.waitPersistedReceiptRetry(signal, Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining));
|
|
3035
|
+
}
|
|
3036
|
+
if (signal.aborted)
|
|
3037
|
+
throw signal.reason ?? new AbortError();
|
|
3038
|
+
return false;
|
|
3039
|
+
}
|
|
3040
|
+
createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, hashCount = 1) {
|
|
3041
|
+
const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
|
|
3042
|
+
if (!Number.isFinite(timeoutMs) ||
|
|
3043
|
+
timeoutMs <= 0 ||
|
|
3044
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS) {
|
|
3045
|
+
throw new Error(`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`);
|
|
3046
|
+
}
|
|
3047
|
+
const minAcks = Math.floor(delivery.minAcks);
|
|
3048
|
+
const deadlineController = new AbortController();
|
|
3049
|
+
const timeout = setTimeout(() => deadlineController.abort(new TimeoutError(`Timed out waiting for ${minAcks} persisted remote replicas.`)), timeoutMs);
|
|
3050
|
+
timeout.unref?.();
|
|
3051
|
+
return {
|
|
3052
|
+
deadline: Date.now() + timeoutMs,
|
|
3053
|
+
signal: AbortSignal.any([
|
|
3054
|
+
delivery.signal,
|
|
3055
|
+
this._closeController.signal,
|
|
3056
|
+
ownershipLifecycleController.signal,
|
|
3057
|
+
deadlineController.signal,
|
|
3058
|
+
].filter((value) => !!value)),
|
|
3059
|
+
dispose: () => clearTimeout(timeout),
|
|
3060
|
+
};
|
|
3061
|
+
}
|
|
3062
|
+
async planPersistedDeliveryLeaders(entries, replicas, ownershipLifecycleController) {
|
|
3063
|
+
if (this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry) {
|
|
3064
|
+
const leaders = [];
|
|
3065
|
+
for (const entry of entries) {
|
|
3066
|
+
leaders.push(await this.findLeadersFromEntry(entry, replicas, { freshLeaderPlan: true }, ownershipLifecycleController));
|
|
3067
|
+
}
|
|
3068
|
+
return leaders;
|
|
3069
|
+
}
|
|
3070
|
+
const items = entries.map((entry) => ({
|
|
3071
|
+
entry,
|
|
3072
|
+
replicas,
|
|
3073
|
+
options: { freshLeaderPlan: true, persist: false },
|
|
3074
|
+
}));
|
|
3075
|
+
const nativeRoutingPlanner = this._nativeRangePlanner ?? this._nativeBackbone;
|
|
3076
|
+
if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
|
|
3077
|
+
const options = items[0].options;
|
|
3078
|
+
const context = await this.createLeaderSelectionContext(options, ownershipLifecycleController);
|
|
3079
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3080
|
+
const nativeOptions = this.createNativeLeaderOptions(context, options);
|
|
3081
|
+
const fullReplicaLeaders = nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(replicas, nativeOptions);
|
|
3082
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3083
|
+
if (fullReplicaLeaders) {
|
|
3084
|
+
// Receipt settlement treats leader maps as read-only. Reusing the one
|
|
3085
|
+
// gid-independent routing result avoids one Map allocation per entry.
|
|
3086
|
+
return new Array(items.length).fill(fullReplicaLeaders);
|
|
3087
|
+
}
|
|
3088
|
+
if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
|
|
3089
|
+
const nativeLeaders = nativeRoutingPlanner.planLeaderSamplesForGidsBatch(items.map((item) => ({
|
|
3090
|
+
gid: this.getEntryGid(item.entry),
|
|
3091
|
+
replicas: item.replicas,
|
|
3092
|
+
})), nativeOptions);
|
|
3093
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3094
|
+
if (nativeLeaders?.length === items.length) {
|
|
3095
|
+
return nativeLeaders;
|
|
3096
|
+
}
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
return (await this.planEntryLeaderBatch(items, ownershipLifecycleController)).map((plan) => plan.leaders);
|
|
3100
|
+
}
|
|
3101
|
+
async settlePersistedDelivery(input, replicas, delivery, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), persistedDeadline, transferOnFirstRound = false) {
|
|
3102
|
+
const minAcks = Math.floor(delivery.minAcks);
|
|
3103
|
+
const entries = new Map(input.map((entry) => [entry.hash, entry]));
|
|
3104
|
+
if (entries.size === 0)
|
|
3105
|
+
return;
|
|
3106
|
+
const committedHashes = [...entries.keys()];
|
|
3107
|
+
const ownedDeadline = !persistedDeadline;
|
|
3108
|
+
const deadline = persistedDeadline ??
|
|
3109
|
+
this.createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, entries.size);
|
|
3110
|
+
const signal = deadline.signal;
|
|
3111
|
+
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
3112
|
+
let initialTransferPending = transferOnFirstRound;
|
|
3113
|
+
let needsInitialLeaderCheck = true;
|
|
3114
|
+
const carriedAcknowledgements = new Map(committedHashes.map((hash) => [hash, new Map()]));
|
|
3115
|
+
const repairsByPeer = new Map();
|
|
3116
|
+
let acknowledgementOwnershipRevision;
|
|
3117
|
+
const purgePeerDeliveryState = (peer) => {
|
|
3118
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3119
|
+
acknowledgements.delete(peer);
|
|
3120
|
+
}
|
|
3121
|
+
repairsByPeer.delete(peer);
|
|
3122
|
+
};
|
|
3123
|
+
try {
|
|
3124
|
+
while (true) {
|
|
3125
|
+
if (signal.aborted) {
|
|
3126
|
+
throw signal.reason ?? new AbortError();
|
|
3127
|
+
}
|
|
3128
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3129
|
+
const ownershipRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
3130
|
+
const isRoundOwnershipCurrent = () => this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
3131
|
+
if (acknowledgementOwnershipRevision !== ownershipRevision) {
|
|
3132
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3133
|
+
acknowledgements.clear();
|
|
3134
|
+
}
|
|
3135
|
+
repairsByPeer.clear();
|
|
3136
|
+
acknowledgementOwnershipRevision = ownershipRevision;
|
|
3137
|
+
}
|
|
3138
|
+
else {
|
|
3139
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3140
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3141
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3142
|
+
if (!current ||
|
|
3143
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3144
|
+
current.peerSession !== captured.peerSession) {
|
|
3145
|
+
purgePeerDeliveryState(peer);
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
}
|
|
3150
|
+
if (!isRoundOwnershipCurrent()) {
|
|
3151
|
+
await this.waitPersistedReceiptRetry(signal, PERSISTED_RECEIPT_RETRY_MS);
|
|
3152
|
+
continue;
|
|
3153
|
+
}
|
|
3154
|
+
// Carry receipts only across retries in the exact same ownership and
|
|
3155
|
+
// transport epoch. A revision/session change purges them before they can
|
|
3156
|
+
// survive an away-and-back leader transition or combine with a later peer.
|
|
3157
|
+
const hashesByPeer = new Map();
|
|
3158
|
+
const entryArray = [...entries.values()];
|
|
3159
|
+
const leadersByEntry = await this.planPersistedDeliveryLeaders(entryArray, replicas, ownershipLifecycleController);
|
|
3160
|
+
if (!isRoundOwnershipCurrent())
|
|
3161
|
+
continue;
|
|
3162
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3163
|
+
if (needsInitialLeaderCheck) {
|
|
3164
|
+
needsInitialLeaderCheck = false;
|
|
3165
|
+
if (leadersByEntry.some((leaders) => leaders.size === 0 ||
|
|
3166
|
+
(leaders.size === 1 && leaders.has(selfHash)))) {
|
|
3167
|
+
throw new NoPeersError(this.rpc.topic);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
3171
|
+
const hash = entryArray[index].hash;
|
|
3172
|
+
const leaders = leadersByEntry[index];
|
|
3173
|
+
if (signal.aborted) {
|
|
3174
|
+
throw signal.reason ?? new AbortError();
|
|
3175
|
+
}
|
|
3176
|
+
if (!isRoundOwnershipCurrent())
|
|
3177
|
+
break;
|
|
3178
|
+
const acknowledgements = carriedAcknowledgements.get(hash);
|
|
3179
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3180
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3181
|
+
if (!leaders.has(peer) ||
|
|
3182
|
+
!current ||
|
|
3183
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3184
|
+
current.peerSession !== captured.peerSession) {
|
|
3185
|
+
acknowledgements.delete(peer);
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
if (acknowledgements.size >= minAcks)
|
|
3189
|
+
continue;
|
|
3190
|
+
for (const peer of leaders.keys()) {
|
|
3191
|
+
if (peer === selfHash)
|
|
3192
|
+
continue;
|
|
3193
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3194
|
+
if (!current)
|
|
3195
|
+
continue;
|
|
3196
|
+
if (acknowledgements.has(peer))
|
|
3197
|
+
continue;
|
|
3198
|
+
const hashes = hashesByPeer.get(peer) ?? [];
|
|
3199
|
+
hashes.push(hash);
|
|
3200
|
+
hashesByPeer.set(peer, hashes);
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3203
|
+
if (!isRoundOwnershipCurrent())
|
|
3204
|
+
continue;
|
|
3205
|
+
const operationQueue = new PQueue({
|
|
3206
|
+
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
3207
|
+
});
|
|
3208
|
+
const candidateWaves = Math.max(1, Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL));
|
|
3209
|
+
const roundController = new AbortController();
|
|
3210
|
+
const roundSignal = AbortSignal.any([signal, roundController.signal]);
|
|
3211
|
+
const getAttemptTimeout = () => Math.max(1, Math.min(maxAttemptMs, Math.floor((deadline.deadline - Date.now()) / candidateWaves)));
|
|
3212
|
+
const requests = new Set();
|
|
3213
|
+
const transferAllOnRound = initialTransferPending;
|
|
3214
|
+
try {
|
|
3215
|
+
for (const [peer, hashes] of hashesByPeer) {
|
|
3216
|
+
let request;
|
|
3217
|
+
request = (async () => {
|
|
3218
|
+
const captured = this.persistedReceiptPeerSession(peer);
|
|
3219
|
+
if (!captured)
|
|
3220
|
+
return;
|
|
3221
|
+
const previousRepair = repairsByPeer.get(peer);
|
|
3222
|
+
if (previousRepair &&
|
|
3223
|
+
(previousRepair.capabilitySession !==
|
|
3224
|
+
captured.capabilitySession ||
|
|
3225
|
+
previousRepair.peerSession !== captured.peerSession)) {
|
|
3226
|
+
repairsByPeer.delete(peer);
|
|
3227
|
+
}
|
|
3228
|
+
const ensureRepairState = () => {
|
|
3229
|
+
let state = repairsByPeer.get(peer);
|
|
3230
|
+
if (!state) {
|
|
3231
|
+
state = {
|
|
3232
|
+
capabilitySession: captured.capabilitySession,
|
|
3233
|
+
peerSession: captured.peerSession,
|
|
3234
|
+
hashes: new Set(),
|
|
3235
|
+
};
|
|
3236
|
+
repairsByPeer.set(peer, state);
|
|
3237
|
+
}
|
|
3238
|
+
return state;
|
|
3239
|
+
};
|
|
3240
|
+
const isPeerRoundCurrent = () => {
|
|
3241
|
+
if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
|
|
3242
|
+
return false;
|
|
3243
|
+
}
|
|
3244
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3245
|
+
return (!!current &&
|
|
3246
|
+
current.capabilitySession === captured.capabilitySession &&
|
|
3247
|
+
current.peerSession === captured.peerSession);
|
|
3248
|
+
};
|
|
3249
|
+
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
3250
|
+
const transferHashes = transferAllOnRound
|
|
3251
|
+
? hashes
|
|
3252
|
+
: repairs
|
|
3253
|
+
? hashes.filter((hash) => repairs.has(hash))
|
|
3254
|
+
: [];
|
|
3255
|
+
let attemptedHashCount = 0;
|
|
3256
|
+
if (transferHashes.length > 0) {
|
|
3257
|
+
try {
|
|
3258
|
+
await this.pushEntryHashes(peer, transferHashes, {
|
|
3259
|
+
acknowledge: false,
|
|
3260
|
+
chunkTimeout: getAttemptTimeout,
|
|
3261
|
+
chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
3262
|
+
isStillCurrent: isPeerRoundCurrent,
|
|
3263
|
+
onChunkAttempted: (chunk) => {
|
|
3264
|
+
attemptedHashCount += chunk.length;
|
|
3265
|
+
if (!transferAllOnRound) {
|
|
3266
|
+
const state = repairsByPeer.get(peer);
|
|
3267
|
+
for (const hash of chunk)
|
|
3268
|
+
state?.hashes.delete(hash);
|
|
3269
|
+
}
|
|
3270
|
+
},
|
|
3271
|
+
onChunkSent: async (chunk) => {
|
|
3272
|
+
return this.waitForPersistedTransferAdmission(peer, chunk, captured, roundSignal, isPeerRoundCurrent);
|
|
3273
|
+
},
|
|
3274
|
+
operationQueue,
|
|
3275
|
+
priority: delivery.priority,
|
|
3276
|
+
repairHint: !transferAllOnRound,
|
|
3277
|
+
signal: roundSignal,
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
catch {
|
|
3281
|
+
// A send can fail after remote admission. The attempted prefix
|
|
3282
|
+
// includes that uncertain chunk for authoritative receipt probing.
|
|
3283
|
+
}
|
|
3284
|
+
if (transferAllOnRound &&
|
|
3285
|
+
attemptedHashCount < transferHashes.length) {
|
|
3286
|
+
const state = ensureRepairState();
|
|
3287
|
+
for (const hash of transferHashes.slice(attemptedHashCount)) {
|
|
3288
|
+
state.hashes.add(hash);
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
const receiptHashes = transferAllOnRound
|
|
3293
|
+
? hashes.slice(0, attemptedHashCount)
|
|
3294
|
+
: hashes;
|
|
3295
|
+
if (!isPeerRoundCurrent()) {
|
|
3296
|
+
purgePeerDeliveryState(peer);
|
|
3297
|
+
return;
|
|
3298
|
+
}
|
|
3299
|
+
// Keep receipt chunks sequential per peer. The shared operation
|
|
3300
|
+
// queue bounds only active sends/requests; admission waits never
|
|
3301
|
+
// occupy its slots and cannot starve a later healthy candidate.
|
|
3302
|
+
for (let offset = 0; offset < receiptHashes.length; offset += PERSISTED_RECEIPT_CHUNK_SIZE) {
|
|
3303
|
+
const requestedHashes = receiptHashes.slice(offset, offset + PERSISTED_RECEIPT_CHUNK_SIZE);
|
|
3304
|
+
if (roundSignal.aborted || !isPeerRoundCurrent()) {
|
|
3305
|
+
if (!isRoundOwnershipCurrent())
|
|
3306
|
+
roundController.abort();
|
|
3307
|
+
break;
|
|
3308
|
+
}
|
|
3309
|
+
let responses;
|
|
3310
|
+
try {
|
|
3311
|
+
await this.waitForPersistedReceiptEgressAdmission(peer, captured.capabilitySession, requestedHashes.length, roundSignal);
|
|
3312
|
+
if (!isPeerRoundCurrent())
|
|
3313
|
+
break;
|
|
3314
|
+
const attemptTimeout = getAttemptTimeout();
|
|
3315
|
+
responses =
|
|
3316
|
+
(await operationQueue.add(async () => {
|
|
3317
|
+
if (!isPeerRoundCurrent())
|
|
3318
|
+
return [];
|
|
3319
|
+
return this.rpc.request(new RequestPersistedEntriesV1({
|
|
3320
|
+
expectedReceiverSession: captured.capabilitySession,
|
|
3321
|
+
hashes: requestedHashes,
|
|
3322
|
+
}), {
|
|
3323
|
+
mode: new SilentDelivery({
|
|
3324
|
+
to: [peer],
|
|
3325
|
+
redundancy: 1,
|
|
3326
|
+
}),
|
|
3327
|
+
amount: 1,
|
|
3328
|
+
priority: delivery.priority,
|
|
3329
|
+
timeout: attemptTimeout,
|
|
3330
|
+
signal: roundSignal,
|
|
3331
|
+
});
|
|
3332
|
+
})) ?? [];
|
|
3333
|
+
}
|
|
3334
|
+
catch {
|
|
3335
|
+
if (roundSignal.aborted)
|
|
3336
|
+
break;
|
|
3337
|
+
// A peer can disconnect or miss this retry while the overall
|
|
3338
|
+
// quorum deadline remains active. Replan on the next round.
|
|
3339
|
+
break;
|
|
3340
|
+
}
|
|
3341
|
+
if (!isRoundOwnershipCurrent()) {
|
|
3342
|
+
roundController.abort();
|
|
3343
|
+
break;
|
|
3344
|
+
}
|
|
3345
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3346
|
+
if (!current ||
|
|
3347
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3348
|
+
current.peerSession !== captured.peerSession) {
|
|
3349
|
+
purgePeerDeliveryState(peer);
|
|
3350
|
+
break;
|
|
3351
|
+
}
|
|
3352
|
+
const requested = new Set(requestedHashes);
|
|
3353
|
+
const confirmed = new Set();
|
|
3354
|
+
let receivedValidConfirmation = false;
|
|
3355
|
+
for (const result of responses) {
|
|
3356
|
+
if (!(result.response instanceof ConfirmEntriesMessage) ||
|
|
3357
|
+
result.from?.hashcode() !== peer ||
|
|
3358
|
+
result.message.header.session !== captured.capabilitySession) {
|
|
3359
|
+
continue;
|
|
3360
|
+
}
|
|
3361
|
+
const unique = new Set(result.response.hashes);
|
|
3362
|
+
if (unique.size !== result.response.hashes.length ||
|
|
3363
|
+
[...unique].some((hash) => !requested.has(hash))) {
|
|
3364
|
+
continue;
|
|
3365
|
+
}
|
|
3366
|
+
receivedValidConfirmation = true;
|
|
3367
|
+
for (const hash of unique) {
|
|
3368
|
+
confirmed.add(hash);
|
|
3369
|
+
carriedAcknowledgements.get(hash)?.set(peer, captured);
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
if (receivedValidConfirmation) {
|
|
3373
|
+
const state = ensureRepairState();
|
|
3374
|
+
for (const hash of requested) {
|
|
3375
|
+
if (confirmed.has(hash)) {
|
|
3376
|
+
state.hashes.delete(hash);
|
|
3377
|
+
}
|
|
3378
|
+
else {
|
|
3379
|
+
state.hashes.add(hash);
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3384
|
+
})()
|
|
3385
|
+
.then(() => undefined)
|
|
3386
|
+
.catch(() => undefined)
|
|
3387
|
+
.finally(() => requests.delete(request));
|
|
3388
|
+
requests.add(request);
|
|
3389
|
+
}
|
|
3390
|
+
const roundComplete = async () => {
|
|
3391
|
+
if (signal.aborted) {
|
|
3392
|
+
throw signal.reason ?? new AbortError();
|
|
3393
|
+
}
|
|
3394
|
+
if (!isRoundOwnershipCurrent())
|
|
3395
|
+
return false;
|
|
3396
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
3397
|
+
if (acknowledgements.size < minAcks)
|
|
3398
|
+
return false;
|
|
3399
|
+
}
|
|
3400
|
+
const validatedLeaders = await this.planPersistedDeliveryLeaders(entryArray, replicas, ownershipLifecycleController);
|
|
3401
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
3402
|
+
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
3403
|
+
if (signal.aborted) {
|
|
3404
|
+
throw signal.reason ?? new AbortError();
|
|
3405
|
+
}
|
|
3406
|
+
return false;
|
|
3407
|
+
}
|
|
3408
|
+
const hash = entryArray[index].hash;
|
|
3409
|
+
const leaders = validatedLeaders[index];
|
|
3410
|
+
let valid = 0;
|
|
3411
|
+
const acknowledgements = carriedAcknowledgements.get(hash);
|
|
3412
|
+
for (const [peer, captured] of acknowledgements) {
|
|
3413
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3414
|
+
if (!current ||
|
|
3415
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
3416
|
+
current.peerSession !== captured.peerSession) {
|
|
3417
|
+
purgePeerDeliveryState(peer);
|
|
3418
|
+
continue;
|
|
3419
|
+
}
|
|
3420
|
+
if (!leaders.has(peer)) {
|
|
3421
|
+
acknowledgements.delete(peer);
|
|
3422
|
+
continue;
|
|
3423
|
+
}
|
|
3424
|
+
if (++valid >= minAcks) {
|
|
3425
|
+
break;
|
|
3426
|
+
}
|
|
3427
|
+
}
|
|
3428
|
+
if (valid < minAcks)
|
|
3429
|
+
return false;
|
|
3430
|
+
}
|
|
3431
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
3432
|
+
if (signal.aborted) {
|
|
3433
|
+
throw signal.reason ?? new AbortError();
|
|
3434
|
+
}
|
|
3435
|
+
return isRoundOwnershipCurrent();
|
|
3436
|
+
};
|
|
3437
|
+
while (requests.size > 0) {
|
|
3438
|
+
await Promise.race(requests);
|
|
3439
|
+
if (await roundComplete()) {
|
|
3440
|
+
return;
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
if (await roundComplete()) {
|
|
3444
|
+
return;
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
finally {
|
|
3448
|
+
roundController.abort();
|
|
3449
|
+
void Promise.allSettled([...requests]);
|
|
3450
|
+
}
|
|
3451
|
+
if (isRoundOwnershipCurrent()) {
|
|
3452
|
+
initialTransferPending = false;
|
|
3453
|
+
}
|
|
3454
|
+
// Keep early retries fair and responsive, then let a caller's longer
|
|
3455
|
+
// overall deadline accommodate a genuinely slow durability barrier.
|
|
3456
|
+
maxAttemptMs = Math.min(MAX_PERSISTED_DELIVERY_TIMEOUT_MS, maxAttemptMs * 2);
|
|
3457
|
+
await this.waitPersistedReceiptRetry(signal, Math.max(0, Math.min(PERSISTED_RECEIPT_RETRY_MS, deadline.deadline - Date.now())));
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
catch (error) {
|
|
3461
|
+
if (error instanceof PersistedDeliveryError) {
|
|
3462
|
+
throw error;
|
|
3463
|
+
}
|
|
3464
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
3465
|
+
}
|
|
3466
|
+
finally {
|
|
3467
|
+
if (ownedDeadline)
|
|
3468
|
+
deadline.dispose();
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
2750
3471
|
async _appendDeliverToReplicators(entry, coordinates, minReplicasValue, leaders, selfHash, isLeader, deliveryArg, nativeDeliveryPlan, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
|
|
2751
3472
|
const throwIfInactive = () => this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
2752
3473
|
throwIfInactive();
|
|
@@ -4959,10 +5680,7 @@ let SharedLog = (() => {
|
|
|
4959
5680
|
try {
|
|
4960
5681
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
4961
5682
|
if (fanoutService?.provide && !this._providerHandle) {
|
|
4962
|
-
this.
|
|
4963
|
-
ttlMs: 120_000,
|
|
4964
|
-
announceIntervalMs: 60_000,
|
|
4965
|
-
});
|
|
5683
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
4966
5684
|
}
|
|
4967
5685
|
}
|
|
4968
5686
|
catch {
|
|
@@ -5363,47 +6081,93 @@ let SharedLog = (() => {
|
|
|
5363
6081
|
});
|
|
5364
6082
|
}
|
|
5365
6083
|
}
|
|
5366
|
-
async
|
|
5367
|
-
if (!isStillCurrent())
|
|
5368
|
-
return;
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
if (
|
|
5372
|
-
|
|
5373
|
-
|
|
5374
|
-
|
|
5375
|
-
const sentMessages = await this.trySendFusedRawExchangeHeads(
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
|
|
5379
|
-
|
|
5380
|
-
|
|
6084
|
+
async pushEntryHashChunk(target, chunk, options, isStillCurrent) {
|
|
6085
|
+
if (!isStillCurrent())
|
|
6086
|
+
return false;
|
|
6087
|
+
const useRaw = this._logProperties?.sync?.rawExchangeHeads === true &&
|
|
6088
|
+
this.peerSupportsRawExchangeHeads(target);
|
|
6089
|
+
if (useRaw) {
|
|
6090
|
+
const reserved = options.repairHint ? new Uint8Array(4) : undefined;
|
|
6091
|
+
if (reserved)
|
|
6092
|
+
reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
6093
|
+
const sentMessages = await this.trySendFusedRawExchangeHeads(chunk, [target], {
|
|
6094
|
+
acknowledge: options.acknowledge,
|
|
6095
|
+
priority: options.priority,
|
|
6096
|
+
reserved,
|
|
6097
|
+
signal: options.signal,
|
|
6098
|
+
});
|
|
6099
|
+
if (!isStillCurrent())
|
|
6100
|
+
return false;
|
|
6101
|
+
if (sentMessages === undefined) {
|
|
6102
|
+
for await (const message of createRawExchangeHeadsMessages(this.log, chunk, this._logProperties?.sync?.profile)) {
|
|
6103
|
+
if (!isStillCurrent())
|
|
6104
|
+
return false;
|
|
6105
|
+
if (options.repairHint) {
|
|
6106
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
6107
|
+
}
|
|
6108
|
+
await this.rpc.send(message, {
|
|
6109
|
+
priority: options.priority,
|
|
6110
|
+
mode: options.acknowledge
|
|
6111
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
6112
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
6113
|
+
signal: options.signal,
|
|
6114
|
+
});
|
|
6115
|
+
}
|
|
5381
6116
|
}
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
6117
|
+
}
|
|
6118
|
+
else {
|
|
6119
|
+
for await (const message of createExchangeHeadsMessages(this.log, chunk)) {
|
|
6120
|
+
if (!isStillCurrent())
|
|
6121
|
+
return false;
|
|
6122
|
+
if (options.repairHint) {
|
|
6123
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5385
6124
|
}
|
|
5386
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5387
6125
|
await this.rpc.send(message, {
|
|
5388
|
-
priority:
|
|
5389
|
-
mode:
|
|
5390
|
-
|
|
6126
|
+
priority: options.priority,
|
|
6127
|
+
mode: options.acknowledge
|
|
6128
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
6129
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
6130
|
+
signal: options.signal,
|
|
5391
6131
|
});
|
|
5392
6132
|
}
|
|
5393
|
-
return;
|
|
5394
6133
|
}
|
|
5395
|
-
|
|
5396
|
-
|
|
6134
|
+
return isStillCurrent();
|
|
6135
|
+
}
|
|
6136
|
+
async pushEntryHashes(target, hashes, options) {
|
|
6137
|
+
const isStillCurrent = options.isStillCurrent ?? (() => true);
|
|
6138
|
+
if (!isStillCurrent())
|
|
6139
|
+
return;
|
|
6140
|
+
const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
|
|
6141
|
+
for (let offset = 0; offset < hashes.length; offset += chunkSize) {
|
|
6142
|
+
const chunk = hashes.slice(offset, offset + chunkSize);
|
|
6143
|
+
const pushChunk = () => {
|
|
6144
|
+
options.onChunkAttempted?.(chunk);
|
|
6145
|
+
const chunkTimeout = options.chunkTimeout?.();
|
|
6146
|
+
const chunkSignal = chunkTimeout == null
|
|
6147
|
+
? options.signal
|
|
6148
|
+
: AbortSignal.any([
|
|
6149
|
+
...(options.signal ? [options.signal] : []),
|
|
6150
|
+
AbortSignal.timeout(Math.max(1, chunkTimeout)),
|
|
6151
|
+
]);
|
|
6152
|
+
return this.pushEntryHashChunk(target, chunk, { ...options, signal: chunkSignal }, isStillCurrent);
|
|
6153
|
+
};
|
|
6154
|
+
const pushed = options.operationQueue
|
|
6155
|
+
? await options.operationQueue.add(pushChunk)
|
|
6156
|
+
: await pushChunk();
|
|
6157
|
+
if (pushed !== true)
|
|
6158
|
+
return;
|
|
6159
|
+
if (options.onChunkSent && !(await options.onChunkSent(chunk)))
|
|
5397
6160
|
return;
|
|
5398
|
-
}
|
|
5399
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
5400
|
-
await this.rpc.send(message, {
|
|
5401
|
-
priority: SYNC_MESSAGE_PRIORITY,
|
|
5402
|
-
mode: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
5403
|
-
signal,
|
|
5404
|
-
});
|
|
5405
6161
|
}
|
|
5406
6162
|
}
|
|
6163
|
+
async pushRepairEntries(target, entries, isStillCurrent = () => true, signal) {
|
|
6164
|
+
return this.pushEntryHashes(target, [...entries.keys()], {
|
|
6165
|
+
isStillCurrent,
|
|
6166
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
6167
|
+
repairHint: true,
|
|
6168
|
+
signal,
|
|
6169
|
+
});
|
|
6170
|
+
}
|
|
5407
6171
|
async sendRepairEntriesWithTransport(target, entries, transport, options) {
|
|
5408
6172
|
const isStillCurrent = options?.isStillCurrent ?? (() => true);
|
|
5409
6173
|
if (!isStillCurrent()) {
|
|
@@ -6781,23 +7545,65 @@ let SharedLog = (() => {
|
|
|
6781
7545
|
}
|
|
6782
7546
|
async append(data, options) {
|
|
6783
7547
|
this.throwIfNativeDurableCommitFailed();
|
|
7548
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
6784
7549
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6785
7550
|
if (this._isAdaptiveReplicating) {
|
|
6786
7551
|
this.markLocalAppendActivity();
|
|
6787
7552
|
}
|
|
6788
7553
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options, ownershipLifecycleController);
|
|
6789
|
-
|
|
6790
|
-
|
|
6791
|
-
|
|
6792
|
-
|
|
6793
|
-
|
|
6794
|
-
|
|
6795
|
-
|
|
6796
|
-
|
|
7554
|
+
let committedHashes;
|
|
7555
|
+
if (persistedDelivery) {
|
|
7556
|
+
appendOptions.__peerbitOnLocalCommit = (hashes) => {
|
|
7557
|
+
// The lower log reports the exact hashes immediately after their
|
|
7558
|
+
// irreversible local mutation and before entry initialization, trim,
|
|
7559
|
+
// or change callbacks can reject the append.
|
|
7560
|
+
committedHashes = hashes;
|
|
7561
|
+
};
|
|
7562
|
+
}
|
|
7563
|
+
let persistedDeadline;
|
|
7564
|
+
const throwIfDeliveryAborted = () => {
|
|
7565
|
+
if (persistedDeadline?.signal.aborted) {
|
|
7566
|
+
throw persistedDeadline.signal.reason ?? new AbortError();
|
|
7567
|
+
}
|
|
7568
|
+
};
|
|
7569
|
+
try {
|
|
7570
|
+
const result = await this.log.append(data, appendOptions);
|
|
7571
|
+
committedHashes ??= [result.entry.hash];
|
|
7572
|
+
persistedDeadline = persistedDelivery
|
|
7573
|
+
? this.createPersistedDeliveryDeadline(persistedDelivery, ownershipLifecycleController, 1)
|
|
7574
|
+
: undefined;
|
|
7575
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7576
|
+
throwIfDeliveryAborted();
|
|
7577
|
+
await this.processLocalAppend(result.entry, result.removed, options, {
|
|
7578
|
+
minReplicasValue,
|
|
7579
|
+
ownershipLifecycleController,
|
|
7580
|
+
});
|
|
7581
|
+
throwIfDeliveryAborted();
|
|
7582
|
+
if (persistedDelivery && persistedDeadline) {
|
|
7583
|
+
await this.settlePersistedDelivery([result.entry], minReplicasValue, persistedDelivery, ownershipLifecycleController, persistedDeadline);
|
|
7584
|
+
}
|
|
7585
|
+
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7586
|
+
return result;
|
|
7587
|
+
}
|
|
7588
|
+
catch (error) {
|
|
7589
|
+
if (persistedDelivery && committedHashes) {
|
|
7590
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
7591
|
+
}
|
|
7592
|
+
throw error;
|
|
7593
|
+
}
|
|
7594
|
+
finally {
|
|
7595
|
+
persistedDeadline?.dispose();
|
|
7596
|
+
}
|
|
7597
|
+
}
|
|
7598
|
+
rejectPersistedDeliveryOnTrustedLocalAppend(options) {
|
|
7599
|
+
if (this.getPersistedDeliveryOptions(options)) {
|
|
7600
|
+
throw new Error("trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit");
|
|
7601
|
+
}
|
|
6797
7602
|
}
|
|
6798
7603
|
// Trusted local append path for callers that already validated the entry.
|
|
6799
7604
|
async appendLocallyValidated(data, options) {
|
|
6800
7605
|
this.throwIfNativeDurableCommitFailed();
|
|
7606
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6801
7607
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6802
7608
|
if (options?.canAppend || options?.onChange) {
|
|
6803
7609
|
throw new Error("appendLocallyValidated does not accept canAppend or onChange hooks");
|
|
@@ -6820,6 +7626,7 @@ let SharedLog = (() => {
|
|
|
6820
7626
|
// Trusted local append path that lets the shared log own change application.
|
|
6821
7627
|
async appendLocallyPrepared(data, options, properties) {
|
|
6822
7628
|
this.throwIfNativeDurableCommitFailed();
|
|
7629
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6823
7630
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
6824
7631
|
if (options?.canAppend || options?.onChange) {
|
|
6825
7632
|
throw new Error("appendLocallyPrepared does not accept canAppend or onChange hooks");
|
|
@@ -6829,11 +7636,15 @@ let SharedLog = (() => {
|
|
|
6829
7636
|
}
|
|
6830
7637
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
6831
7638
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7639
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
6832
7640
|
const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(data, appendOptions, {
|
|
6833
7641
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
6834
7642
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
6835
7643
|
payloadData: properties?.payloadData,
|
|
6836
7644
|
});
|
|
7645
|
+
if (properties?.localCommitEvidence) {
|
|
7646
|
+
properties.localCommitEvidence.committedHashes.add(result.appendFacts.hash);
|
|
7647
|
+
}
|
|
6837
7648
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
6838
7649
|
const nativePreparedCommit = await this.processNativePreparedTargetNoneAppend(result, options, {
|
|
6839
7650
|
minReplicasValue,
|
|
@@ -6982,11 +7793,15 @@ let SharedLog = (() => {
|
|
|
6982
7793
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
6983
7794
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
6984
7795
|
payloadData,
|
|
7796
|
+
...(properties?.localCommitEvidence
|
|
7797
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
7798
|
+
: undefined),
|
|
6985
7799
|
});
|
|
6986
7800
|
}
|
|
6987
7801
|
// Trusted local payload append path that keeps the public Entry lazy.
|
|
6988
7802
|
appendLocallyPreparedPayloadCommitOnly(payloadData, options, properties) {
|
|
6989
7803
|
this.throwIfNativeDurableCommitFailed();
|
|
7804
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
6990
7805
|
if (options?.canAppend || options?.onChange) {
|
|
6991
7806
|
throw new Error("appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
6992
7807
|
}
|
|
@@ -7003,6 +7818,7 @@ let SharedLog = (() => {
|
|
|
7003
7818
|
}
|
|
7004
7819
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
7005
7820
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7821
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
7006
7822
|
const deferHeadCoordinatePersistence = this.shouldDeferHeadCoordinatePersistence(options);
|
|
7007
7823
|
const nativeBackboneResult = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(payloadData, appendOptions, options, properties, minReplicasValue, deferHeadCoordinatePersistence, ownershipLifecycleController);
|
|
7008
7824
|
if (nativeBackboneResult) {
|
|
@@ -7019,6 +7835,7 @@ let SharedLog = (() => {
|
|
|
7019
7835
|
// Strict native document path. Never falls back to compatibility append.
|
|
7020
7836
|
appendStrictNativeDocumentPayloadCommitOnly(payloadData, options, properties) {
|
|
7021
7837
|
this.throwIfNativeDurableCommitFailed();
|
|
7838
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
7022
7839
|
if (options?.canAppend || options?.onChange) {
|
|
7023
7840
|
throw new Error("appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks");
|
|
7024
7841
|
}
|
|
@@ -7035,6 +7852,7 @@ let SharedLog = (() => {
|
|
|
7035
7852
|
}
|
|
7036
7853
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
7037
7854
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
7855
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
7038
7856
|
const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(payloadData, appendOptions, options, properties, minReplicasValue, this.shouldDeferHeadCoordinatePersistence(options), ownershipLifecycleController);
|
|
7039
7857
|
return mapMaybePromise(result, (commitOnly) => {
|
|
7040
7858
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
@@ -7051,6 +7869,9 @@ let SharedLog = (() => {
|
|
|
7051
7869
|
includeAppendFactsBytes: !deferHeadCoordinatePersistence,
|
|
7052
7870
|
});
|
|
7053
7871
|
return mapMaybePromise(resultMaybe, (result) => {
|
|
7872
|
+
if (result && properties?.localCommitEvidence) {
|
|
7873
|
+
properties.localCommitEvidence.committedHashes.add(result.appendFacts.hash);
|
|
7874
|
+
}
|
|
7054
7875
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
7055
7876
|
return this.finishPreparedPayloadCommitOnlyAppend(result, options, minReplicasValue, ownershipLifecycleController);
|
|
7056
7877
|
});
|
|
@@ -7431,17 +8252,16 @@ let SharedLog = (() => {
|
|
|
7431
8252
|
// boundary separates the catch above from this statement and
|
|
7432
8253
|
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
7433
8254
|
// (the retire below only warns), so the token is terminal here.
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
removedHashes: prepared.removedHashes,
|
|
7437
|
-
removedGids: prepared.removedGids,
|
|
7438
|
-
});
|
|
7439
|
-
try {
|
|
7440
|
-
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7441
|
-
}
|
|
7442
|
-
catch (error) {
|
|
7443
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
8255
|
+
if (properties?.localCommitEvidence) {
|
|
8256
|
+
properties.localCommitEvidence.committedHashes.add(prepared.appendFacts.hash);
|
|
7444
8257
|
}
|
|
8258
|
+
await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
8259
|
+
this._coordinates.settleResidentCoordinateSnapshot(lowerPublicationRollback?.coordinateEntries);
|
|
8260
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
8261
|
+
removedHashes: prepared.removedHashes,
|
|
8262
|
+
removedGids: prepared.removedGids,
|
|
8263
|
+
});
|
|
8264
|
+
});
|
|
7445
8265
|
return finishResult;
|
|
7446
8266
|
});
|
|
7447
8267
|
}
|
|
@@ -7882,18 +8702,17 @@ let SharedLog = (() => {
|
|
|
7882
8702
|
// Success seam: the finalizer acknowledge above is the last
|
|
7883
8703
|
// await inside the protected try, so `rollback` can no
|
|
7884
8704
|
// longer fire and the retire below only warns.
|
|
7885
|
-
|
|
7886
|
-
|
|
7887
|
-
forgetNativeCoordinates: false,
|
|
7888
|
-
removedHashes: prepared.removedHashes,
|
|
7889
|
-
removedGids: prepared.removedGids,
|
|
7890
|
-
});
|
|
7891
|
-
try {
|
|
7892
|
-
await this.completeNativeStrictDurableTransaction(nativeStrictTransaction);
|
|
7893
|
-
}
|
|
7894
|
-
catch (error) {
|
|
7895
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
8705
|
+
if (properties?.localCommitEvidence) {
|
|
8706
|
+
properties.localCommitEvidence.committedHashes.add(prepared.appendFacts.hash);
|
|
7896
8707
|
}
|
|
8708
|
+
await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
8709
|
+
this._coordinates.settleResidentCoordinateSnapshot(rollbackCoordinateEntries);
|
|
8710
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(prepared.appendFacts, prepared.removed, prepared.materializeEntry, {
|
|
8711
|
+
forgetNativeCoordinates: false,
|
|
8712
|
+
removedHashes: prepared.removedHashes,
|
|
8713
|
+
removedGids: prepared.removedGids,
|
|
8714
|
+
});
|
|
8715
|
+
});
|
|
7897
8716
|
if (commitBlocksInBackbone &&
|
|
7898
8717
|
!runtimeOnlyCoordinates &&
|
|
7899
8718
|
this.remoteBlocks.hasNotifyStoredHook()) {
|
|
@@ -8562,50 +9381,49 @@ let SharedLog = (() => {
|
|
|
8562
9381
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8563
9382
|
return this.markNativeStrictDurableTransactionLowerMarker(nativeStrictTransaction);
|
|
8564
9383
|
});
|
|
8565
|
-
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8566
9384
|
}
|
|
8567
9385
|
catch (error) {
|
|
8568
9386
|
return rollbackBatch(error);
|
|
8569
9387
|
}
|
|
8570
9388
|
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
8571
9389
|
// above), and everything from here on escapes without any rollback.
|
|
8572
|
-
|
|
8573
|
-
|
|
8574
|
-
|
|
8575
|
-
for (let i = 0; i < coordinateRows.length; i++) {
|
|
8576
|
-
const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
|
|
8577
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
|
|
8578
|
-
forgetNativeCoordinates: false,
|
|
8579
|
-
removedHashes: plannedCoordinateDeleteHashes,
|
|
8580
|
-
removedGids: backboneAppend.trimmedGids ??
|
|
8581
|
-
(backboneAppend.trimmed.length > 0
|
|
8582
|
-
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
8583
|
-
: undefined),
|
|
8584
|
-
});
|
|
8585
|
-
if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
|
|
8586
|
-
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
9390
|
+
if (properties?.localCommitEvidence) {
|
|
9391
|
+
for (const facts of appended.appendFacts) {
|
|
9392
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
8587
9393
|
}
|
|
8588
|
-
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
|
|
8589
|
-
hashNumber: backboneAppend.coordinate.hashNumber,
|
|
8590
|
-
coordinateFields,
|
|
8591
|
-
});
|
|
8592
|
-
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
8593
|
-
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
8594
|
-
appended.documentTrimmedHeadsProcessed?.[i];
|
|
8595
|
-
appendCommit.documentPreviousContext =
|
|
8596
|
-
backboneAppend.documentPreviousContext;
|
|
8597
|
-
appendCommits.push(appendCommit);
|
|
8598
9394
|
}
|
|
8599
|
-
|
|
8600
|
-
|
|
9395
|
+
const appendCommits = await this.finishCommittedNativeStrictDurableTransaction(nativeStrictTransaction, () => {
|
|
9396
|
+
this._coordinates.settleResidentCoordinateSnapshot(batchCoordinateRollback);
|
|
8601
9397
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8602
|
-
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
9398
|
+
const commits = [];
|
|
9399
|
+
for (let i = 0; i < coordinateRows.length; i++) {
|
|
9400
|
+
const { facts, backboneAppend, coordinateFields, plannedCoordinateDeleteHashes, } = coordinateRows[i];
|
|
9401
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(facts, [], appended.materializeEntries[i], {
|
|
9402
|
+
forgetNativeCoordinates: false,
|
|
9403
|
+
removedHashes: plannedCoordinateDeleteHashes,
|
|
9404
|
+
removedGids: backboneAppend.trimmedGids ??
|
|
9405
|
+
(backboneAppend.trimmed.length > 0
|
|
9406
|
+
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
9407
|
+
: undefined),
|
|
9408
|
+
});
|
|
9409
|
+
if (!runtimeOnlyCoordinates &&
|
|
9410
|
+
this.remoteBlocks.hasNotifyStoredHook()) {
|
|
9411
|
+
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
9412
|
+
}
|
|
9413
|
+
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(facts, {
|
|
9414
|
+
hashNumber: backboneAppend.coordinate
|
|
9415
|
+
.hashNumber,
|
|
9416
|
+
coordinateFields,
|
|
9417
|
+
});
|
|
9418
|
+
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
9419
|
+
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
9420
|
+
appended.documentTrimmedHeadsProcessed?.[i];
|
|
9421
|
+
appendCommit.documentPreviousContext =
|
|
9422
|
+
backboneAppend.documentPreviousContext;
|
|
9423
|
+
commits.push(appendCommit);
|
|
9424
|
+
}
|
|
9425
|
+
return commits;
|
|
9426
|
+
}, () => this.isRepairLifecycleActive(ownershipLifecycleController));
|
|
8609
9427
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8610
9428
|
const delayAdaptiveRebalance = this.shouldDelayAdaptiveRebalance();
|
|
8611
9429
|
if (!delayAdaptiveRebalance) {
|
|
@@ -8622,6 +9440,7 @@ let SharedLog = (() => {
|
|
|
8622
9440
|
}
|
|
8623
9441
|
async appendLocallyPreparedManyIndependent(data, options, properties) {
|
|
8624
9442
|
this.throwIfNativeDurableCommitFailed();
|
|
9443
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
8625
9444
|
if (data.length === 0) {
|
|
8626
9445
|
return { entries: [], removed: [], appendCommits: [] };
|
|
8627
9446
|
}
|
|
@@ -8634,6 +9453,7 @@ let SharedLog = (() => {
|
|
|
8634
9453
|
}
|
|
8635
9454
|
const { appendOptions, minReplicasValue } = this.createLogAppendOptions(options);
|
|
8636
9455
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
9456
|
+
attachTrustedLocalCommitEvidence(appendOptions, properties?.localCommitEvidence);
|
|
8637
9457
|
const nativeBackboneBatch = await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(data, appendOptions, options, properties, minReplicasValue, ownershipLifecycleController);
|
|
8638
9458
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8639
9459
|
if (nativeBackboneBatch) {
|
|
@@ -8644,6 +9464,11 @@ let SharedLog = (() => {
|
|
|
8644
9464
|
payloadDatas: properties?.payloadDatas,
|
|
8645
9465
|
nexts: properties?.nexts,
|
|
8646
9466
|
});
|
|
9467
|
+
if (result && properties?.localCommitEvidence) {
|
|
9468
|
+
for (const facts of result.appendFacts) {
|
|
9469
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
9470
|
+
}
|
|
9471
|
+
}
|
|
8647
9472
|
this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
8648
9473
|
if (!result) {
|
|
8649
9474
|
return undefined;
|
|
@@ -8712,13 +9537,20 @@ let SharedLog = (() => {
|
|
|
8712
9537
|
nexts: properties?.nexts,
|
|
8713
9538
|
nativeBackboneDocumentIndexes: properties?.nativeBackboneDocumentIndexes,
|
|
8714
9539
|
retainMaterializationBytes: properties?.retainMaterializationBytes,
|
|
9540
|
+
...(properties?.localCommitEvidence
|
|
9541
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
9542
|
+
: undefined),
|
|
8715
9543
|
});
|
|
8716
9544
|
}
|
|
8717
9545
|
async appendMany(data, options) {
|
|
8718
9546
|
this.throwIfNativeDurableCommitFailed();
|
|
9547
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
8719
9548
|
if (data.length === 0) {
|
|
8720
9549
|
return { entries: [], removed: [] };
|
|
8721
9550
|
}
|
|
9551
|
+
if (persistedDelivery) {
|
|
9552
|
+
throw new Error("persisted delivery is not supported for chained appendMany; use independent document puts");
|
|
9553
|
+
}
|
|
8722
9554
|
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
8723
9555
|
if (this._isAdaptiveReplicating) {
|
|
8724
9556
|
this.markLocalAppendActivity();
|
|
@@ -8761,6 +9593,69 @@ let SharedLog = (() => {
|
|
|
8761
9593
|
}
|
|
8762
9594
|
return result;
|
|
8763
9595
|
}
|
|
9596
|
+
/**
|
|
9597
|
+
* Deliver entries that were already committed locally, then wait for the
|
|
9598
|
+
* requested persisted remote quorum. This is the post-commit seam used by
|
|
9599
|
+
* higher-level transactional/batched writers so a receipt timeout never
|
|
9600
|
+
* rolls back or hides their successful local commit.
|
|
9601
|
+
*/
|
|
9602
|
+
async deliverPersistedPlanningEntries(resolveEntries, committedHashes, options) {
|
|
9603
|
+
let persistedDeadline;
|
|
9604
|
+
try {
|
|
9605
|
+
const delivery = this.getPersistedDeliveryOptions(options);
|
|
9606
|
+
if (!delivery) {
|
|
9607
|
+
throw new Error('deliverPersistedEntries requires reliability="persisted"');
|
|
9608
|
+
}
|
|
9609
|
+
const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
|
|
9610
|
+
const deadline = this.createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, committedHashes.length);
|
|
9611
|
+
persistedDeadline = deadline;
|
|
9612
|
+
const throwIfDeliveryAborted = () => {
|
|
9613
|
+
if (deadline.signal.aborted) {
|
|
9614
|
+
throw deadline.signal.reason ?? new AbortError();
|
|
9615
|
+
}
|
|
9616
|
+
if (Date.now() >= deadline.deadline) {
|
|
9617
|
+
throw new TimeoutError(`Timed out waiting for ${Math.floor(delivery.minAcks)} persisted remote replicas.`);
|
|
9618
|
+
}
|
|
9619
|
+
};
|
|
9620
|
+
const entries = resolveEntries();
|
|
9621
|
+
throwIfDeliveryAborted();
|
|
9622
|
+
if (entries.length === 0)
|
|
9623
|
+
return;
|
|
9624
|
+
const { minReplicasValue } = this.createLogAppendOptions(options, ownershipLifecycleController);
|
|
9625
|
+
throwIfDeliveryAborted();
|
|
9626
|
+
await this.settlePersistedDelivery(entries, minReplicasValue, delivery, ownershipLifecycleController, deadline, true);
|
|
9627
|
+
}
|
|
9628
|
+
catch (error) {
|
|
9629
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
9630
|
+
}
|
|
9631
|
+
finally {
|
|
9632
|
+
persistedDeadline?.dispose();
|
|
9633
|
+
}
|
|
9634
|
+
}
|
|
9635
|
+
createPersistedDeliveryPlanningEntries(appendCommits, materializeEntries) {
|
|
9636
|
+
let materializedEntries;
|
|
9637
|
+
const requiresFullEntries = this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
|
|
9638
|
+
const getMaterializedEntry = (appendCommit, index) => {
|
|
9639
|
+
materializedEntries ??= materializeEntries();
|
|
9640
|
+
const entry = materializedEntries[index];
|
|
9641
|
+
if (!entry || entry.hash !== appendCommit.hash) {
|
|
9642
|
+
throw new Error(`Persisted delivery materializer did not return committed entry ${appendCommit.hash}`);
|
|
9643
|
+
}
|
|
9644
|
+
return entry;
|
|
9645
|
+
};
|
|
9646
|
+
return appendCommits.map((appendCommit, index) => {
|
|
9647
|
+
if (!requiresFullEntries && appendCommit.coordinateFields) {
|
|
9648
|
+
return this._coordinates.materializeResidentCoordinateEntry(appendCommit.coordinateFields);
|
|
9649
|
+
}
|
|
9650
|
+
return getMaterializedEntry(appendCommit, index);
|
|
9651
|
+
});
|
|
9652
|
+
}
|
|
9653
|
+
deliverPersistedAppendCommits(appendCommits, materializeEntries, options) {
|
|
9654
|
+
return this.deliverPersistedPlanningEntries(() => this.createPersistedDeliveryPlanningEntries(appendCommits, materializeEntries), appendCommits.map((appendCommit) => appendCommit.hash), options);
|
|
9655
|
+
}
|
|
9656
|
+
async deliverPersistedEntries(entries, options) {
|
|
9657
|
+
return this.deliverPersistedPlanningEntries(() => entries, entries.map((entry) => entry.hash), options);
|
|
9658
|
+
}
|
|
8764
9659
|
canCoalesceLocalAppendMany(entries, options) {
|
|
8765
9660
|
if (entries.length <= 1 ||
|
|
8766
9661
|
options?.target === "all" ||
|
|
@@ -9525,6 +10420,9 @@ let SharedLog = (() => {
|
|
|
9525
10420
|
this._peerSyncCapabilities = new Map();
|
|
9526
10421
|
this._peerSyncCapabilitySessions = new Map();
|
|
9527
10422
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
10423
|
+
this._persistedReceiptStorage = undefined;
|
|
10424
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
10425
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
9528
10426
|
this._liveRawGossipBatches = new Map();
|
|
9529
10427
|
this._liveRawGossipFlushScheduled = false;
|
|
9530
10428
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
@@ -9611,6 +10509,43 @@ let SharedLog = (() => {
|
|
|
9611
10509
|
]);
|
|
9612
10510
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
9613
10511
|
const blockProviderNamespace = (cid) => `cid:${cid}`;
|
|
10512
|
+
const logProviderNamespace = `shared-log|${this.topic}`;
|
|
10513
|
+
const announceBlockProvider = async (cid) => {
|
|
10514
|
+
try {
|
|
10515
|
+
await fanoutService?.announceProvider(blockProviderNamespace(cid), {
|
|
10516
|
+
ttlMs: 120_000,
|
|
10517
|
+
bootstrapMaxPeers: 2,
|
|
10518
|
+
});
|
|
10519
|
+
}
|
|
10520
|
+
catch {
|
|
10521
|
+
// Provider publication is best-effort.
|
|
10522
|
+
}
|
|
10523
|
+
};
|
|
10524
|
+
const announceBlockProviders = async (cids) => {
|
|
10525
|
+
const batchedAnnounce = fanoutService?.announceProviders;
|
|
10526
|
+
if (typeof batchedAnnounce === "function") {
|
|
10527
|
+
const namespaces = function* () {
|
|
10528
|
+
for (const cid of cids)
|
|
10529
|
+
yield blockProviderNamespace(cid);
|
|
10530
|
+
};
|
|
10531
|
+
await batchedAnnounce.call(fanoutService, namespaces(), {
|
|
10532
|
+
ttlMs: 120_000,
|
|
10533
|
+
bootstrapMaxPeers: 2,
|
|
10534
|
+
});
|
|
10535
|
+
return;
|
|
10536
|
+
}
|
|
10537
|
+
// Tolerate a skewed runtime that supplied an older FanoutTree service.
|
|
10538
|
+
let nextIndex = 0;
|
|
10539
|
+
const worker = async () => {
|
|
10540
|
+
for (;;) {
|
|
10541
|
+
const index = nextIndex++;
|
|
10542
|
+
if (index >= cids.length)
|
|
10543
|
+
return;
|
|
10544
|
+
await announceBlockProvider(cids[index]);
|
|
10545
|
+
}
|
|
10546
|
+
};
|
|
10547
|
+
await Promise.all(Array.from({ length: Math.min(8, cids.length) }, () => worker()));
|
|
10548
|
+
};
|
|
9614
10549
|
const [replicationIndex, logIndex] = await Promise.all([
|
|
9615
10550
|
logScope.scope("replication"),
|
|
9616
10551
|
logScope.scope("log"),
|
|
@@ -9692,14 +10627,22 @@ let SharedLog = (() => {
|
|
|
9692
10627
|
}
|
|
9693
10628
|
let directoryProviders = [];
|
|
9694
10629
|
try {
|
|
9695
|
-
|
|
9696
|
-
|
|
9697
|
-
|
|
9698
|
-
|
|
9699
|
-
|
|
9700
|
-
|
|
9701
|
-
|
|
9702
|
-
|
|
10630
|
+
const query = (namespace) => fanoutService?.queryProviders(namespace, {
|
|
10631
|
+
want: maxPeers,
|
|
10632
|
+
timeoutMs: 2_000,
|
|
10633
|
+
queryTimeoutMs: 500,
|
|
10634
|
+
bootstrapMaxPeers: 2,
|
|
10635
|
+
signal: opts?.signal,
|
|
10636
|
+
}) ?? Promise.resolve([]);
|
|
10637
|
+
const results = await Promise.allSettled([
|
|
10638
|
+
query(blockProviderNamespace(cid)),
|
|
10639
|
+
query(logProviderNamespace),
|
|
10640
|
+
]);
|
|
10641
|
+
for (const result of results) {
|
|
10642
|
+
if (result.status === "fulfilled") {
|
|
10643
|
+
directoryProviders.push(...result.value);
|
|
10644
|
+
}
|
|
10645
|
+
}
|
|
9703
10646
|
}
|
|
9704
10647
|
catch {
|
|
9705
10648
|
// Ignore discovery failures; local evidence remains usable.
|
|
@@ -9707,9 +10650,7 @@ let SharedLog = (() => {
|
|
|
9707
10650
|
const selected = [];
|
|
9708
10651
|
const selectedSet = new Set();
|
|
9709
10652
|
const add = (peer) => {
|
|
9710
|
-
if (!peer ||
|
|
9711
|
-
selectedSet.has(peer) ||
|
|
9712
|
-
selected.length >= maxPeers) {
|
|
10653
|
+
if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
|
|
9713
10654
|
return;
|
|
9714
10655
|
}
|
|
9715
10656
|
selectedSet.add(peer);
|
|
@@ -9723,27 +10664,46 @@ let SharedLog = (() => {
|
|
|
9723
10664
|
return selected;
|
|
9724
10665
|
},
|
|
9725
10666
|
watchProviders: fanoutService
|
|
9726
|
-
? (cid, opts) =>
|
|
9727
|
-
|
|
9728
|
-
|
|
9729
|
-
|
|
9730
|
-
|
|
9731
|
-
|
|
9732
|
-
|
|
9733
|
-
|
|
10667
|
+
? (cid, opts) => {
|
|
10668
|
+
const watch = (namespace) => fanoutService.watchProviders(namespace, {
|
|
10669
|
+
signal: opts.signal,
|
|
10670
|
+
want: 8,
|
|
10671
|
+
ttlMs: 10_000,
|
|
10672
|
+
renewIntervalMs: 5_000,
|
|
10673
|
+
bootstrapMaxPeers: 2,
|
|
10674
|
+
onProviders: (providers) => opts.onProviders(providers.map((provider) => provider.hash)),
|
|
10675
|
+
});
|
|
10676
|
+
const cidWatch = watch(blockProviderNamespace(cid));
|
|
10677
|
+
try {
|
|
10678
|
+
const logWatch = watch(logProviderNamespace);
|
|
10679
|
+
return {
|
|
10680
|
+
close: () => {
|
|
10681
|
+
cidWatch.close();
|
|
10682
|
+
logWatch.close();
|
|
10683
|
+
},
|
|
10684
|
+
};
|
|
10685
|
+
}
|
|
10686
|
+
catch (error) {
|
|
10687
|
+
cidWatch.close();
|
|
10688
|
+
throw error;
|
|
10689
|
+
}
|
|
10690
|
+
}
|
|
9734
10691
|
: undefined,
|
|
9735
|
-
onPut: fanoutService
|
|
9736
|
-
|
|
9737
|
-
|
|
10692
|
+
onPut: fanoutService ? announceBlockProvider : undefined,
|
|
10693
|
+
onPutMany: fanoutService
|
|
10694
|
+
? (cids) => {
|
|
10695
|
+
// A renewable log-wide provider lease makes every CID in a
|
|
10696
|
+
// stored batch discoverable to current readers. Retain the
|
|
10697
|
+
// per-CID directory publications for released readers that only
|
|
10698
|
+
// know the legacy namespace; the bounded workers avoid creating
|
|
10699
|
+
// one in-flight promise per block.
|
|
9738
10700
|
try {
|
|
9739
|
-
|
|
9740
|
-
ttlMs: 120_000,
|
|
9741
|
-
bootstrapMaxPeers: 2,
|
|
9742
|
-
});
|
|
10701
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
9743
10702
|
}
|
|
9744
10703
|
catch {
|
|
9745
10704
|
// ignore announce failures
|
|
9746
10705
|
}
|
|
10706
|
+
return announceBlockProviders(cids);
|
|
9747
10707
|
}
|
|
9748
10708
|
: undefined,
|
|
9749
10709
|
});
|
|
@@ -9936,6 +10896,7 @@ let SharedLog = (() => {
|
|
|
9936
10896
|
},
|
|
9937
10897
|
indexer: logIndex,
|
|
9938
10898
|
});
|
|
10899
|
+
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
9939
10900
|
try {
|
|
9940
10901
|
const recovered = await this.recoverNativeStrictDurableTransactionIntent();
|
|
9941
10902
|
if (recovered) {
|
|
@@ -10542,8 +11503,7 @@ let SharedLog = (() => {
|
|
|
10542
11503
|
const subscriberDiscoveryPromise = this._getTopicSubscribers(this.topic);
|
|
10543
11504
|
const existingSubscribersPromise = this.node.services.pubsub.getSubscribers(this.topic);
|
|
10544
11505
|
// We do this here, because these calls requires this.closed == false
|
|
10545
|
-
void this.pruneOfflineReplicators()
|
|
10546
|
-
.catch((error) => {
|
|
11506
|
+
void this.pruneOfflineReplicators().catch((error) => {
|
|
10547
11507
|
if (isNotStartedError(error)) {
|
|
10548
11508
|
return;
|
|
10549
11509
|
}
|
|
@@ -11920,6 +12880,9 @@ let SharedLog = (() => {
|
|
|
11920
12880
|
this._peerSyncCapabilities?.clear();
|
|
11921
12881
|
this._peerSyncCapabilitySessions?.clear();
|
|
11922
12882
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
12883
|
+
this._persistedReceiptStorage = undefined;
|
|
12884
|
+
this._persistedReceiptRequestsInFlight?.clear();
|
|
12885
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
11923
12886
|
this._v2Receive?.clearForClose();
|
|
11924
12887
|
this._v2Send?.clearForClose();
|
|
11925
12888
|
this._liveRawGossipBatches?.clear();
|
|
@@ -12575,7 +13538,6 @@ let SharedLog = (() => {
|
|
|
12575
13538
|
preparedSelection: rawPreparedReceiveSelectionValue,
|
|
12576
13539
|
};
|
|
12577
13540
|
}
|
|
12578
|
-
// Callback for receiving a message from the network
|
|
12579
13541
|
async onMessage(msg, context) {
|
|
12580
13542
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
12581
13543
|
? msg
|
|
@@ -12637,6 +13599,7 @@ let SharedLog = (() => {
|
|
|
12637
13599
|
}
|
|
12638
13600
|
if (!context.from.equals(this.node.identity.publicKey) &&
|
|
12639
13601
|
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
13602
|
+
!(msg instanceof RequestPersistedEntriesV1) &&
|
|
12640
13603
|
!isReplicationInfoV2Message(msg)) {
|
|
12641
13604
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
12642
13605
|
}
|
|
@@ -13980,7 +14943,10 @@ let SharedLog = (() => {
|
|
|
13980
14943
|
};
|
|
13981
14944
|
// The prelude already threw when `context.from` was missing.
|
|
13982
14945
|
const laneRequestContext = context;
|
|
13983
|
-
if (msg instanceof
|
|
14946
|
+
if (msg instanceof RequestPersistedEntriesV1) {
|
|
14947
|
+
return await this.handleRequestPersistedEntriesV1(msg, laneRequestContext, lane);
|
|
14948
|
+
}
|
|
14949
|
+
else if (msg instanceof RequestIPruneV2) {
|
|
13984
14950
|
await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
|
|
13985
14951
|
}
|
|
13986
14952
|
else if (msg instanceof ResponseIPruneV2) {
|
|
@@ -14173,6 +15139,221 @@ let SharedLog = (() => {
|
|
|
14173
15139
|
// `onMessage`'s shared catch/finally envelope (error classification,
|
|
14174
15140
|
// wire-stash release, lease release, durable-poison recheck).
|
|
14175
15141
|
// -----------------------------------------------------------------
|
|
15142
|
+
isPersistedReceiptRequestSessionCurrent(request, context, lane) {
|
|
15143
|
+
const session = lane.session;
|
|
15144
|
+
return (!!this._persistedReceiptStorage &&
|
|
15145
|
+
!context.from.equals(this.node.identity.publicKey) &&
|
|
15146
|
+
request.expectedReceiverSession === this.ownTransportSession() &&
|
|
15147
|
+
session !== null &&
|
|
15148
|
+
session.phase === "open" &&
|
|
15149
|
+
this._peerSessions.isCurrent(lane.fromHash, session) &&
|
|
15150
|
+
!this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
|
|
15151
|
+
this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
|
|
15152
|
+
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
15153
|
+
context.message.header.session &&
|
|
15154
|
+
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
15155
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController));
|
|
15156
|
+
}
|
|
15157
|
+
admitPersistedReceiptIngress(peer, transportSession, hashCount, now = Date.now()) {
|
|
15158
|
+
// Charge malformed empty/oversized vectors too. The request was already
|
|
15159
|
+
// decoded by the transport, so letting an invalid shape bypass this bucket
|
|
15160
|
+
// would leave an authenticated peer with an unmetered validation/logging
|
|
15161
|
+
// path. Clamp only the accounting cost; shape validation still rejects the
|
|
15162
|
+
// request below.
|
|
15163
|
+
const hashCost = Math.max(1, Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount));
|
|
15164
|
+
let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
|
|
15165
|
+
if (!nodeBudget) {
|
|
15166
|
+
nodeBudget = {
|
|
15167
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
15168
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
15169
|
+
refilledAt: now,
|
|
15170
|
+
peerSessions: new Map(),
|
|
15171
|
+
};
|
|
15172
|
+
persistedReceiptIngressBudgets.set(this.node, nodeBudget);
|
|
15173
|
+
}
|
|
15174
|
+
refillPersistedReceiptIngressBucket(nodeBudget, now, PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND);
|
|
15175
|
+
const peerSessionKey = `${peer}\0${transportSession}`;
|
|
15176
|
+
let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
|
|
15177
|
+
if (!peerBudget) {
|
|
15178
|
+
while (nodeBudget.peerSessions.size >=
|
|
15179
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS) {
|
|
15180
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
15181
|
+
if (oldest === undefined)
|
|
15182
|
+
break;
|
|
15183
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
15184
|
+
}
|
|
15185
|
+
peerBudget = {
|
|
15186
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
15187
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
15188
|
+
refilledAt: now,
|
|
15189
|
+
};
|
|
15190
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
15191
|
+
}
|
|
15192
|
+
else {
|
|
15193
|
+
// Refresh insertion order so bounded eviction prefers inactive sessions.
|
|
15194
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
15195
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
15196
|
+
}
|
|
15197
|
+
refillPersistedReceiptIngressBucket(peerBudget, now, PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY, PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND, PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND);
|
|
15198
|
+
if (nodeBudget.requestTokens < 1 ||
|
|
15199
|
+
nodeBudget.hashTokens < hashCost ||
|
|
15200
|
+
peerBudget.requestTokens < 1 ||
|
|
15201
|
+
peerBudget.hashTokens < hashCost) {
|
|
15202
|
+
return false;
|
|
15203
|
+
}
|
|
15204
|
+
nodeBudget.requestTokens -= 1;
|
|
15205
|
+
nodeBudget.hashTokens -= hashCost;
|
|
15206
|
+
peerBudget.requestTokens -= 1;
|
|
15207
|
+
peerBudget.hashTokens -= hashCost;
|
|
15208
|
+
return true;
|
|
15209
|
+
}
|
|
15210
|
+
async handleRequestPersistedEntriesV1(request, context, lane) {
|
|
15211
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15212
|
+
return undefined;
|
|
15213
|
+
}
|
|
15214
|
+
if (!this.admitPersistedReceiptIngress(lane.fromHash, context.message.header.session, request.hashes.length)) {
|
|
15215
|
+
return undefined;
|
|
15216
|
+
}
|
|
15217
|
+
// Charge the request before shape validation and attacker-controlled CID
|
|
15218
|
+
// parsing. Invalid shapes are rejected quietly here so they cannot turn the
|
|
15219
|
+
// outer receive error logger into a post-budget work amplifier.
|
|
15220
|
+
try {
|
|
15221
|
+
this.validatePersistedReceiptRequestShape(request);
|
|
15222
|
+
}
|
|
15223
|
+
catch {
|
|
15224
|
+
return undefined;
|
|
15225
|
+
}
|
|
15226
|
+
if (!this.hasValidPersistedReceiptHashes(request)) {
|
|
15227
|
+
return undefined;
|
|
15228
|
+
}
|
|
15229
|
+
const peerInFlight = this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
|
|
15230
|
+
if (peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
|
|
15231
|
+
this._persistedReceiptRequestsInFlightTotal >=
|
|
15232
|
+
MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL) {
|
|
15233
|
+
return undefined;
|
|
15234
|
+
}
|
|
15235
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
|
|
15236
|
+
this._persistedReceiptRequestsInFlightTotal++;
|
|
15237
|
+
try {
|
|
15238
|
+
// A positive receipt requires the block itself. Reject entirely absent
|
|
15239
|
+
// batches before the serialized mutation lane and, critically, before a
|
|
15240
|
+
// request can force unrelated pending index/coordinate journals durable.
|
|
15241
|
+
// A concurrent put can yield a harmless false negative: the sender retries.
|
|
15242
|
+
const blockPresence = await this.remoteBlocks.localStore.hasMany(request.hashes);
|
|
15243
|
+
const presentBlockHashes = request.hashes.filter((_hash, index) => blockPresence[index] === true);
|
|
15244
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15245
|
+
return undefined;
|
|
15246
|
+
}
|
|
15247
|
+
if (presentBlockHashes.length === 0) {
|
|
15248
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15249
|
+
}
|
|
15250
|
+
this._liveness.markReplicatorActivity(lane.fromHash);
|
|
15251
|
+
return await this.withReplicationRangeMutationQueue(async () => {
|
|
15252
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15253
|
+
return undefined;
|
|
15254
|
+
}
|
|
15255
|
+
this.throwIfNativeDurableCommitFailed();
|
|
15256
|
+
const storage = this.resolvePersistedReceiptStorage();
|
|
15257
|
+
if (!storage) {
|
|
15258
|
+
return undefined;
|
|
15259
|
+
}
|
|
15260
|
+
await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
|
|
15261
|
+
await this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
15262
|
+
const candidates = new Map(await Promise.all(presentBlockHashes.map(async (hash) => {
|
|
15263
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
15264
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15265
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15266
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15267
|
+
]);
|
|
15268
|
+
return [hash, { blockPresent, lowerRow, coordinate }];
|
|
15269
|
+
})));
|
|
15270
|
+
const presentHashes = presentBlockHashes.filter((hash) => {
|
|
15271
|
+
const candidate = candidates.get(hash);
|
|
15272
|
+
return (candidate.blockPresent &&
|
|
15273
|
+
candidate.lowerRow != null &&
|
|
15274
|
+
candidate.coordinate != null);
|
|
15275
|
+
});
|
|
15276
|
+
if (presentHashes.length === 0) {
|
|
15277
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15278
|
+
}
|
|
15279
|
+
await Promise.all([...new Set([storage.block, storage.lower, storage.coordinate])].map((store) => store.barrier()));
|
|
15280
|
+
this.throwIfNativeDurableCommitFailed();
|
|
15281
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15282
|
+
return undefined;
|
|
15283
|
+
}
|
|
15284
|
+
const ownershipRevision = this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
15285
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
15286
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15287
|
+
}
|
|
15288
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
15289
|
+
const confirmed = [];
|
|
15290
|
+
for (const hash of presentHashes) {
|
|
15291
|
+
const candidate = candidates.get(hash);
|
|
15292
|
+
if (!candidate.blockPresent ||
|
|
15293
|
+
!candidate.lowerRow ||
|
|
15294
|
+
!candidate.coordinate ||
|
|
15295
|
+
this._checkedPrune.hasActiveWork(hash) ||
|
|
15296
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane) ||
|
|
15297
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
15298
|
+
continue;
|
|
15299
|
+
}
|
|
15300
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
15301
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15302
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15303
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15304
|
+
]);
|
|
15305
|
+
if (!blockPresent || !lowerRow || !coordinate) {
|
|
15306
|
+
continue;
|
|
15307
|
+
}
|
|
15308
|
+
const replicas = decodeReplicas(coordinate).getValue(this);
|
|
15309
|
+
const leaders = await this.findLeadersFromEntry(coordinate, replicas, { freshLeaderPlan: true }, lane.ownershipLifecycleController);
|
|
15310
|
+
if (!leaders.has(selfHash)) {
|
|
15311
|
+
continue;
|
|
15312
|
+
}
|
|
15313
|
+
if (!this._checkedPrune.hasActiveWork(hash) &&
|
|
15314
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
|
|
15315
|
+
this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15316
|
+
confirmed.push(hash);
|
|
15317
|
+
}
|
|
15318
|
+
}
|
|
15319
|
+
if (confirmed.length === 0 ||
|
|
15320
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
15321
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15322
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15323
|
+
}
|
|
15324
|
+
const finalRows = await Promise.all(confirmed.map(async (hash) => {
|
|
15325
|
+
const [block, lower, coordinate] = await Promise.all([
|
|
15326
|
+
this.remoteBlocks.localStore.has(hash),
|
|
15327
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
15328
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
15329
|
+
]);
|
|
15330
|
+
return { hash, block, lower, coordinate };
|
|
15331
|
+
}));
|
|
15332
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
15333
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
15334
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
15335
|
+
}
|
|
15336
|
+
return new ConfirmEntriesMessage({
|
|
15337
|
+
hashes: finalRows
|
|
15338
|
+
.filter((row) => row.block &&
|
|
15339
|
+
row.lower &&
|
|
15340
|
+
row.coordinate &&
|
|
15341
|
+
!this._checkedPrune.hasActiveWork(row.hash))
|
|
15342
|
+
.map((row) => row.hash),
|
|
15343
|
+
});
|
|
15344
|
+
}, lane.ownershipLifecycleController);
|
|
15345
|
+
}
|
|
15346
|
+
finally {
|
|
15347
|
+
const remaining = (this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
|
|
15348
|
+
if (remaining <= 0) {
|
|
15349
|
+
this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
|
|
15350
|
+
}
|
|
15351
|
+
else {
|
|
15352
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
|
|
15353
|
+
}
|
|
15354
|
+
this._persistedReceiptRequestsInFlightTotal = Math.max(0, this._persistedReceiptRequestsInFlightTotal - 1);
|
|
15355
|
+
}
|
|
15356
|
+
}
|
|
14176
15357
|
async handleRequestIPruneV2(msg, context, lane) {
|
|
14177
15358
|
const syncProfile = lane.syncProfile;
|
|
14178
15359
|
const receiveReplicationLifecycleController = lane.lifecycleController;
|