@peerbit/shared-log 16.0.14 → 16.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/coordinate-persistence.d.ts +10 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +18 -0
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/errors.d.ts +11 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +23 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +6 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +6 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +60 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1315 -141
- package/dist/src/index.js.map +1 -1
- package/dist/src/native-write-through-block-store.d.ts +4 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -1
- package/dist/src/native-write-through-block-store.js +13 -0
- package/dist/src/native-write-through-block-store.js.map +1 -1
- package/dist/src/sync/simple.d.ts +15 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +44 -0
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +19 -19
- package/src/coordinate-persistence.ts +21 -0
- package/src/errors.ts +35 -0
- package/src/exchange-heads.ts +7 -0
- package/src/index.ts +1988 -261
- package/src/native-write-through-block-store.ts +14 -0
- package/src/sync/simple.ts +22 -0
package/src/index.ts
CHANGED
|
@@ -115,6 +115,7 @@ import {
|
|
|
115
115
|
AbortError,
|
|
116
116
|
TimeoutError,
|
|
117
117
|
debounceFixedInterval,
|
|
118
|
+
delay,
|
|
118
119
|
waitFor,
|
|
119
120
|
} from "@peerbit/time";
|
|
120
121
|
import pDefer, { type DeferredPromise } from "p-defer";
|
|
@@ -147,6 +148,7 @@ import {
|
|
|
147
148
|
CompatibilityModeRetiredError,
|
|
148
149
|
NativeDurableCommitError,
|
|
149
150
|
NoPeersError,
|
|
151
|
+
PersistedDeliveryError,
|
|
150
152
|
isNotStartedError,
|
|
151
153
|
} from "./errors.js";
|
|
152
154
|
import {
|
|
@@ -162,6 +164,7 @@ import {
|
|
|
162
164
|
RequestIPruneV2,
|
|
163
165
|
ResponseIPrune,
|
|
164
166
|
ResponseIPruneV2,
|
|
167
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS,
|
|
165
168
|
SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
|
|
166
169
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY,
|
|
167
170
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM,
|
|
@@ -306,6 +309,7 @@ import {
|
|
|
306
309
|
import {
|
|
307
310
|
ConfirmEntriesMessage,
|
|
308
311
|
RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
|
|
312
|
+
RequestPersistedEntriesV1,
|
|
309
313
|
SYNC_MESSAGE_PRIORITY,
|
|
310
314
|
SimpleSyncronizer,
|
|
311
315
|
} from "./sync/simple.js";
|
|
@@ -449,6 +453,7 @@ export {
|
|
|
449
453
|
CompatibilityModeRetiredError,
|
|
450
454
|
NativeDurableCommitError,
|
|
451
455
|
NoPeersError,
|
|
456
|
+
PersistedDeliveryError,
|
|
452
457
|
};
|
|
453
458
|
export { MAX_U32, MAX_U64, type NumberFromType };
|
|
454
459
|
export type {
|
|
@@ -551,6 +556,7 @@ type LeaderMap = Map<string, { intersecting: boolean }>;
|
|
|
551
556
|
type LeaderSelectionOptions<R extends "u32" | "u64"> = {
|
|
552
557
|
roleAge?: number;
|
|
553
558
|
candidates?: Iterable<string>;
|
|
559
|
+
freshLeaderPlan?: boolean;
|
|
554
560
|
onLeader?: (key: string) => void;
|
|
555
561
|
persist?:
|
|
556
562
|
| {
|
|
@@ -698,6 +704,10 @@ type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
|
|
|
698
704
|
};
|
|
699
705
|
};
|
|
700
706
|
|
|
707
|
+
type PersistedDeliveryPlanningEntry<T, R extends "u32" | "u64"> =
|
|
708
|
+
| ShallowOrFullEntry<T>
|
|
709
|
+
| EntryReplicated<R>;
|
|
710
|
+
|
|
701
711
|
type NativeBackboneSimpleDocumentProjectionPlan = {
|
|
702
712
|
documentVariantType?: "u8" | "string";
|
|
703
713
|
documentVariantValue?: string;
|
|
@@ -908,10 +918,15 @@ const nativeStrictDurableTransactionJournalRecordBytes = (
|
|
|
908
918
|
return new TextEncoder().encode(JSON.stringify(record));
|
|
909
919
|
};
|
|
910
920
|
|
|
921
|
+
type TrustedLocalCommitEvidence = {
|
|
922
|
+
committedHashes: Set<string>;
|
|
923
|
+
};
|
|
924
|
+
|
|
911
925
|
type PreparedPayloadCommitOnlyProperties =
|
|
912
926
|
NativeBackboneDocumentCommitOptions & {
|
|
913
927
|
skipMissingNextJoin?: boolean;
|
|
914
928
|
resolveTrimmedEntries?: boolean;
|
|
929
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
915
930
|
};
|
|
916
931
|
|
|
917
932
|
type PreparedPayloadsManyIndependentProperties<T> = {
|
|
@@ -920,6 +935,7 @@ type PreparedPayloadsManyIndependentProperties<T> = {
|
|
|
920
935
|
nexts?: ShallowOrFullEntry<T>[][];
|
|
921
936
|
nativeBackboneDocumentIndexes?: NativeBackboneDocumentIndexCommitInput[];
|
|
922
937
|
retainMaterializationBytes?: boolean;
|
|
938
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
923
939
|
};
|
|
924
940
|
|
|
925
941
|
type PreparedPayloadCommitOnlyResult<T, R extends "u32" | "u64"> = {
|
|
@@ -1681,10 +1697,23 @@ export type Args<
|
|
|
1681
1697
|
: "u32",
|
|
1682
1698
|
> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D, R>;
|
|
1683
1699
|
|
|
1684
|
-
|
|
1700
|
+
/**
|
|
1701
|
+
* `persisted` waits, after the local commit, for `minAcks` distinct remote
|
|
1702
|
+
* leaders that advertised crash-safe receipt support. Each receipt proves that
|
|
1703
|
+
* the exact block, lower-log row, and replication-coordinate row crossed that
|
|
1704
|
+
* peer's storage barriers at the receipt instant. It is cooperative-peer
|
|
1705
|
+
* evidence, not a Byzantine proof or a promise that the peer will retain the
|
|
1706
|
+
* entry forever. Only current capable leaders count.
|
|
1707
|
+
*/
|
|
1708
|
+
export type DeliveryReliability = "ack" | "best-effort" | "persisted";
|
|
1685
1709
|
|
|
1686
1710
|
export type DeliveryOptions = {
|
|
1687
1711
|
reliability?: DeliveryReliability;
|
|
1712
|
+
/**
|
|
1713
|
+
* Required for persisted delivery; counts distinct current remote leaders.
|
|
1714
|
+
* This does not increase the entry's replication degree, so the configured
|
|
1715
|
+
* replication must make at least this many capable remote leaders eligible.
|
|
1716
|
+
*/
|
|
1688
1717
|
minAcks?: number;
|
|
1689
1718
|
requireRecipients?: boolean;
|
|
1690
1719
|
/**
|
|
@@ -1692,10 +1721,131 @@ export type DeliveryOptions = {
|
|
|
1692
1721
|
* its control lane, so this only changes the direct/fallback RPC path.
|
|
1693
1722
|
*/
|
|
1694
1723
|
priority?: number;
|
|
1724
|
+
/**
|
|
1725
|
+
* Overall delivery deadline in milliseconds. For persisted delivery it
|
|
1726
|
+
* starts after the local append has returned and includes leader planning,
|
|
1727
|
+
* transfer, receipt requests, and final ownership/session validation. The
|
|
1728
|
+
* omitted persisted default is 10 seconds plus one admission-attempt budget
|
|
1729
|
+
* per transfer chunk and the minimum receipt sender-pacing time implied by
|
|
1730
|
+
* the batch size. An explicit timeout remains exact.
|
|
1731
|
+
*/
|
|
1695
1732
|
timeout?: number;
|
|
1696
1733
|
signal?: AbortSignal;
|
|
1697
1734
|
};
|
|
1698
1735
|
|
|
1736
|
+
type CrashSafeStorageBarrier = {
|
|
1737
|
+
readonly crashSafe: true;
|
|
1738
|
+
barrier(): MaybePromise<void>;
|
|
1739
|
+
};
|
|
1740
|
+
|
|
1741
|
+
type PersistedReceiptStorage = {
|
|
1742
|
+
block: CrashSafeStorageBarrier;
|
|
1743
|
+
lower: CrashSafeStorageBarrier;
|
|
1744
|
+
coordinate: CrashSafeStorageBarrier;
|
|
1745
|
+
};
|
|
1746
|
+
|
|
1747
|
+
type PersistedDeliveryDeadline = {
|
|
1748
|
+
deadline: number;
|
|
1749
|
+
signal: AbortSignal;
|
|
1750
|
+
dispose(): void;
|
|
1751
|
+
};
|
|
1752
|
+
|
|
1753
|
+
const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
|
|
1754
|
+
const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
|
|
1755
|
+
const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
|
|
1756
|
+
const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
|
|
1757
|
+
const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
|
|
1758
|
+
const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
|
|
1759
|
+
const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
1760
|
+
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
1761
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
1762
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
1763
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
1764
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
1765
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
1766
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
|
|
1767
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
|
|
1768
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
|
|
1769
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
|
|
1770
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
|
|
1771
|
+
const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
|
|
1772
|
+
|
|
1773
|
+
type PersistedReceiptIngressBucket = {
|
|
1774
|
+
requestTokens: number;
|
|
1775
|
+
hashTokens: number;
|
|
1776
|
+
refilledAt: number;
|
|
1777
|
+
};
|
|
1778
|
+
|
|
1779
|
+
type PersistedReceiptNodeIngressBudget = PersistedReceiptIngressBucket & {
|
|
1780
|
+
peerSessions: Map<string, PersistedReceiptIngressBucket>;
|
|
1781
|
+
};
|
|
1782
|
+
|
|
1783
|
+
type PersistedReceiptNodeEgressBudget = {
|
|
1784
|
+
peerSessions: Map<string, PersistedReceiptIngressBucket>;
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
const persistedReceiptIngressBudgets = new WeakMap<
|
|
1788
|
+
object,
|
|
1789
|
+
PersistedReceiptNodeIngressBudget
|
|
1790
|
+
>();
|
|
1791
|
+
|
|
1792
|
+
// Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
|
|
1793
|
+
// on the Peerbit node (rather than one SharedLog) prevents independent programs
|
|
1794
|
+
// from silently overrunning the same remote receiver together.
|
|
1795
|
+
const persistedReceiptEgressBudgets = new WeakMap<
|
|
1796
|
+
object,
|
|
1797
|
+
PersistedReceiptNodeEgressBudget
|
|
1798
|
+
>();
|
|
1799
|
+
|
|
1800
|
+
const refillPersistedReceiptIngressBucket = (
|
|
1801
|
+
bucket: PersistedReceiptIngressBucket,
|
|
1802
|
+
now: number,
|
|
1803
|
+
requestCapacity: number,
|
|
1804
|
+
hashCapacity: number,
|
|
1805
|
+
requestsPerSecond: number,
|
|
1806
|
+
hashesPerSecond: number,
|
|
1807
|
+
) => {
|
|
1808
|
+
// Never move the refill watermark backwards. Date.now() can jump after a
|
|
1809
|
+
// clock correction; accepting that earlier timestamp would grant the same
|
|
1810
|
+
// elapsed interval again on the next request.
|
|
1811
|
+
if (now <= bucket.refilledAt) {
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
|
|
1815
|
+
if (elapsedSeconds > 0) {
|
|
1816
|
+
bucket.requestTokens = Math.min(
|
|
1817
|
+
requestCapacity,
|
|
1818
|
+
bucket.requestTokens + elapsedSeconds * requestsPerSecond,
|
|
1819
|
+
);
|
|
1820
|
+
bucket.hashTokens = Math.min(
|
|
1821
|
+
hashCapacity,
|
|
1822
|
+
bucket.hashTokens + elapsedSeconds * hashesPerSecond,
|
|
1823
|
+
);
|
|
1824
|
+
}
|
|
1825
|
+
bucket.refilledAt = now;
|
|
1826
|
+
};
|
|
1827
|
+
|
|
1828
|
+
const persistedReceiptPacingFloorMs = (hashCount: number): number => {
|
|
1829
|
+
const boundedHashCount = Math.max(0, Math.floor(hashCount));
|
|
1830
|
+
const requestCount = Math.ceil(
|
|
1831
|
+
boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE,
|
|
1832
|
+
);
|
|
1833
|
+
return Math.max(
|
|
1834
|
+
0,
|
|
1835
|
+
((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
|
|
1836
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
1837
|
+
1_000,
|
|
1838
|
+
((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
|
|
1839
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
1840
|
+
1_000,
|
|
1841
|
+
);
|
|
1842
|
+
};
|
|
1843
|
+
|
|
1844
|
+
const persistedTransferAdmissionBudgetMs = (hashCount: number): number =>
|
|
1845
|
+
Math.ceil(
|
|
1846
|
+
Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
1847
|
+
) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
1848
|
+
|
|
1699
1849
|
export type SharedLogFanoutOptions = {
|
|
1700
1850
|
root?: string;
|
|
1701
1851
|
channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
|
|
@@ -1709,6 +1859,19 @@ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
|
|
|
1709
1859
|
|
|
1710
1860
|
type TrustedLogAppendOptions<T> = AppendOptions<T> & {
|
|
1711
1861
|
__peerbitCanAppendAlreadyValidated?: boolean;
|
|
1862
|
+
__peerbitOnLocalCommit?: (hashes: readonly string[]) => void;
|
|
1863
|
+
};
|
|
1864
|
+
|
|
1865
|
+
const attachTrustedLocalCommitEvidence = <T>(
|
|
1866
|
+
options: AppendOptions<T>,
|
|
1867
|
+
evidence: TrustedLocalCommitEvidence | undefined,
|
|
1868
|
+
): void => {
|
|
1869
|
+
if (!evidence) return;
|
|
1870
|
+
(options as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (hashes) => {
|
|
1871
|
+
for (const hash of hashes) {
|
|
1872
|
+
evidence.committedHashes.add(hash);
|
|
1873
|
+
}
|
|
1874
|
+
};
|
|
1712
1875
|
};
|
|
1713
1876
|
|
|
1714
1877
|
export type SharedAppendOptions<T> =
|
|
@@ -3019,6 +3182,37 @@ export class SharedLog<
|
|
|
3019
3182
|
}
|
|
3020
3183
|
}
|
|
3021
3184
|
|
|
3185
|
+
private async finishCommittedNativeStrictDurableTransaction<TValue>(
|
|
3186
|
+
handle: NativeStrictDurableTransactionHandle | undefined,
|
|
3187
|
+
finish: () => MaybePromise<TValue>,
|
|
3188
|
+
shouldWarnOnRetirementFailure: () => boolean = () => true,
|
|
3189
|
+
): Promise<TValue> {
|
|
3190
|
+
let finishResult: TValue | undefined;
|
|
3191
|
+
let finishError: unknown;
|
|
3192
|
+
let finishFailed = false;
|
|
3193
|
+
try {
|
|
3194
|
+
finishResult = await finish();
|
|
3195
|
+
} catch (error) {
|
|
3196
|
+
finishError = error;
|
|
3197
|
+
finishFailed = true;
|
|
3198
|
+
}
|
|
3199
|
+
try {
|
|
3200
|
+
await this.completeNativeStrictDurableTransaction(handle);
|
|
3201
|
+
} catch (error) {
|
|
3202
|
+
// The acknowledged lower commit is already final. Completion poisons and
|
|
3203
|
+
// releases the in-memory transaction before throwing, so preserve an
|
|
3204
|
+
// earlier post-commit failure while leaving recovery to retire the intent.
|
|
3205
|
+
if (!finishFailed && !shouldWarnOnRetirementFailure()) {
|
|
3206
|
+
throw error;
|
|
3207
|
+
}
|
|
3208
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
3209
|
+
}
|
|
3210
|
+
if (finishFailed) {
|
|
3211
|
+
throw finishError;
|
|
3212
|
+
}
|
|
3213
|
+
return finishResult as TValue;
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3022
3216
|
private releaseNativeStrictDurableTransaction(
|
|
3023
3217
|
handle: NativeStrictDurableTransactionHandle | undefined,
|
|
3024
3218
|
cause: unknown = new Error(
|
|
@@ -3504,6 +3698,9 @@ export class SharedLog<
|
|
|
3504
3698
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
3505
3699
|
private _peerSyncCapabilitySessions!: Map<string, bigint>;
|
|
3506
3700
|
private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
|
|
3701
|
+
private _persistedReceiptStorage?: PersistedReceiptStorage;
|
|
3702
|
+
private _persistedReceiptRequestsInFlight!: Map<string, number>;
|
|
3703
|
+
private _persistedReceiptRequestsInFlightTotal!: number;
|
|
3507
3704
|
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
3508
3705
|
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
3509
3706
|
// is hit). Only used when every recipient advertised raw capability.
|
|
@@ -3578,10 +3775,7 @@ export class SharedLog<
|
|
|
3578
3775
|
return new ReplicationInfoV2SendCoordinator<R>({
|
|
3579
3776
|
getRpc: () => this.rpc,
|
|
3580
3777
|
getSelfKey: () => this.node.identity.publicKey,
|
|
3581
|
-
getSenderTransportSession: () =>
|
|
3582
|
-
BigInt(
|
|
3583
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3584
|
-
),
|
|
3778
|
+
getSenderTransportSession: () => this.ownTransportSession(),
|
|
3585
3779
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
3586
3780
|
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3587
3781
|
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
@@ -3606,12 +3800,37 @@ export class SharedLog<
|
|
|
3606
3800
|
});
|
|
3607
3801
|
}
|
|
3608
3802
|
|
|
3803
|
+
private resolvePersistedReceiptStorage():
|
|
3804
|
+
| PersistedReceiptStorage
|
|
3805
|
+
| undefined {
|
|
3806
|
+
if (this.log.appendDurability !== "strict") {
|
|
3807
|
+
return undefined;
|
|
3808
|
+
}
|
|
3809
|
+
const block = this.remoteBlocks.crashSafeDurability;
|
|
3810
|
+
const lower = this.log.entryIndex.properties.index.crashSafeDurability;
|
|
3811
|
+
const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
|
|
3812
|
+
if (!block || !lower || !coordinate) {
|
|
3813
|
+
return undefined;
|
|
3814
|
+
}
|
|
3815
|
+
return { block, lower, coordinate };
|
|
3816
|
+
}
|
|
3817
|
+
|
|
3818
|
+
private ownTransportSession(): bigint {
|
|
3819
|
+
return BigInt(
|
|
3820
|
+
(this.node.services.pubsub as unknown as { session: number | bigint })
|
|
3821
|
+
.session,
|
|
3822
|
+
);
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3609
3825
|
private replicationInfoV2ReceiveCapabilities(): number {
|
|
3610
3826
|
return (
|
|
3611
3827
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
3612
3828
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
3613
3829
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
3614
3830
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
|
|
3831
|
+
(this._persistedReceiptStorage
|
|
3832
|
+
? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
|
|
3833
|
+
: 0) |
|
|
3615
3834
|
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
3616
3835
|
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
3617
3836
|
: 0)
|
|
@@ -3627,9 +3846,7 @@ export class SharedLog<
|
|
|
3627
3846
|
{ receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
|
|
3628
3847
|
> {
|
|
3629
3848
|
const peerHash = properties.target.hashcode();
|
|
3630
|
-
const receiverTransportSession =
|
|
3631
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3632
|
-
);
|
|
3849
|
+
const receiverTransportSession = this.ownTransportSession();
|
|
3633
3850
|
await this.rpc.send(
|
|
3634
3851
|
new SyncCapabilitiesMessage({
|
|
3635
3852
|
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
@@ -3655,9 +3872,7 @@ export class SharedLog<
|
|
|
3655
3872
|
) ||
|
|
3656
3873
|
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
3657
3874
|
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
3658
|
-
|
|
3659
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3660
|
-
) !== receiverTransportSession
|
|
3875
|
+
this.ownTransportSession() !== receiverTransportSession
|
|
3661
3876
|
) {
|
|
3662
3877
|
return undefined;
|
|
3663
3878
|
}
|
|
@@ -3670,10 +3885,7 @@ export class SharedLog<
|
|
|
3670
3885
|
private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
|
|
3671
3886
|
return new ReplicationInfoV2ReceiveCoordinator({
|
|
3672
3887
|
getSelfKey: () => this.node.identity.publicKey,
|
|
3673
|
-
getReceiverTransportSession: () =>
|
|
3674
|
-
BigInt(
|
|
3675
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3676
|
-
),
|
|
3888
|
+
getReceiverTransportSession: () => this.ownTransportSession(),
|
|
3677
3889
|
isClosed: () => this.closed,
|
|
3678
3890
|
isPeerSessionCurrent: (peerHash, peerSession) =>
|
|
3679
3891
|
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
@@ -3843,6 +4055,9 @@ export class SharedLog<
|
|
|
3843
4055
|
this._peerSyncCapabilities = new Map();
|
|
3844
4056
|
this._peerSyncCapabilitySessions = new Map();
|
|
3845
4057
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
4058
|
+
this._persistedReceiptStorage = undefined;
|
|
4059
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
4060
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
3846
4061
|
this._liveRawGossipBatches = new Map();
|
|
3847
4062
|
this._liveRawGossipFlushScheduled = false;
|
|
3848
4063
|
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
@@ -4012,6 +4227,14 @@ export class SharedLog<
|
|
|
4012
4227
|
this._fanoutChannel = undefined;
|
|
4013
4228
|
}
|
|
4014
4229
|
|
|
4230
|
+
private ensureLogProviderHandle(fanoutService: FanoutTree): void {
|
|
4231
|
+
if (this._providerHandle || this._closeController.signal.aborted) return;
|
|
4232
|
+
this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
|
|
4233
|
+
ttlMs: 120_000,
|
|
4234
|
+
announceIntervalMs: 60_000,
|
|
4235
|
+
});
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4015
4238
|
private async _onFanoutData(detail: FanoutTreeDataEvent) {
|
|
4016
4239
|
let envelope: FanoutEnvelope;
|
|
4017
4240
|
try {
|
|
@@ -4138,11 +4361,23 @@ export class SharedLog<
|
|
|
4138
4361
|
const reliability: DeliveryReliability = delivery.reliability ?? "ack";
|
|
4139
4362
|
const deliveryTimeout = delivery.timeout;
|
|
4140
4363
|
const deliverySignal = delivery.signal;
|
|
4141
|
-
const requireRecipients =
|
|
4364
|
+
const requireRecipients =
|
|
4365
|
+
reliability === "persisted" || delivery.requireRecipients === true;
|
|
4142
4366
|
const minAcks =
|
|
4143
4367
|
delivery.minAcks != null && Number.isFinite(delivery.minAcks)
|
|
4144
4368
|
? Math.max(0, Math.floor(delivery.minAcks))
|
|
4145
4369
|
: undefined;
|
|
4370
|
+
if (reliability === "persisted") {
|
|
4371
|
+
if (
|
|
4372
|
+
delivery.minAcks == null ||
|
|
4373
|
+
!Number.isSafeInteger(delivery.minAcks) ||
|
|
4374
|
+
delivery.minAcks <= 0
|
|
4375
|
+
) {
|
|
4376
|
+
throw new Error(
|
|
4377
|
+
'persisted delivery requires a positive explicit "minAcks"',
|
|
4378
|
+
);
|
|
4379
|
+
}
|
|
4380
|
+
}
|
|
4146
4381
|
|
|
4147
4382
|
const wrap =
|
|
4148
4383
|
deliveryTimeout == null && deliverySignal == null
|
|
@@ -4217,6 +4452,82 @@ export class SharedLog<
|
|
|
4217
4452
|
};
|
|
4218
4453
|
}
|
|
4219
4454
|
|
|
4455
|
+
private validatePersistedReceiptRequestShape(
|
|
4456
|
+
request: RequestPersistedEntriesV1,
|
|
4457
|
+
): void {
|
|
4458
|
+
if (
|
|
4459
|
+
request.hashes.length === 0 ||
|
|
4460
|
+
request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES
|
|
4461
|
+
) {
|
|
4462
|
+
throw new Error(
|
|
4463
|
+
`Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`,
|
|
4464
|
+
);
|
|
4465
|
+
}
|
|
4466
|
+
let bytes = 0;
|
|
4467
|
+
const encoder = new TextEncoder();
|
|
4468
|
+
for (const hash of request.hashes) {
|
|
4469
|
+
if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
4470
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
4471
|
+
}
|
|
4472
|
+
bytes += encoder.encode(hash).byteLength;
|
|
4473
|
+
if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
4474
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
4475
|
+
}
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
|
|
4479
|
+
private hasValidPersistedReceiptHashes(
|
|
4480
|
+
request: RequestPersistedEntriesV1,
|
|
4481
|
+
): boolean {
|
|
4482
|
+
const seen = new Set<string>();
|
|
4483
|
+
try {
|
|
4484
|
+
for (const hash of request.hashes) {
|
|
4485
|
+
if (seen.has(hash)) return false;
|
|
4486
|
+
cidifyString(hash);
|
|
4487
|
+
seen.add(hash);
|
|
4488
|
+
}
|
|
4489
|
+
} catch {
|
|
4490
|
+
return false;
|
|
4491
|
+
}
|
|
4492
|
+
return true;
|
|
4493
|
+
}
|
|
4494
|
+
|
|
4495
|
+
private getPersistedDeliveryOptions(
|
|
4496
|
+
options?: SharedAppendOptions<T>,
|
|
4497
|
+
): DeliveryOptions | undefined {
|
|
4498
|
+
const delivery = options?.delivery;
|
|
4499
|
+
if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
|
|
4500
|
+
return undefined;
|
|
4501
|
+
}
|
|
4502
|
+
const parsed = this._parseDeliveryOptions(options?.delivery);
|
|
4503
|
+
const target = (options as { target?: string } | undefined)?.target;
|
|
4504
|
+
if (target !== undefined && target !== "replicators") {
|
|
4505
|
+
throw new Error(
|
|
4506
|
+
'persisted delivery requires target="replicators" (or an omitted target)',
|
|
4507
|
+
);
|
|
4508
|
+
}
|
|
4509
|
+
if (
|
|
4510
|
+
parsed.delivery?.timeout != null &&
|
|
4511
|
+
(!Number.isFinite(parsed.delivery.timeout) ||
|
|
4512
|
+
parsed.delivery.timeout <= 0 ||
|
|
4513
|
+
parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)
|
|
4514
|
+
) {
|
|
4515
|
+
throw new Error(
|
|
4516
|
+
`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
|
|
4517
|
+
);
|
|
4518
|
+
}
|
|
4519
|
+
if (parsed.delivery?.signal?.aborted) {
|
|
4520
|
+
throw parsed.delivery.signal.reason ?? new AbortError();
|
|
4521
|
+
}
|
|
4522
|
+
return parsed.delivery!;
|
|
4523
|
+
}
|
|
4524
|
+
|
|
4525
|
+
private assertPersistedDeliveryOptions(
|
|
4526
|
+
options?: SharedAppendOptions<T>,
|
|
4527
|
+
): void {
|
|
4528
|
+
this.getPersistedDeliveryOptions(options);
|
|
4529
|
+
}
|
|
4530
|
+
|
|
4220
4531
|
private async _getSortedRouteHints(targetHash: string): Promise<RouteHint[]> {
|
|
4221
4532
|
const pubsub: any = this.node.services.pubsub as any;
|
|
4222
4533
|
const maybeHints = await pubsub?.getUnifiedRouteHints?.(
|
|
@@ -4631,6 +4942,7 @@ export class SharedLog<
|
|
|
4631
4942
|
plan: RawExchangeHeadSendPlan,
|
|
4632
4943
|
to: string[] | Set<string>,
|
|
4633
4944
|
options?: {
|
|
4945
|
+
acknowledge?: boolean;
|
|
4634
4946
|
priority?: number;
|
|
4635
4947
|
reserved?: Uint8Array;
|
|
4636
4948
|
signal?: AbortSignal;
|
|
@@ -4645,7 +4957,7 @@ export class SharedLog<
|
|
|
4645
4957
|
payload: Uint8Array,
|
|
4646
4958
|
properties: { topics: string[] },
|
|
4647
4959
|
options: {
|
|
4648
|
-
mode: SilentDelivery;
|
|
4960
|
+
mode: SilentDelivery | AcknowledgeDelivery;
|
|
4649
4961
|
priority?: number;
|
|
4650
4962
|
signal?: AbortSignal;
|
|
4651
4963
|
},
|
|
@@ -4733,7 +5045,9 @@ export class SharedLog<
|
|
|
4733
5045
|
item.payload,
|
|
4734
5046
|
{ topics: [topic] },
|
|
4735
5047
|
{
|
|
4736
|
-
mode:
|
|
5048
|
+
mode: options?.acknowledge
|
|
5049
|
+
? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
|
|
5050
|
+
: new SilentDelivery({ redundancy: 1, to: [...to] }),
|
|
4737
5051
|
priority: options?.priority,
|
|
4738
5052
|
signal: options?.signal,
|
|
4739
5053
|
},
|
|
@@ -4760,34 +5074,762 @@ export class SharedLog<
|
|
|
4760
5074
|
});
|
|
4761
5075
|
}
|
|
4762
5076
|
}
|
|
4763
|
-
return sentMessages;
|
|
4764
|
-
}
|
|
4765
|
-
|
|
4766
|
-
/**
|
|
4767
|
-
* `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
|
|
4768
|
-
* responses: resolves the head/reference plan like the TS raw path and
|
|
4769
|
-
* ships it fused when possible.
|
|
4770
|
-
*/
|
|
4771
|
-
private async trySendFusedRawExchangeHeads(
|
|
4772
|
-
hashes: string[],
|
|
4773
|
-
to: string[],
|
|
4774
|
-
options?: {
|
|
4775
|
-
|
|
4776
|
-
|
|
4777
|
-
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
5077
|
+
return sentMessages;
|
|
5078
|
+
}
|
|
5079
|
+
|
|
5080
|
+
/**
|
|
5081
|
+
* `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
|
|
5082
|
+
* responses: resolves the head/reference plan like the TS raw path and
|
|
5083
|
+
* ships it fused when possible.
|
|
5084
|
+
*/
|
|
5085
|
+
private async trySendFusedRawExchangeHeads(
|
|
5086
|
+
hashes: string[],
|
|
5087
|
+
to: string[],
|
|
5088
|
+
options?: {
|
|
5089
|
+
acknowledge?: boolean;
|
|
5090
|
+
priority?: number;
|
|
5091
|
+
reserved?: Uint8Array;
|
|
5092
|
+
signal?: AbortSignal;
|
|
5093
|
+
},
|
|
5094
|
+
): Promise<number | undefined> {
|
|
5095
|
+
if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
|
|
5096
|
+
return undefined;
|
|
5097
|
+
}
|
|
5098
|
+
const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
|
|
5099
|
+
if (!plan) {
|
|
5100
|
+
return undefined;
|
|
5101
|
+
}
|
|
5102
|
+
if (plan.hashes.length === 0) {
|
|
5103
|
+
return 0;
|
|
5104
|
+
}
|
|
5105
|
+
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
5106
|
+
}
|
|
5107
|
+
|
|
5108
|
+
private persistedReceiptPeerSession(
|
|
5109
|
+
peerHash: string,
|
|
5110
|
+
): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
|
|
5111
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5112
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
5113
|
+
if (
|
|
5114
|
+
capabilitySession == null ||
|
|
5115
|
+
!peerSession ||
|
|
5116
|
+
peerSession.phase !== "open" ||
|
|
5117
|
+
!this._peerSessions.isCurrent(peerHash, peerSession) ||
|
|
5118
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
5119
|
+
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
5120
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
|
|
5121
|
+
0
|
|
5122
|
+
) {
|
|
5123
|
+
return undefined;
|
|
5124
|
+
}
|
|
5125
|
+
return { capabilitySession, peerSession };
|
|
5126
|
+
}
|
|
5127
|
+
|
|
5128
|
+
private async waitPersistedReceiptRetry(
|
|
5129
|
+
signal: AbortSignal,
|
|
5130
|
+
ms: number,
|
|
5131
|
+
): Promise<void> {
|
|
5132
|
+
try {
|
|
5133
|
+
await delay(ms, { signal });
|
|
5134
|
+
} catch (error) {
|
|
5135
|
+
throw signal.aborted ? (signal.reason ?? error) : error;
|
|
5136
|
+
}
|
|
5137
|
+
}
|
|
5138
|
+
|
|
5139
|
+
private reservePersistedReceiptEgress(
|
|
5140
|
+
peer: string,
|
|
5141
|
+
capabilitySession: bigint,
|
|
5142
|
+
hashCount: number,
|
|
5143
|
+
now = Date.now(),
|
|
5144
|
+
): number {
|
|
5145
|
+
let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
|
|
5146
|
+
if (!nodeBudget) {
|
|
5147
|
+
nodeBudget = { peerSessions: new Map() };
|
|
5148
|
+
persistedReceiptEgressBudgets.set(this.node, nodeBudget);
|
|
5149
|
+
}
|
|
5150
|
+
const peerSessionKey = `${peer}\0${capabilitySession}`;
|
|
5151
|
+
let bucket = nodeBudget.peerSessions.get(peerSessionKey);
|
|
5152
|
+
if (!bucket) {
|
|
5153
|
+
while (
|
|
5154
|
+
nodeBudget.peerSessions.size >=
|
|
5155
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
|
|
5156
|
+
) {
|
|
5157
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
5158
|
+
if (oldest === undefined) break;
|
|
5159
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
5160
|
+
}
|
|
5161
|
+
bucket = {
|
|
5162
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
5163
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
5164
|
+
refilledAt: now,
|
|
5165
|
+
};
|
|
5166
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
5167
|
+
} else {
|
|
5168
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
5169
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
5170
|
+
}
|
|
5171
|
+
refillPersistedReceiptIngressBucket(
|
|
5172
|
+
bucket,
|
|
5173
|
+
now,
|
|
5174
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
5175
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
5176
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
|
|
5177
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
|
|
5178
|
+
);
|
|
5179
|
+
if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
|
|
5180
|
+
bucket.requestTokens -= 1;
|
|
5181
|
+
bucket.hashTokens -= hashCount;
|
|
5182
|
+
return 0;
|
|
5183
|
+
}
|
|
5184
|
+
return Math.max(
|
|
5185
|
+
1,
|
|
5186
|
+
Math.ceil(
|
|
5187
|
+
Math.max(
|
|
5188
|
+
((1 - bucket.requestTokens) /
|
|
5189
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
5190
|
+
1_000,
|
|
5191
|
+
((hashCount - bucket.hashTokens) /
|
|
5192
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
5193
|
+
1_000,
|
|
5194
|
+
),
|
|
5195
|
+
),
|
|
5196
|
+
);
|
|
5197
|
+
}
|
|
5198
|
+
|
|
5199
|
+
private async waitForPersistedReceiptEgressAdmission(
|
|
5200
|
+
peer: string,
|
|
5201
|
+
capabilitySession: bigint,
|
|
5202
|
+
hashCount: number,
|
|
5203
|
+
signal: AbortSignal,
|
|
5204
|
+
): Promise<void> {
|
|
5205
|
+
while (true) {
|
|
5206
|
+
if (signal.aborted) {
|
|
5207
|
+
throw signal.reason ?? new AbortError();
|
|
5208
|
+
}
|
|
5209
|
+
const waitMs = this.reservePersistedReceiptEgress(
|
|
5210
|
+
peer,
|
|
5211
|
+
capabilitySession,
|
|
5212
|
+
hashCount,
|
|
5213
|
+
);
|
|
5214
|
+
if (waitMs === 0) return;
|
|
5215
|
+
await this.waitPersistedReceiptRetry(signal, waitMs);
|
|
5216
|
+
}
|
|
5217
|
+
}
|
|
5218
|
+
|
|
5219
|
+
private persistedDeliveryTimeoutMs(
|
|
5220
|
+
delivery: DeliveryOptions,
|
|
5221
|
+
hashCount: number,
|
|
5222
|
+
): number {
|
|
5223
|
+
return (
|
|
5224
|
+
delivery.timeout ??
|
|
5225
|
+
Math.min(
|
|
5226
|
+
MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
|
|
5227
|
+
DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
|
|
5228
|
+
persistedTransferAdmissionBudgetMs(hashCount) +
|
|
5229
|
+
Math.ceil(persistedReceiptPacingFloorMs(hashCount)),
|
|
5230
|
+
)
|
|
5231
|
+
);
|
|
5232
|
+
}
|
|
5233
|
+
|
|
5234
|
+
private async waitForPersistedTransferAdmission(
|
|
5235
|
+
peer: string,
|
|
5236
|
+
hashes: readonly string[],
|
|
5237
|
+
captured: { capabilitySession: bigint; peerSession: PeerSession },
|
|
5238
|
+
signal: AbortSignal,
|
|
5239
|
+
isStillCurrent: () => boolean,
|
|
5240
|
+
): Promise<boolean> {
|
|
5241
|
+
const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
5242
|
+
while (!signal.aborted && isStillCurrent()) {
|
|
5243
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5244
|
+
if (
|
|
5245
|
+
!current ||
|
|
5246
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5247
|
+
current.peerSession !== captured.peerSession
|
|
5248
|
+
) {
|
|
5249
|
+
return false;
|
|
5250
|
+
}
|
|
5251
|
+
if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
|
|
5252
|
+
return true;
|
|
5253
|
+
}
|
|
5254
|
+
const remaining = expiresAt - Date.now();
|
|
5255
|
+
if (remaining <= 0) return false;
|
|
5256
|
+
await this.waitPersistedReceiptRetry(
|
|
5257
|
+
signal,
|
|
5258
|
+
Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining),
|
|
5259
|
+
);
|
|
5260
|
+
}
|
|
5261
|
+
if (signal.aborted) throw signal.reason ?? new AbortError();
|
|
5262
|
+
return false;
|
|
5263
|
+
}
|
|
5264
|
+
|
|
5265
|
+
private createPersistedDeliveryDeadline(
|
|
5266
|
+
delivery: DeliveryOptions,
|
|
5267
|
+
ownershipLifecycleController: AbortController,
|
|
5268
|
+
hashCount = 1,
|
|
5269
|
+
): PersistedDeliveryDeadline {
|
|
5270
|
+
const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
|
|
5271
|
+
if (
|
|
5272
|
+
!Number.isFinite(timeoutMs) ||
|
|
5273
|
+
timeoutMs <= 0 ||
|
|
5274
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
|
|
5275
|
+
) {
|
|
5276
|
+
throw new Error(
|
|
5277
|
+
`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
|
|
5278
|
+
);
|
|
5279
|
+
}
|
|
5280
|
+
const minAcks = Math.floor(delivery.minAcks!);
|
|
5281
|
+
const deadlineController = new AbortController();
|
|
5282
|
+
const timeout = setTimeout(
|
|
5283
|
+
() =>
|
|
5284
|
+
deadlineController.abort(
|
|
5285
|
+
new TimeoutError(
|
|
5286
|
+
`Timed out waiting for ${minAcks} persisted remote replicas.`,
|
|
5287
|
+
),
|
|
5288
|
+
),
|
|
5289
|
+
timeoutMs,
|
|
5290
|
+
);
|
|
5291
|
+
timeout.unref?.();
|
|
5292
|
+
return {
|
|
5293
|
+
deadline: Date.now() + timeoutMs,
|
|
5294
|
+
signal: AbortSignal.any(
|
|
5295
|
+
[
|
|
5296
|
+
delivery.signal,
|
|
5297
|
+
this._closeController.signal,
|
|
5298
|
+
ownershipLifecycleController.signal,
|
|
5299
|
+
deadlineController.signal,
|
|
5300
|
+
].filter((value): value is AbortSignal => !!value),
|
|
5301
|
+
),
|
|
5302
|
+
dispose: () => clearTimeout(timeout),
|
|
5303
|
+
};
|
|
5304
|
+
}
|
|
5305
|
+
|
|
5306
|
+
private async planPersistedDeliveryLeaders(
|
|
5307
|
+
entries: PersistedDeliveryPlanningEntry<T, R>[],
|
|
5308
|
+
replicas: number,
|
|
5309
|
+
ownershipLifecycleController: AbortController,
|
|
5310
|
+
): Promise<LeaderMap[]> {
|
|
5311
|
+
if (
|
|
5312
|
+
this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry
|
|
5313
|
+
) {
|
|
5314
|
+
const leaders: LeaderMap[] = [];
|
|
5315
|
+
for (const entry of entries) {
|
|
5316
|
+
leaders.push(
|
|
5317
|
+
await this.findLeadersFromEntry(
|
|
5318
|
+
entry,
|
|
5319
|
+
replicas,
|
|
5320
|
+
{ freshLeaderPlan: true },
|
|
5321
|
+
ownershipLifecycleController,
|
|
5322
|
+
),
|
|
5323
|
+
);
|
|
5324
|
+
}
|
|
5325
|
+
return leaders;
|
|
5326
|
+
}
|
|
5327
|
+
const items: EntryLeaderBatchItem<R>[] = entries.map((entry) => ({
|
|
5328
|
+
entry,
|
|
5329
|
+
replicas,
|
|
5330
|
+
options: { freshLeaderPlan: true, persist: false },
|
|
5331
|
+
}));
|
|
5332
|
+
const nativeRoutingPlanner =
|
|
5333
|
+
this._nativeRangePlanner ?? this._nativeBackbone;
|
|
5334
|
+
if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
|
|
5335
|
+
const options = items[0]!.options!;
|
|
5336
|
+
const context = await this.createLeaderSelectionContext(
|
|
5337
|
+
options,
|
|
5338
|
+
ownershipLifecycleController,
|
|
5339
|
+
);
|
|
5340
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5341
|
+
ownershipLifecycleController,
|
|
5342
|
+
);
|
|
5343
|
+
const nativeOptions = this.createNativeLeaderOptions(context, options);
|
|
5344
|
+
const fullReplicaLeaders =
|
|
5345
|
+
nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(
|
|
5346
|
+
replicas,
|
|
5347
|
+
nativeOptions,
|
|
5348
|
+
);
|
|
5349
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5350
|
+
ownershipLifecycleController,
|
|
5351
|
+
);
|
|
5352
|
+
if (fullReplicaLeaders) {
|
|
5353
|
+
// Receipt settlement treats leader maps as read-only. Reusing the one
|
|
5354
|
+
// gid-independent routing result avoids one Map allocation per entry.
|
|
5355
|
+
return new Array<LeaderMap>(items.length).fill(fullReplicaLeaders);
|
|
5356
|
+
}
|
|
5357
|
+
if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
|
|
5358
|
+
const nativeLeaders =
|
|
5359
|
+
nativeRoutingPlanner.planLeaderSamplesForGidsBatch(
|
|
5360
|
+
items.map((item) => ({
|
|
5361
|
+
gid: this.getEntryGid(item.entry),
|
|
5362
|
+
replicas: item.replicas,
|
|
5363
|
+
})),
|
|
5364
|
+
nativeOptions,
|
|
5365
|
+
);
|
|
5366
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5367
|
+
ownershipLifecycleController,
|
|
5368
|
+
);
|
|
5369
|
+
if (nativeLeaders?.length === items.length) {
|
|
5370
|
+
return nativeLeaders;
|
|
5371
|
+
}
|
|
5372
|
+
}
|
|
5373
|
+
}
|
|
5374
|
+
return (
|
|
5375
|
+
await this.planEntryLeaderBatch(items, ownershipLifecycleController)
|
|
5376
|
+
).map((plan) => plan.leaders);
|
|
5377
|
+
}
|
|
5378
|
+
|
|
5379
|
+
private async settlePersistedDelivery(
|
|
5380
|
+
input: PersistedDeliveryPlanningEntry<T, R>[],
|
|
5381
|
+
replicas: number,
|
|
5382
|
+
delivery: DeliveryOptions,
|
|
5383
|
+
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
5384
|
+
persistedDeadline?: PersistedDeliveryDeadline,
|
|
5385
|
+
transferOnFirstRound = false,
|
|
5386
|
+
): Promise<void> {
|
|
5387
|
+
const minAcks = Math.floor(delivery.minAcks!);
|
|
5388
|
+
const entries = new Map(input.map((entry) => [entry.hash, entry]));
|
|
5389
|
+
if (entries.size === 0) return;
|
|
5390
|
+
|
|
5391
|
+
const committedHashes = [...entries.keys()];
|
|
5392
|
+
const ownedDeadline = !persistedDeadline;
|
|
5393
|
+
const deadline =
|
|
5394
|
+
persistedDeadline ??
|
|
5395
|
+
this.createPersistedDeliveryDeadline(
|
|
5396
|
+
delivery,
|
|
5397
|
+
ownershipLifecycleController,
|
|
5398
|
+
entries.size,
|
|
5399
|
+
);
|
|
5400
|
+
const signal = deadline.signal;
|
|
5401
|
+
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
5402
|
+
let initialTransferPending = transferOnFirstRound;
|
|
5403
|
+
let needsInitialLeaderCheck = true;
|
|
5404
|
+
const carriedAcknowledgements = new Map<
|
|
5405
|
+
string,
|
|
5406
|
+
Map<string, { capabilitySession: bigint; peerSession: PeerSession }>
|
|
5407
|
+
>(committedHashes.map((hash) => [hash, new Map()]));
|
|
5408
|
+
const repairsByPeer = new Map<
|
|
5409
|
+
string,
|
|
5410
|
+
{
|
|
5411
|
+
capabilitySession: bigint;
|
|
5412
|
+
peerSession: PeerSession;
|
|
5413
|
+
hashes: Set<string>;
|
|
5414
|
+
}
|
|
5415
|
+
>();
|
|
5416
|
+
let acknowledgementOwnershipRevision: number | undefined;
|
|
5417
|
+
const purgePeerDeliveryState = (peer: string) => {
|
|
5418
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5419
|
+
acknowledgements.delete(peer);
|
|
5420
|
+
}
|
|
5421
|
+
repairsByPeer.delete(peer);
|
|
5422
|
+
};
|
|
5423
|
+
try {
|
|
5424
|
+
while (true) {
|
|
5425
|
+
if (signal.aborted) {
|
|
5426
|
+
throw signal.reason ?? new AbortError();
|
|
5427
|
+
}
|
|
5428
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5429
|
+
ownershipLifecycleController,
|
|
5430
|
+
);
|
|
5431
|
+
const ownershipRevision =
|
|
5432
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
5433
|
+
const isRoundOwnershipCurrent = () =>
|
|
5434
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
5435
|
+
if (acknowledgementOwnershipRevision !== ownershipRevision) {
|
|
5436
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5437
|
+
acknowledgements.clear();
|
|
5438
|
+
}
|
|
5439
|
+
repairsByPeer.clear();
|
|
5440
|
+
acknowledgementOwnershipRevision = ownershipRevision;
|
|
5441
|
+
} else {
|
|
5442
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5443
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5444
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5445
|
+
if (
|
|
5446
|
+
!current ||
|
|
5447
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5448
|
+
current.peerSession !== captured.peerSession
|
|
5449
|
+
) {
|
|
5450
|
+
purgePeerDeliveryState(peer);
|
|
5451
|
+
}
|
|
5452
|
+
}
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
if (!isRoundOwnershipCurrent()) {
|
|
5456
|
+
await this.waitPersistedReceiptRetry(
|
|
5457
|
+
signal,
|
|
5458
|
+
PERSISTED_RECEIPT_RETRY_MS,
|
|
5459
|
+
);
|
|
5460
|
+
continue;
|
|
5461
|
+
}
|
|
5462
|
+
|
|
5463
|
+
// Carry receipts only across retries in the exact same ownership and
|
|
5464
|
+
// transport epoch. A revision/session change purges them before they can
|
|
5465
|
+
// survive an away-and-back leader transition or combine with a later peer.
|
|
5466
|
+
const hashesByPeer = new Map<string, string[]>();
|
|
5467
|
+
const entryArray = [...entries.values()];
|
|
5468
|
+
const leadersByEntry = await this.planPersistedDeliveryLeaders(
|
|
5469
|
+
entryArray,
|
|
5470
|
+
replicas,
|
|
5471
|
+
ownershipLifecycleController,
|
|
5472
|
+
);
|
|
5473
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5474
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
5475
|
+
if (needsInitialLeaderCheck) {
|
|
5476
|
+
needsInitialLeaderCheck = false;
|
|
5477
|
+
if (
|
|
5478
|
+
leadersByEntry.some(
|
|
5479
|
+
(leaders) =>
|
|
5480
|
+
leaders.size === 0 ||
|
|
5481
|
+
(leaders.size === 1 && leaders.has(selfHash)),
|
|
5482
|
+
)
|
|
5483
|
+
) {
|
|
5484
|
+
throw new NoPeersError(this.rpc.topic);
|
|
5485
|
+
}
|
|
5486
|
+
}
|
|
5487
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
5488
|
+
const hash = entryArray[index]!.hash;
|
|
5489
|
+
const leaders = leadersByEntry[index]!;
|
|
5490
|
+
if (signal.aborted) {
|
|
5491
|
+
throw signal.reason ?? new AbortError();
|
|
5492
|
+
}
|
|
5493
|
+
if (!isRoundOwnershipCurrent()) break;
|
|
5494
|
+
const acknowledgements = carriedAcknowledgements.get(hash)!;
|
|
5495
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5496
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5497
|
+
if (
|
|
5498
|
+
!leaders.has(peer) ||
|
|
5499
|
+
!current ||
|
|
5500
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5501
|
+
current.peerSession !== captured.peerSession
|
|
5502
|
+
) {
|
|
5503
|
+
acknowledgements.delete(peer);
|
|
5504
|
+
}
|
|
5505
|
+
}
|
|
5506
|
+
if (acknowledgements.size >= minAcks) continue;
|
|
5507
|
+
for (const peer of leaders.keys()) {
|
|
5508
|
+
if (peer === selfHash) continue;
|
|
5509
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5510
|
+
if (!current) continue;
|
|
5511
|
+
if (acknowledgements.has(peer)) continue;
|
|
5512
|
+
const hashes = hashesByPeer.get(peer) ?? [];
|
|
5513
|
+
hashes.push(hash);
|
|
5514
|
+
hashesByPeer.set(peer, hashes);
|
|
5515
|
+
}
|
|
5516
|
+
}
|
|
5517
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5518
|
+
|
|
5519
|
+
const operationQueue = new PQueue({
|
|
5520
|
+
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
5521
|
+
});
|
|
5522
|
+
const candidateWaves = Math.max(
|
|
5523
|
+
1,
|
|
5524
|
+
Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL),
|
|
5525
|
+
);
|
|
5526
|
+
const roundController = new AbortController();
|
|
5527
|
+
const roundSignal = AbortSignal.any([signal, roundController.signal]);
|
|
5528
|
+
const getAttemptTimeout = () =>
|
|
5529
|
+
Math.max(
|
|
5530
|
+
1,
|
|
5531
|
+
Math.min(
|
|
5532
|
+
maxAttemptMs,
|
|
5533
|
+
Math.floor((deadline.deadline - Date.now()) / candidateWaves),
|
|
5534
|
+
),
|
|
5535
|
+
);
|
|
5536
|
+
const requests = new Set<Promise<void>>();
|
|
5537
|
+
const transferAllOnRound = initialTransferPending;
|
|
5538
|
+
try {
|
|
5539
|
+
for (const [peer, hashes] of hashesByPeer) {
|
|
5540
|
+
let request!: Promise<void>;
|
|
5541
|
+
request = (async () => {
|
|
5542
|
+
const captured = this.persistedReceiptPeerSession(peer);
|
|
5543
|
+
if (!captured) return;
|
|
5544
|
+
const previousRepair = repairsByPeer.get(peer);
|
|
5545
|
+
if (
|
|
5546
|
+
previousRepair &&
|
|
5547
|
+
(previousRepair.capabilitySession !==
|
|
5548
|
+
captured.capabilitySession ||
|
|
5549
|
+
previousRepair.peerSession !== captured.peerSession)
|
|
5550
|
+
) {
|
|
5551
|
+
repairsByPeer.delete(peer);
|
|
5552
|
+
}
|
|
5553
|
+
const ensureRepairState = () => {
|
|
5554
|
+
let state = repairsByPeer.get(peer);
|
|
5555
|
+
if (!state) {
|
|
5556
|
+
state = {
|
|
5557
|
+
capabilitySession: captured.capabilitySession,
|
|
5558
|
+
peerSession: captured.peerSession,
|
|
5559
|
+
hashes: new Set<string>(),
|
|
5560
|
+
};
|
|
5561
|
+
repairsByPeer.set(peer, state);
|
|
5562
|
+
}
|
|
5563
|
+
return state;
|
|
5564
|
+
};
|
|
5565
|
+
const isPeerRoundCurrent = () => {
|
|
5566
|
+
if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
|
|
5567
|
+
return false;
|
|
5568
|
+
}
|
|
5569
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5570
|
+
return (
|
|
5571
|
+
!!current &&
|
|
5572
|
+
current.capabilitySession === captured.capabilitySession &&
|
|
5573
|
+
current.peerSession === captured.peerSession
|
|
5574
|
+
);
|
|
5575
|
+
};
|
|
5576
|
+
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
5577
|
+
const transferHashes = transferAllOnRound
|
|
5578
|
+
? hashes
|
|
5579
|
+
: repairs
|
|
5580
|
+
? hashes.filter((hash) => repairs.has(hash))
|
|
5581
|
+
: [];
|
|
5582
|
+
let attemptedHashCount = 0;
|
|
5583
|
+
if (transferHashes.length > 0) {
|
|
5584
|
+
try {
|
|
5585
|
+
await this.pushEntryHashes(peer, transferHashes, {
|
|
5586
|
+
acknowledge: false,
|
|
5587
|
+
chunkTimeout: getAttemptTimeout,
|
|
5588
|
+
chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
5589
|
+
isStillCurrent: isPeerRoundCurrent,
|
|
5590
|
+
onChunkAttempted: (chunk) => {
|
|
5591
|
+
attemptedHashCount += chunk.length;
|
|
5592
|
+
if (!transferAllOnRound) {
|
|
5593
|
+
const state = repairsByPeer.get(peer);
|
|
5594
|
+
for (const hash of chunk) state?.hashes.delete(hash);
|
|
5595
|
+
}
|
|
5596
|
+
},
|
|
5597
|
+
onChunkSent: async (chunk) => {
|
|
5598
|
+
return this.waitForPersistedTransferAdmission(
|
|
5599
|
+
peer,
|
|
5600
|
+
chunk,
|
|
5601
|
+
captured,
|
|
5602
|
+
roundSignal,
|
|
5603
|
+
isPeerRoundCurrent,
|
|
5604
|
+
);
|
|
5605
|
+
},
|
|
5606
|
+
operationQueue,
|
|
5607
|
+
priority: delivery.priority,
|
|
5608
|
+
repairHint: !transferAllOnRound,
|
|
5609
|
+
signal: roundSignal,
|
|
5610
|
+
});
|
|
5611
|
+
} catch {
|
|
5612
|
+
// A send can fail after remote admission. The attempted prefix
|
|
5613
|
+
// includes that uncertain chunk for authoritative receipt probing.
|
|
5614
|
+
}
|
|
5615
|
+
if (
|
|
5616
|
+
transferAllOnRound &&
|
|
5617
|
+
attemptedHashCount < transferHashes.length
|
|
5618
|
+
) {
|
|
5619
|
+
const state = ensureRepairState();
|
|
5620
|
+
for (const hash of transferHashes.slice(attemptedHashCount)) {
|
|
5621
|
+
state.hashes.add(hash);
|
|
5622
|
+
}
|
|
5623
|
+
}
|
|
5624
|
+
}
|
|
5625
|
+
const receiptHashes = transferAllOnRound
|
|
5626
|
+
? hashes.slice(0, attemptedHashCount)
|
|
5627
|
+
: hashes;
|
|
5628
|
+
if (!isPeerRoundCurrent()) {
|
|
5629
|
+
purgePeerDeliveryState(peer);
|
|
5630
|
+
return;
|
|
5631
|
+
}
|
|
5632
|
+
// Keep receipt chunks sequential per peer. The shared operation
|
|
5633
|
+
// queue bounds only active sends/requests; admission waits never
|
|
5634
|
+
// occupy its slots and cannot starve a later healthy candidate.
|
|
5635
|
+
for (
|
|
5636
|
+
let offset = 0;
|
|
5637
|
+
offset < receiptHashes.length;
|
|
5638
|
+
offset += PERSISTED_RECEIPT_CHUNK_SIZE
|
|
5639
|
+
) {
|
|
5640
|
+
const requestedHashes = receiptHashes.slice(
|
|
5641
|
+
offset,
|
|
5642
|
+
offset + PERSISTED_RECEIPT_CHUNK_SIZE,
|
|
5643
|
+
);
|
|
5644
|
+
if (roundSignal.aborted || !isPeerRoundCurrent()) {
|
|
5645
|
+
if (!isRoundOwnershipCurrent()) roundController.abort();
|
|
5646
|
+
break;
|
|
5647
|
+
}
|
|
5648
|
+
let responses;
|
|
5649
|
+
try {
|
|
5650
|
+
await this.waitForPersistedReceiptEgressAdmission(
|
|
5651
|
+
peer,
|
|
5652
|
+
captured.capabilitySession,
|
|
5653
|
+
requestedHashes.length,
|
|
5654
|
+
roundSignal,
|
|
5655
|
+
);
|
|
5656
|
+
if (!isPeerRoundCurrent()) break;
|
|
5657
|
+
const attemptTimeout = getAttemptTimeout();
|
|
5658
|
+
responses =
|
|
5659
|
+
(await operationQueue.add(async () => {
|
|
5660
|
+
if (!isPeerRoundCurrent()) return [];
|
|
5661
|
+
return this.rpc.request(
|
|
5662
|
+
new RequestPersistedEntriesV1({
|
|
5663
|
+
expectedReceiverSession: captured.capabilitySession,
|
|
5664
|
+
hashes: requestedHashes,
|
|
5665
|
+
}),
|
|
5666
|
+
{
|
|
5667
|
+
mode: new SilentDelivery({
|
|
5668
|
+
to: [peer],
|
|
5669
|
+
redundancy: 1,
|
|
5670
|
+
}),
|
|
5671
|
+
amount: 1,
|
|
5672
|
+
priority: delivery.priority,
|
|
5673
|
+
timeout: attemptTimeout,
|
|
5674
|
+
signal: roundSignal,
|
|
5675
|
+
},
|
|
5676
|
+
);
|
|
5677
|
+
})) ?? [];
|
|
5678
|
+
} catch {
|
|
5679
|
+
if (roundSignal.aborted) break;
|
|
5680
|
+
// A peer can disconnect or miss this retry while the overall
|
|
5681
|
+
// quorum deadline remains active. Replan on the next round.
|
|
5682
|
+
break;
|
|
5683
|
+
}
|
|
5684
|
+
if (!isRoundOwnershipCurrent()) {
|
|
5685
|
+
roundController.abort();
|
|
5686
|
+
break;
|
|
5687
|
+
}
|
|
5688
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5689
|
+
if (
|
|
5690
|
+
!current ||
|
|
5691
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5692
|
+
current.peerSession !== captured.peerSession
|
|
5693
|
+
) {
|
|
5694
|
+
purgePeerDeliveryState(peer);
|
|
5695
|
+
break;
|
|
5696
|
+
}
|
|
5697
|
+
const requested = new Set(requestedHashes);
|
|
5698
|
+
const confirmed = new Set<string>();
|
|
5699
|
+
let receivedValidConfirmation = false;
|
|
5700
|
+
for (const result of responses) {
|
|
5701
|
+
if (
|
|
5702
|
+
!(result.response instanceof ConfirmEntriesMessage) ||
|
|
5703
|
+
result.from?.hashcode() !== peer ||
|
|
5704
|
+
result.message.header.session !== captured.capabilitySession
|
|
5705
|
+
) {
|
|
5706
|
+
continue;
|
|
5707
|
+
}
|
|
5708
|
+
const unique = new Set(result.response.hashes);
|
|
5709
|
+
if (
|
|
5710
|
+
unique.size !== result.response.hashes.length ||
|
|
5711
|
+
[...unique].some((hash) => !requested.has(hash))
|
|
5712
|
+
) {
|
|
5713
|
+
continue;
|
|
5714
|
+
}
|
|
5715
|
+
receivedValidConfirmation = true;
|
|
5716
|
+
for (const hash of unique) {
|
|
5717
|
+
confirmed.add(hash);
|
|
5718
|
+
carriedAcknowledgements.get(hash)?.set(peer, captured);
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
if (receivedValidConfirmation) {
|
|
5722
|
+
const state = ensureRepairState();
|
|
5723
|
+
for (const hash of requested) {
|
|
5724
|
+
if (confirmed.has(hash)) {
|
|
5725
|
+
state.hashes.delete(hash);
|
|
5726
|
+
} else {
|
|
5727
|
+
state.hashes.add(hash);
|
|
5728
|
+
}
|
|
5729
|
+
}
|
|
5730
|
+
}
|
|
5731
|
+
}
|
|
5732
|
+
})()
|
|
5733
|
+
.then(() => undefined)
|
|
5734
|
+
.catch(() => undefined)
|
|
5735
|
+
.finally(() => requests.delete(request));
|
|
5736
|
+
requests.add(request);
|
|
5737
|
+
}
|
|
5738
|
+
|
|
5739
|
+
const roundComplete = async (): Promise<boolean> => {
|
|
5740
|
+
if (signal.aborted) {
|
|
5741
|
+
throw signal.reason ?? new AbortError();
|
|
5742
|
+
}
|
|
5743
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
5744
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5745
|
+
if (acknowledgements.size < minAcks) return false;
|
|
5746
|
+
}
|
|
5747
|
+
const validatedLeaders = await this.planPersistedDeliveryLeaders(
|
|
5748
|
+
entryArray,
|
|
5749
|
+
replicas,
|
|
5750
|
+
ownershipLifecycleController,
|
|
5751
|
+
);
|
|
5752
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
5753
|
+
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
5754
|
+
if (signal.aborted) {
|
|
5755
|
+
throw signal.reason ?? new AbortError();
|
|
5756
|
+
}
|
|
5757
|
+
return false;
|
|
5758
|
+
}
|
|
5759
|
+
const hash = entryArray[index]!.hash;
|
|
5760
|
+
const leaders = validatedLeaders[index]!;
|
|
5761
|
+
let valid = 0;
|
|
5762
|
+
const acknowledgements = carriedAcknowledgements.get(hash)!;
|
|
5763
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5764
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5765
|
+
if (
|
|
5766
|
+
!current ||
|
|
5767
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5768
|
+
current.peerSession !== captured.peerSession
|
|
5769
|
+
) {
|
|
5770
|
+
purgePeerDeliveryState(peer);
|
|
5771
|
+
continue;
|
|
5772
|
+
}
|
|
5773
|
+
if (!leaders.has(peer)) {
|
|
5774
|
+
acknowledgements.delete(peer);
|
|
5775
|
+
continue;
|
|
5776
|
+
}
|
|
5777
|
+
if (++valid >= minAcks) {
|
|
5778
|
+
break;
|
|
5779
|
+
}
|
|
5780
|
+
}
|
|
5781
|
+
if (valid < minAcks) return false;
|
|
5782
|
+
}
|
|
5783
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5784
|
+
ownershipLifecycleController,
|
|
5785
|
+
);
|
|
5786
|
+
if (signal.aborted) {
|
|
5787
|
+
throw signal.reason ?? new AbortError();
|
|
5788
|
+
}
|
|
5789
|
+
return isRoundOwnershipCurrent();
|
|
5790
|
+
};
|
|
5791
|
+
|
|
5792
|
+
while (requests.size > 0) {
|
|
5793
|
+
await Promise.race(requests);
|
|
5794
|
+
if (await roundComplete()) {
|
|
5795
|
+
return;
|
|
5796
|
+
}
|
|
5797
|
+
}
|
|
5798
|
+
if (await roundComplete()) {
|
|
5799
|
+
return;
|
|
5800
|
+
}
|
|
5801
|
+
} finally {
|
|
5802
|
+
roundController.abort();
|
|
5803
|
+
void Promise.allSettled([...requests]);
|
|
5804
|
+
}
|
|
5805
|
+
if (isRoundOwnershipCurrent()) {
|
|
5806
|
+
initialTransferPending = false;
|
|
5807
|
+
}
|
|
5808
|
+
// Keep early retries fair and responsive, then let a caller's longer
|
|
5809
|
+
// overall deadline accommodate a genuinely slow durability barrier.
|
|
5810
|
+
maxAttemptMs = Math.min(
|
|
5811
|
+
MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
|
|
5812
|
+
maxAttemptMs * 2,
|
|
5813
|
+
);
|
|
5814
|
+
await this.waitPersistedReceiptRetry(
|
|
5815
|
+
signal,
|
|
5816
|
+
Math.max(
|
|
5817
|
+
0,
|
|
5818
|
+
Math.min(
|
|
5819
|
+
PERSISTED_RECEIPT_RETRY_MS,
|
|
5820
|
+
deadline.deadline - Date.now(),
|
|
5821
|
+
),
|
|
5822
|
+
),
|
|
5823
|
+
);
|
|
5824
|
+
}
|
|
5825
|
+
} catch (error) {
|
|
5826
|
+
if (error instanceof PersistedDeliveryError) {
|
|
5827
|
+
throw error;
|
|
5828
|
+
}
|
|
5829
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
5830
|
+
} finally {
|
|
5831
|
+
if (ownedDeadline) deadline.dispose();
|
|
5832
|
+
}
|
|
4791
5833
|
}
|
|
4792
5834
|
|
|
4793
5835
|
private async _appendDeliverToReplicators(
|
|
@@ -7850,13 +8892,7 @@ export class SharedLog<
|
|
|
7850
8892
|
try {
|
|
7851
8893
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
7852
8894
|
if (fanoutService?.provide && !this._providerHandle) {
|
|
7853
|
-
this.
|
|
7854
|
-
`shared-log|${this.topic}`,
|
|
7855
|
-
{
|
|
7856
|
-
ttlMs: 120_000,
|
|
7857
|
-
announceIntervalMs: 60_000,
|
|
7858
|
-
},
|
|
7859
|
-
);
|
|
8895
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
7860
8896
|
}
|
|
7861
8897
|
} catch {
|
|
7862
8898
|
// Best-effort only.
|
|
@@ -8368,63 +9404,135 @@ export class SharedLog<
|
|
|
8368
9404
|
}
|
|
8369
9405
|
}
|
|
8370
9406
|
|
|
8371
|
-
private async
|
|
9407
|
+
private async pushEntryHashChunk(
|
|
8372
9408
|
target: string,
|
|
8373
|
-
|
|
8374
|
-
|
|
8375
|
-
|
|
8376
|
-
|
|
8377
|
-
|
|
8378
|
-
|
|
8379
|
-
}
|
|
8380
|
-
|
|
8381
|
-
|
|
9409
|
+
chunk: string[],
|
|
9410
|
+
options: {
|
|
9411
|
+
acknowledge?: boolean;
|
|
9412
|
+
priority?: number;
|
|
9413
|
+
repairHint?: boolean;
|
|
9414
|
+
signal?: AbortSignal;
|
|
9415
|
+
},
|
|
9416
|
+
isStillCurrent: () => boolean,
|
|
9417
|
+
): Promise<boolean> {
|
|
9418
|
+
if (!isStillCurrent()) return false;
|
|
9419
|
+
const useRaw =
|
|
8382
9420
|
this._logProperties?.sync?.rawExchangeHeads === true &&
|
|
8383
|
-
this.peerSupportsRawExchangeHeads(target)
|
|
8384
|
-
) {
|
|
8385
|
-
const reserved = new Uint8Array(4);
|
|
8386
|
-
reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
9421
|
+
this.peerSupportsRawExchangeHeads(target);
|
|
9422
|
+
if (useRaw) {
|
|
9423
|
+
const reserved = options.repairHint ? new Uint8Array(4) : undefined;
|
|
9424
|
+
if (reserved) reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8387
9425
|
const sentMessages = await this.trySendFusedRawExchangeHeads(
|
|
8388
|
-
|
|
9426
|
+
chunk,
|
|
8389
9427
|
[target],
|
|
8390
|
-
{
|
|
9428
|
+
{
|
|
9429
|
+
acknowledge: options.acknowledge,
|
|
9430
|
+
priority: options.priority,
|
|
9431
|
+
reserved,
|
|
9432
|
+
signal: options.signal,
|
|
9433
|
+
},
|
|
8391
9434
|
);
|
|
8392
|
-
if (!isStillCurrent())
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
9435
|
+
if (!isStillCurrent()) return false;
|
|
9436
|
+
if (sentMessages === undefined) {
|
|
9437
|
+
for await (const message of createRawExchangeHeadsMessages(
|
|
9438
|
+
this.log,
|
|
9439
|
+
chunk,
|
|
9440
|
+
this._logProperties?.sync?.profile,
|
|
9441
|
+
)) {
|
|
9442
|
+
if (!isStillCurrent()) return false;
|
|
9443
|
+
if (options.repairHint) {
|
|
9444
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
9445
|
+
}
|
|
9446
|
+
await this.rpc.send(message, {
|
|
9447
|
+
priority: options.priority,
|
|
9448
|
+
mode: options.acknowledge
|
|
9449
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
9450
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
9451
|
+
signal: options.signal,
|
|
9452
|
+
});
|
|
9453
|
+
}
|
|
8397
9454
|
}
|
|
8398
|
-
|
|
9455
|
+
} else {
|
|
9456
|
+
for await (const message of createExchangeHeadsMessages(
|
|
8399
9457
|
this.log,
|
|
8400
|
-
|
|
8401
|
-
this._logProperties?.sync?.profile,
|
|
9458
|
+
chunk,
|
|
8402
9459
|
)) {
|
|
8403
|
-
if (!isStillCurrent())
|
|
8404
|
-
|
|
9460
|
+
if (!isStillCurrent()) return false;
|
|
9461
|
+
if (options.repairHint) {
|
|
9462
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8405
9463
|
}
|
|
8406
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8407
9464
|
await this.rpc.send(message, {
|
|
8408
|
-
priority:
|
|
8409
|
-
mode:
|
|
8410
|
-
|
|
9465
|
+
priority: options.priority,
|
|
9466
|
+
mode: options.acknowledge
|
|
9467
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
9468
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
9469
|
+
signal: options.signal,
|
|
8411
9470
|
});
|
|
8412
9471
|
}
|
|
8413
|
-
return;
|
|
8414
9472
|
}
|
|
8415
|
-
|
|
8416
|
-
|
|
8417
|
-
|
|
8418
|
-
|
|
8419
|
-
|
|
8420
|
-
|
|
8421
|
-
|
|
8422
|
-
|
|
8423
|
-
|
|
8424
|
-
|
|
9473
|
+
return isStillCurrent();
|
|
9474
|
+
}
|
|
9475
|
+
|
|
9476
|
+
private async pushEntryHashes(
|
|
9477
|
+
target: string,
|
|
9478
|
+
hashes: string[],
|
|
9479
|
+
options: {
|
|
9480
|
+
acknowledge?: boolean;
|
|
9481
|
+
chunkTimeout?: () => number;
|
|
9482
|
+
chunkSize?: number;
|
|
9483
|
+
isStillCurrent?: () => boolean;
|
|
9484
|
+
onChunkAttempted?: (hashes: readonly string[]) => void;
|
|
9485
|
+
onChunkSent?: (hashes: readonly string[]) => Promise<boolean>;
|
|
9486
|
+
operationQueue?: PQueue;
|
|
9487
|
+
priority?: number;
|
|
9488
|
+
repairHint?: boolean;
|
|
9489
|
+
signal?: AbortSignal;
|
|
9490
|
+
},
|
|
9491
|
+
) {
|
|
9492
|
+
const isStillCurrent = options.isStillCurrent ?? (() => true);
|
|
9493
|
+
if (!isStillCurrent()) return;
|
|
9494
|
+
const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
|
|
9495
|
+
for (let offset = 0; offset < hashes.length; offset += chunkSize) {
|
|
9496
|
+
const chunk = hashes.slice(offset, offset + chunkSize);
|
|
9497
|
+
const pushChunk = () => {
|
|
9498
|
+
options.onChunkAttempted?.(chunk);
|
|
9499
|
+
const chunkTimeout = options.chunkTimeout?.();
|
|
9500
|
+
const chunkSignal =
|
|
9501
|
+
chunkTimeout == null
|
|
9502
|
+
? options.signal
|
|
9503
|
+
: AbortSignal.any([
|
|
9504
|
+
...(options.signal ? [options.signal] : []),
|
|
9505
|
+
AbortSignal.timeout(Math.max(1, chunkTimeout)),
|
|
9506
|
+
]);
|
|
9507
|
+
return this.pushEntryHashChunk(
|
|
9508
|
+
target,
|
|
9509
|
+
chunk,
|
|
9510
|
+
{ ...options, signal: chunkSignal },
|
|
9511
|
+
isStillCurrent,
|
|
9512
|
+
);
|
|
9513
|
+
};
|
|
9514
|
+
const pushed = options.operationQueue
|
|
9515
|
+
? await options.operationQueue.add(pushChunk)
|
|
9516
|
+
: await pushChunk();
|
|
9517
|
+
if (pushed !== true) return;
|
|
9518
|
+
if (options.onChunkSent && !(await options.onChunkSent(chunk))) return;
|
|
8425
9519
|
}
|
|
8426
9520
|
}
|
|
8427
9521
|
|
|
9522
|
+
private async pushRepairEntries(
|
|
9523
|
+
target: string,
|
|
9524
|
+
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
9525
|
+
isStillCurrent: () => boolean = () => true,
|
|
9526
|
+
signal?: AbortSignal,
|
|
9527
|
+
) {
|
|
9528
|
+
return this.pushEntryHashes(target, [...entries.keys()], {
|
|
9529
|
+
isStillCurrent,
|
|
9530
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
9531
|
+
repairHint: true,
|
|
9532
|
+
signal,
|
|
9533
|
+
});
|
|
9534
|
+
}
|
|
9535
|
+
|
|
8428
9536
|
private async sendRepairEntriesWithTransport(
|
|
8429
9537
|
target: string,
|
|
8430
9538
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
@@ -10395,6 +11503,7 @@ export class SharedLog<
|
|
|
10395
11503
|
removed: ShallowOrFullEntry<T>[];
|
|
10396
11504
|
}> {
|
|
10397
11505
|
this.throwIfNativeDurableCommitFailed();
|
|
11506
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
10398
11507
|
const ownershipLifecycleController =
|
|
10399
11508
|
this.captureReplicationOwnershipLifecycle();
|
|
10400
11509
|
if (this._isAdaptiveReplicating) {
|
|
@@ -10405,18 +11514,73 @@ export class SharedLog<
|
|
|
10405
11514
|
options,
|
|
10406
11515
|
ownershipLifecycleController,
|
|
10407
11516
|
);
|
|
10408
|
-
|
|
10409
|
-
|
|
10410
|
-
|
|
10411
|
-
|
|
10412
|
-
|
|
10413
|
-
|
|
10414
|
-
|
|
10415
|
-
|
|
10416
|
-
|
|
10417
|
-
|
|
10418
|
-
|
|
10419
|
-
|
|
11517
|
+
let committedHashes: readonly string[] | undefined;
|
|
11518
|
+
if (persistedDelivery) {
|
|
11519
|
+
(appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
|
|
11520
|
+
hashes,
|
|
11521
|
+
) => {
|
|
11522
|
+
// The lower log reports the exact hashes immediately after their
|
|
11523
|
+
// irreversible local mutation and before entry initialization, trim,
|
|
11524
|
+
// or change callbacks can reject the append.
|
|
11525
|
+
committedHashes = hashes;
|
|
11526
|
+
};
|
|
11527
|
+
}
|
|
11528
|
+
let persistedDeadline: PersistedDeliveryDeadline | undefined;
|
|
11529
|
+
const throwIfDeliveryAborted = () => {
|
|
11530
|
+
if (persistedDeadline?.signal.aborted) {
|
|
11531
|
+
throw persistedDeadline.signal.reason ?? new AbortError();
|
|
11532
|
+
}
|
|
11533
|
+
};
|
|
11534
|
+
try {
|
|
11535
|
+
const result = await this.log.append(data, appendOptions);
|
|
11536
|
+
committedHashes ??= [result.entry.hash];
|
|
11537
|
+
persistedDeadline = persistedDelivery
|
|
11538
|
+
? this.createPersistedDeliveryDeadline(
|
|
11539
|
+
persistedDelivery,
|
|
11540
|
+
ownershipLifecycleController,
|
|
11541
|
+
1,
|
|
11542
|
+
)
|
|
11543
|
+
: undefined;
|
|
11544
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
11545
|
+
ownershipLifecycleController,
|
|
11546
|
+
);
|
|
11547
|
+
throwIfDeliveryAborted();
|
|
11548
|
+
await this.processLocalAppend(result.entry, result.removed, options, {
|
|
11549
|
+
minReplicasValue,
|
|
11550
|
+
ownershipLifecycleController,
|
|
11551
|
+
});
|
|
11552
|
+
throwIfDeliveryAborted();
|
|
11553
|
+
if (persistedDelivery && persistedDeadline) {
|
|
11554
|
+
await this.settlePersistedDelivery(
|
|
11555
|
+
[result.entry],
|
|
11556
|
+
minReplicasValue,
|
|
11557
|
+
persistedDelivery,
|
|
11558
|
+
ownershipLifecycleController,
|
|
11559
|
+
persistedDeadline,
|
|
11560
|
+
);
|
|
11561
|
+
}
|
|
11562
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
11563
|
+
ownershipLifecycleController,
|
|
11564
|
+
);
|
|
11565
|
+
return result;
|
|
11566
|
+
} catch (error) {
|
|
11567
|
+
if (persistedDelivery && committedHashes) {
|
|
11568
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
11569
|
+
}
|
|
11570
|
+
throw error;
|
|
11571
|
+
} finally {
|
|
11572
|
+
persistedDeadline?.dispose();
|
|
11573
|
+
}
|
|
11574
|
+
}
|
|
11575
|
+
|
|
11576
|
+
private rejectPersistedDeliveryOnTrustedLocalAppend(
|
|
11577
|
+
options?: SharedAppendOptions<T>,
|
|
11578
|
+
): void {
|
|
11579
|
+
if (this.getPersistedDeliveryOptions(options)) {
|
|
11580
|
+
throw new Error(
|
|
11581
|
+
"trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit",
|
|
11582
|
+
);
|
|
11583
|
+
}
|
|
10420
11584
|
}
|
|
10421
11585
|
|
|
10422
11586
|
// Trusted local append path for callers that already validated the entry.
|
|
@@ -10428,6 +11592,7 @@ export class SharedLog<
|
|
|
10428
11592
|
removed: ShallowOrFullEntry<T>[];
|
|
10429
11593
|
}> {
|
|
10430
11594
|
this.throwIfNativeDurableCommitFailed();
|
|
11595
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10431
11596
|
const ownershipLifecycleController =
|
|
10432
11597
|
this.captureReplicationOwnershipLifecycle();
|
|
10433
11598
|
if (options?.canAppend || options?.onChange) {
|
|
@@ -10468,6 +11633,7 @@ export class SharedLog<
|
|
|
10468
11633
|
skipMissingNextJoin?: boolean;
|
|
10469
11634
|
resolveTrimmedEntries?: boolean;
|
|
10470
11635
|
payloadData?: Uint8Array;
|
|
11636
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
10471
11637
|
},
|
|
10472
11638
|
): Promise<{
|
|
10473
11639
|
entry: Entry<T>;
|
|
@@ -10477,6 +11643,7 @@ export class SharedLog<
|
|
|
10477
11643
|
appendCommit: PreparedLocalAppendCommit<R>;
|
|
10478
11644
|
}> {
|
|
10479
11645
|
this.throwIfNativeDurableCommitFailed();
|
|
11646
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10480
11647
|
const ownershipLifecycleController =
|
|
10481
11648
|
this.captureReplicationOwnershipLifecycle();
|
|
10482
11649
|
if (options?.canAppend || options?.onChange) {
|
|
@@ -10491,6 +11658,10 @@ export class SharedLog<
|
|
|
10491
11658
|
const { appendOptions, minReplicasValue } =
|
|
10492
11659
|
this.createLogAppendOptions(options);
|
|
10493
11660
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
11661
|
+
attachTrustedLocalCommitEvidence(
|
|
11662
|
+
appendOptions,
|
|
11663
|
+
properties?.localCommitEvidence,
|
|
11664
|
+
);
|
|
10494
11665
|
const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(
|
|
10495
11666
|
data,
|
|
10496
11667
|
appendOptions,
|
|
@@ -10500,6 +11671,11 @@ export class SharedLog<
|
|
|
10500
11671
|
payloadData: properties?.payloadData,
|
|
10501
11672
|
},
|
|
10502
11673
|
);
|
|
11674
|
+
if (properties?.localCommitEvidence) {
|
|
11675
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
11676
|
+
result.appendFacts.hash,
|
|
11677
|
+
);
|
|
11678
|
+
}
|
|
10503
11679
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
10504
11680
|
ownershipLifecycleController,
|
|
10505
11681
|
);
|
|
@@ -10748,6 +11924,9 @@ export class SharedLog<
|
|
|
10748
11924
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
10749
11925
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
10750
11926
|
payloadData,
|
|
11927
|
+
...(properties?.localCommitEvidence
|
|
11928
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
11929
|
+
: undefined),
|
|
10751
11930
|
});
|
|
10752
11931
|
}
|
|
10753
11932
|
|
|
@@ -10758,6 +11937,7 @@ export class SharedLog<
|
|
|
10758
11937
|
properties?: PreparedPayloadCommitOnlyProperties,
|
|
10759
11938
|
): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
|
|
10760
11939
|
this.throwIfNativeDurableCommitFailed();
|
|
11940
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10761
11941
|
if (options?.canAppend || options?.onChange) {
|
|
10762
11942
|
throw new Error(
|
|
10763
11943
|
"appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks",
|
|
@@ -10781,6 +11961,10 @@ export class SharedLog<
|
|
|
10781
11961
|
const { appendOptions, minReplicasValue } =
|
|
10782
11962
|
this.createLogAppendOptions(options);
|
|
10783
11963
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
11964
|
+
attachTrustedLocalCommitEvidence(
|
|
11965
|
+
appendOptions,
|
|
11966
|
+
properties?.localCommitEvidence,
|
|
11967
|
+
);
|
|
10784
11968
|
const deferHeadCoordinatePersistence =
|
|
10785
11969
|
this.shouldDeferHeadCoordinatePersistence(options);
|
|
10786
11970
|
const nativeBackboneResult =
|
|
@@ -10830,6 +12014,7 @@ export class SharedLog<
|
|
|
10830
12014
|
properties?: PreparedPayloadCommitOnlyProperties,
|
|
10831
12015
|
): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
|
|
10832
12016
|
this.throwIfNativeDurableCommitFailed();
|
|
12017
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10833
12018
|
if (options?.canAppend || options?.onChange) {
|
|
10834
12019
|
throw new Error(
|
|
10835
12020
|
"appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks",
|
|
@@ -10853,6 +12038,10 @@ export class SharedLog<
|
|
|
10853
12038
|
const { appendOptions, minReplicasValue } =
|
|
10854
12039
|
this.createLogAppendOptions(options);
|
|
10855
12040
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
12041
|
+
attachTrustedLocalCommitEvidence(
|
|
12042
|
+
appendOptions,
|
|
12043
|
+
properties?.localCommitEvidence,
|
|
12044
|
+
);
|
|
10856
12045
|
const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(
|
|
10857
12046
|
payloadData,
|
|
10858
12047
|
appendOptions,
|
|
@@ -10892,6 +12081,11 @@ export class SharedLog<
|
|
|
10892
12081
|
includeAppendFactsBytes: !deferHeadCoordinatePersistence,
|
|
10893
12082
|
});
|
|
10894
12083
|
return mapMaybePromise(resultMaybe, (result) => {
|
|
12084
|
+
if (result && properties?.localCommitEvidence) {
|
|
12085
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
12086
|
+
result.appendFacts.hash,
|
|
12087
|
+
);
|
|
12088
|
+
}
|
|
10895
12089
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
10896
12090
|
ownershipLifecycleController,
|
|
10897
12091
|
);
|
|
@@ -11472,25 +12666,28 @@ export class SharedLog<
|
|
|
11472
12666
|
// boundary separates the catch above from this statement and
|
|
11473
12667
|
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
11474
12668
|
// (the retire below only warns), so the token is terminal here.
|
|
11475
|
-
|
|
11476
|
-
|
|
11477
|
-
|
|
11478
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
11479
|
-
prepared.appendFacts,
|
|
11480
|
-
prepared.removed,
|
|
11481
|
-
prepared.materializeEntry,
|
|
11482
|
-
{
|
|
11483
|
-
removedHashes: prepared.removedHashes,
|
|
11484
|
-
removedGids: prepared.removedGids,
|
|
11485
|
-
},
|
|
11486
|
-
);
|
|
11487
|
-
try {
|
|
11488
|
-
await this.completeNativeStrictDurableTransaction(
|
|
11489
|
-
nativeStrictTransaction,
|
|
12669
|
+
if (properties?.localCommitEvidence) {
|
|
12670
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
12671
|
+
prepared.appendFacts.hash,
|
|
11490
12672
|
);
|
|
11491
|
-
} catch (error) {
|
|
11492
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
11493
12673
|
}
|
|
12674
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
12675
|
+
nativeStrictTransaction,
|
|
12676
|
+
() => {
|
|
12677
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
12678
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
12679
|
+
);
|
|
12680
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
12681
|
+
prepared.appendFacts,
|
|
12682
|
+
prepared.removed,
|
|
12683
|
+
prepared.materializeEntry,
|
|
12684
|
+
{
|
|
12685
|
+
removedHashes: prepared.removedHashes,
|
|
12686
|
+
removedGids: prepared.removedGids,
|
|
12687
|
+
},
|
|
12688
|
+
);
|
|
12689
|
+
},
|
|
12690
|
+
);
|
|
11494
12691
|
return finishResult;
|
|
11495
12692
|
});
|
|
11496
12693
|
}
|
|
@@ -12123,26 +13320,29 @@ export class SharedLog<
|
|
|
12123
13320
|
// Success seam: the finalizer acknowledge above is the last
|
|
12124
13321
|
// await inside the protected try, so `rollback` can no
|
|
12125
13322
|
// longer fire and the retire below only warns.
|
|
12126
|
-
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
12130
|
-
prepared.appendFacts,
|
|
12131
|
-
prepared.removed,
|
|
12132
|
-
prepared.materializeEntry,
|
|
12133
|
-
{
|
|
12134
|
-
forgetNativeCoordinates: false,
|
|
12135
|
-
removedHashes: prepared.removedHashes,
|
|
12136
|
-
removedGids: prepared.removedGids,
|
|
12137
|
-
},
|
|
12138
|
-
);
|
|
12139
|
-
try {
|
|
12140
|
-
await this.completeNativeStrictDurableTransaction(
|
|
12141
|
-
nativeStrictTransaction,
|
|
13323
|
+
if (properties?.localCommitEvidence) {
|
|
13324
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
13325
|
+
prepared.appendFacts.hash,
|
|
12142
13326
|
);
|
|
12143
|
-
} catch (error) {
|
|
12144
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
12145
13327
|
}
|
|
13328
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
13329
|
+
nativeStrictTransaction,
|
|
13330
|
+
() => {
|
|
13331
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
13332
|
+
rollbackCoordinateEntries,
|
|
13333
|
+
);
|
|
13334
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
13335
|
+
prepared.appendFacts,
|
|
13336
|
+
prepared.removed,
|
|
13337
|
+
prepared.materializeEntry,
|
|
13338
|
+
{
|
|
13339
|
+
forgetNativeCoordinates: false,
|
|
13340
|
+
removedHashes: prepared.removedHashes,
|
|
13341
|
+
removedGids: prepared.removedGids,
|
|
13342
|
+
},
|
|
13343
|
+
);
|
|
13344
|
+
},
|
|
13345
|
+
);
|
|
12146
13346
|
if (
|
|
12147
13347
|
commitBlocksInBackbone &&
|
|
12148
13348
|
!runtimeOnlyCoordinates &&
|
|
@@ -13235,73 +14435,73 @@ export class SharedLog<
|
|
|
13235
14435
|
nativeStrictTransaction,
|
|
13236
14436
|
);
|
|
13237
14437
|
});
|
|
13238
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13239
|
-
ownershipLifecycleController,
|
|
13240
|
-
);
|
|
13241
14438
|
} catch (error) {
|
|
13242
14439
|
return rollbackBatch(error);
|
|
13243
14440
|
}
|
|
13244
14441
|
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
13245
14442
|
// above), and everything from here on escapes without any rollback.
|
|
13246
|
-
|
|
13247
|
-
|
|
13248
|
-
|
|
13249
|
-
|
|
13250
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13251
|
-
ownershipLifecycleController,
|
|
13252
|
-
);
|
|
13253
|
-
const appendCommits: PreparedLocalAppendCommit<R>[] = [];
|
|
13254
|
-
for (let i = 0; i < coordinateRows.length; i++) {
|
|
13255
|
-
const {
|
|
13256
|
-
facts,
|
|
13257
|
-
backboneAppend,
|
|
13258
|
-
coordinateFields,
|
|
13259
|
-
plannedCoordinateDeleteHashes,
|
|
13260
|
-
} = coordinateRows[i]!;
|
|
13261
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
13262
|
-
facts,
|
|
13263
|
-
[],
|
|
13264
|
-
appended.materializeEntries[i]!,
|
|
13265
|
-
{
|
|
13266
|
-
forgetNativeCoordinates: false,
|
|
13267
|
-
removedHashes: plannedCoordinateDeleteHashes,
|
|
13268
|
-
removedGids:
|
|
13269
|
-
backboneAppend.trimmedGids ??
|
|
13270
|
-
(backboneAppend.trimmed.length > 0
|
|
13271
|
-
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
13272
|
-
: undefined),
|
|
13273
|
-
},
|
|
13274
|
-
);
|
|
13275
|
-
if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
|
|
13276
|
-
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
14443
|
+
if (properties?.localCommitEvidence) {
|
|
14444
|
+
for (const facts of appended.appendFacts) {
|
|
14445
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
13277
14446
|
}
|
|
13278
|
-
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
|
|
13279
|
-
facts,
|
|
13280
|
-
{
|
|
13281
|
-
hashNumber: backboneAppend.coordinate.hashNumber as NumberFromType<R>,
|
|
13282
|
-
coordinateFields,
|
|
13283
|
-
},
|
|
13284
|
-
);
|
|
13285
|
-
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
13286
|
-
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
13287
|
-
appended.documentTrimmedHeadsProcessed?.[i];
|
|
13288
|
-
appendCommit.documentPreviousContext =
|
|
13289
|
-
backboneAppend.documentPreviousContext;
|
|
13290
|
-
appendCommits.push(appendCommit);
|
|
13291
14447
|
}
|
|
13292
|
-
|
|
13293
|
-
await this.
|
|
14448
|
+
const appendCommits =
|
|
14449
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
13294
14450
|
nativeStrictTransaction,
|
|
14451
|
+
() => {
|
|
14452
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
14453
|
+
batchCoordinateRollback,
|
|
14454
|
+
);
|
|
14455
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
14456
|
+
ownershipLifecycleController,
|
|
14457
|
+
);
|
|
14458
|
+
const commits: PreparedLocalAppendCommit<R>[] = [];
|
|
14459
|
+
for (let i = 0; i < coordinateRows.length; i++) {
|
|
14460
|
+
const {
|
|
14461
|
+
facts,
|
|
14462
|
+
backboneAppend,
|
|
14463
|
+
coordinateFields,
|
|
14464
|
+
plannedCoordinateDeleteHashes,
|
|
14465
|
+
} = coordinateRows[i]!;
|
|
14466
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
14467
|
+
facts,
|
|
14468
|
+
[],
|
|
14469
|
+
appended.materializeEntries[i]!,
|
|
14470
|
+
{
|
|
14471
|
+
forgetNativeCoordinates: false,
|
|
14472
|
+
removedHashes: plannedCoordinateDeleteHashes,
|
|
14473
|
+
removedGids:
|
|
14474
|
+
backboneAppend.trimmedGids ??
|
|
14475
|
+
(backboneAppend.trimmed.length > 0
|
|
14476
|
+
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
14477
|
+
: undefined),
|
|
14478
|
+
},
|
|
14479
|
+
);
|
|
14480
|
+
if (
|
|
14481
|
+
!runtimeOnlyCoordinates &&
|
|
14482
|
+
this.remoteBlocks.hasNotifyStoredHook()
|
|
14483
|
+
) {
|
|
14484
|
+
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
14485
|
+
}
|
|
14486
|
+
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
|
|
14487
|
+
facts,
|
|
14488
|
+
{
|
|
14489
|
+
hashNumber: backboneAppend.coordinate
|
|
14490
|
+
.hashNumber as NumberFromType<R>,
|
|
14491
|
+
coordinateFields,
|
|
14492
|
+
},
|
|
14493
|
+
);
|
|
14494
|
+
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
14495
|
+
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
14496
|
+
appended.documentTrimmedHeadsProcessed?.[i];
|
|
14497
|
+
appendCommit.documentPreviousContext =
|
|
14498
|
+
backboneAppend.documentPreviousContext;
|
|
14499
|
+
commits.push(appendCommit);
|
|
14500
|
+
}
|
|
14501
|
+
return commits;
|
|
14502
|
+
},
|
|
14503
|
+
() => this.isRepairLifecycleActive(ownershipLifecycleController),
|
|
13295
14504
|
);
|
|
13296
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13297
|
-
ownershipLifecycleController,
|
|
13298
|
-
);
|
|
13299
|
-
} catch (error) {
|
|
13300
|
-
if (!this.isRepairLifecycleActive(ownershipLifecycleController)) {
|
|
13301
|
-
throw error;
|
|
13302
|
-
}
|
|
13303
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
13304
|
-
}
|
|
13305
14505
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13306
14506
|
ownershipLifecycleController,
|
|
13307
14507
|
);
|
|
@@ -13333,6 +14533,7 @@ export class SharedLog<
|
|
|
13333
14533
|
| undefined
|
|
13334
14534
|
> {
|
|
13335
14535
|
this.throwIfNativeDurableCommitFailed();
|
|
14536
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
13336
14537
|
if (data.length === 0) {
|
|
13337
14538
|
return { entries: [], removed: [], appendCommits: [] };
|
|
13338
14539
|
}
|
|
@@ -13350,6 +14551,10 @@ export class SharedLog<
|
|
|
13350
14551
|
const { appendOptions, minReplicasValue } =
|
|
13351
14552
|
this.createLogAppendOptions(options);
|
|
13352
14553
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
14554
|
+
attachTrustedLocalCommitEvidence(
|
|
14555
|
+
appendOptions,
|
|
14556
|
+
properties?.localCommitEvidence,
|
|
14557
|
+
);
|
|
13353
14558
|
const nativeBackboneBatch =
|
|
13354
14559
|
await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(
|
|
13355
14560
|
data,
|
|
@@ -13372,6 +14577,11 @@ export class SharedLog<
|
|
|
13372
14577
|
payloadDatas: properties?.payloadDatas,
|
|
13373
14578
|
nexts: properties?.nexts,
|
|
13374
14579
|
});
|
|
14580
|
+
if (result && properties?.localCommitEvidence) {
|
|
14581
|
+
for (const facts of result.appendFacts) {
|
|
14582
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
14583
|
+
}
|
|
14584
|
+
}
|
|
13375
14585
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13376
14586
|
ownershipLifecycleController,
|
|
13377
14587
|
);
|
|
@@ -13501,6 +14711,9 @@ export class SharedLog<
|
|
|
13501
14711
|
nativeBackboneDocumentIndexes:
|
|
13502
14712
|
properties?.nativeBackboneDocumentIndexes,
|
|
13503
14713
|
retainMaterializationBytes: properties?.retainMaterializationBytes,
|
|
14714
|
+
...(properties?.localCommitEvidence
|
|
14715
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
14716
|
+
: undefined),
|
|
13504
14717
|
},
|
|
13505
14718
|
);
|
|
13506
14719
|
}
|
|
@@ -13513,9 +14726,15 @@ export class SharedLog<
|
|
|
13513
14726
|
removed: ShallowOrFullEntry<T>[];
|
|
13514
14727
|
}> {
|
|
13515
14728
|
this.throwIfNativeDurableCommitFailed();
|
|
14729
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
13516
14730
|
if (data.length === 0) {
|
|
13517
14731
|
return { entries: [], removed: [] };
|
|
13518
14732
|
}
|
|
14733
|
+
if (persistedDelivery) {
|
|
14734
|
+
throw new Error(
|
|
14735
|
+
"persisted delivery is not supported for chained appendMany; use independent document puts",
|
|
14736
|
+
);
|
|
14737
|
+
}
|
|
13519
14738
|
const ownershipLifecycleController =
|
|
13520
14739
|
this.captureReplicationOwnershipLifecycle();
|
|
13521
14740
|
if (this._isAdaptiveReplicating) {
|
|
@@ -13592,6 +14811,123 @@ export class SharedLog<
|
|
|
13592
14811
|
return result;
|
|
13593
14812
|
}
|
|
13594
14813
|
|
|
14814
|
+
/**
|
|
14815
|
+
* Deliver entries that were already committed locally, then wait for the
|
|
14816
|
+
* requested persisted remote quorum. This is the post-commit seam used by
|
|
14817
|
+
* higher-level transactional/batched writers so a receipt timeout never
|
|
14818
|
+
* rolls back or hides their successful local commit.
|
|
14819
|
+
*/
|
|
14820
|
+
private async deliverPersistedPlanningEntries(
|
|
14821
|
+
resolveEntries: () => PersistedDeliveryPlanningEntry<T, R>[],
|
|
14822
|
+
committedHashes: string[],
|
|
14823
|
+
options: SharedAppendOptions<T>,
|
|
14824
|
+
): Promise<void> {
|
|
14825
|
+
let persistedDeadline: PersistedDeliveryDeadline | undefined;
|
|
14826
|
+
try {
|
|
14827
|
+
const delivery = this.getPersistedDeliveryOptions(options);
|
|
14828
|
+
if (!delivery) {
|
|
14829
|
+
throw new Error(
|
|
14830
|
+
'deliverPersistedEntries requires reliability="persisted"',
|
|
14831
|
+
);
|
|
14832
|
+
}
|
|
14833
|
+
const ownershipLifecycleController =
|
|
14834
|
+
this.captureReplicationOwnershipLifecycle();
|
|
14835
|
+
const deadline = this.createPersistedDeliveryDeadline(
|
|
14836
|
+
delivery,
|
|
14837
|
+
ownershipLifecycleController,
|
|
14838
|
+
committedHashes.length,
|
|
14839
|
+
);
|
|
14840
|
+
persistedDeadline = deadline;
|
|
14841
|
+
const throwIfDeliveryAborted = () => {
|
|
14842
|
+
if (deadline.signal.aborted) {
|
|
14843
|
+
throw deadline.signal.reason ?? new AbortError();
|
|
14844
|
+
}
|
|
14845
|
+
if (Date.now() >= deadline.deadline) {
|
|
14846
|
+
throw new TimeoutError(
|
|
14847
|
+
`Timed out waiting for ${Math.floor(delivery.minAcks!)} persisted remote replicas.`,
|
|
14848
|
+
);
|
|
14849
|
+
}
|
|
14850
|
+
};
|
|
14851
|
+
const entries = resolveEntries();
|
|
14852
|
+
throwIfDeliveryAborted();
|
|
14853
|
+
if (entries.length === 0) return;
|
|
14854
|
+
const { minReplicasValue } = this.createLogAppendOptions(
|
|
14855
|
+
options,
|
|
14856
|
+
ownershipLifecycleController,
|
|
14857
|
+
);
|
|
14858
|
+
throwIfDeliveryAborted();
|
|
14859
|
+
await this.settlePersistedDelivery(
|
|
14860
|
+
entries,
|
|
14861
|
+
minReplicasValue,
|
|
14862
|
+
delivery,
|
|
14863
|
+
ownershipLifecycleController,
|
|
14864
|
+
deadline,
|
|
14865
|
+
true,
|
|
14866
|
+
);
|
|
14867
|
+
} catch (error) {
|
|
14868
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
14869
|
+
} finally {
|
|
14870
|
+
persistedDeadline?.dispose();
|
|
14871
|
+
}
|
|
14872
|
+
}
|
|
14873
|
+
|
|
14874
|
+
private createPersistedDeliveryPlanningEntries(
|
|
14875
|
+
appendCommits: PreparedLocalAppendCommit<R>[],
|
|
14876
|
+
materializeEntries: () => Entry<T>[],
|
|
14877
|
+
): PersistedDeliveryPlanningEntry<T, R>[] {
|
|
14878
|
+
let materializedEntries: Entry<T>[] | undefined;
|
|
14879
|
+
const requiresFullEntries =
|
|
14880
|
+
this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
|
|
14881
|
+
const getMaterializedEntry = (
|
|
14882
|
+
appendCommit: PreparedLocalAppendCommit<R>,
|
|
14883
|
+
index: number,
|
|
14884
|
+
) => {
|
|
14885
|
+
materializedEntries ??= materializeEntries();
|
|
14886
|
+
const entry = materializedEntries[index];
|
|
14887
|
+
if (!entry || entry.hash !== appendCommit.hash) {
|
|
14888
|
+
throw new Error(
|
|
14889
|
+
`Persisted delivery materializer did not return committed entry ${appendCommit.hash}`,
|
|
14890
|
+
);
|
|
14891
|
+
}
|
|
14892
|
+
return entry;
|
|
14893
|
+
};
|
|
14894
|
+
return appendCommits.map((appendCommit, index) => {
|
|
14895
|
+
if (!requiresFullEntries && appendCommit.coordinateFields) {
|
|
14896
|
+
return this._coordinates.materializeResidentCoordinateEntry(
|
|
14897
|
+
appendCommit.coordinateFields,
|
|
14898
|
+
);
|
|
14899
|
+
}
|
|
14900
|
+
return getMaterializedEntry(appendCommit, index);
|
|
14901
|
+
});
|
|
14902
|
+
}
|
|
14903
|
+
|
|
14904
|
+
private deliverPersistedAppendCommits(
|
|
14905
|
+
appendCommits: PreparedLocalAppendCommit<R>[],
|
|
14906
|
+
materializeEntries: () => Entry<T>[],
|
|
14907
|
+
options: SharedAppendOptions<T>,
|
|
14908
|
+
): Promise<void> {
|
|
14909
|
+
return this.deliverPersistedPlanningEntries(
|
|
14910
|
+
() =>
|
|
14911
|
+
this.createPersistedDeliveryPlanningEntries(
|
|
14912
|
+
appendCommits,
|
|
14913
|
+
materializeEntries,
|
|
14914
|
+
),
|
|
14915
|
+
appendCommits.map((appendCommit) => appendCommit.hash),
|
|
14916
|
+
options,
|
|
14917
|
+
);
|
|
14918
|
+
}
|
|
14919
|
+
|
|
14920
|
+
async deliverPersistedEntries(
|
|
14921
|
+
entries: Entry<T>[],
|
|
14922
|
+
options: SharedAppendOptions<T>,
|
|
14923
|
+
): Promise<void> {
|
|
14924
|
+
return this.deliverPersistedPlanningEntries(
|
|
14925
|
+
() => entries,
|
|
14926
|
+
entries.map((entry) => entry.hash),
|
|
14927
|
+
options,
|
|
14928
|
+
);
|
|
14929
|
+
}
|
|
14930
|
+
|
|
13595
14931
|
private canCoalesceLocalAppendMany(
|
|
13596
14932
|
entries: Entry<T>[],
|
|
13597
14933
|
options?: SharedAppendOptions<T>,
|
|
@@ -14733,6 +16069,9 @@ export class SharedLog<
|
|
|
14733
16069
|
this._peerSyncCapabilities = new Map();
|
|
14734
16070
|
this._peerSyncCapabilitySessions = new Map();
|
|
14735
16071
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
16072
|
+
this._persistedReceiptStorage = undefined;
|
|
16073
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
16074
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
14736
16075
|
this._liveRawGossipBatches = new Map();
|
|
14737
16076
|
this._liveRawGossipFlushScheduled = false;
|
|
14738
16077
|
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
@@ -14875,6 +16214,43 @@ export class SharedLog<
|
|
|
14875
16214
|
|
|
14876
16215
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
14877
16216
|
const blockProviderNamespace = (cid: string) => `cid:${cid}`;
|
|
16217
|
+
const logProviderNamespace = `shared-log|${this.topic}`;
|
|
16218
|
+
const announceBlockProvider = async (cid: string): Promise<void> => {
|
|
16219
|
+
try {
|
|
16220
|
+
await fanoutService?.announceProvider(blockProviderNamespace(cid), {
|
|
16221
|
+
ttlMs: 120_000,
|
|
16222
|
+
bootstrapMaxPeers: 2,
|
|
16223
|
+
});
|
|
16224
|
+
} catch {
|
|
16225
|
+
// Provider publication is best-effort.
|
|
16226
|
+
}
|
|
16227
|
+
};
|
|
16228
|
+
const announceBlockProviders = async (cids: string[]): Promise<void> => {
|
|
16229
|
+
const batchedAnnounce = fanoutService?.announceProviders;
|
|
16230
|
+
if (typeof batchedAnnounce === "function") {
|
|
16231
|
+
const namespaces = function* () {
|
|
16232
|
+
for (const cid of cids) yield blockProviderNamespace(cid);
|
|
16233
|
+
};
|
|
16234
|
+
await batchedAnnounce.call(fanoutService, namespaces(), {
|
|
16235
|
+
ttlMs: 120_000,
|
|
16236
|
+
bootstrapMaxPeers: 2,
|
|
16237
|
+
});
|
|
16238
|
+
return;
|
|
16239
|
+
}
|
|
16240
|
+
|
|
16241
|
+
// Tolerate a skewed runtime that supplied an older FanoutTree service.
|
|
16242
|
+
let nextIndex = 0;
|
|
16243
|
+
const worker = async () => {
|
|
16244
|
+
for (;;) {
|
|
16245
|
+
const index = nextIndex++;
|
|
16246
|
+
if (index >= cids.length) return;
|
|
16247
|
+
await announceBlockProvider(cids[index]!);
|
|
16248
|
+
}
|
|
16249
|
+
};
|
|
16250
|
+
await Promise.all(
|
|
16251
|
+
Array.from({ length: Math.min(8, cids.length) }, () => worker()),
|
|
16252
|
+
);
|
|
16253
|
+
};
|
|
14878
16254
|
const [replicationIndex, logIndex] = await Promise.all([
|
|
14879
16255
|
logScope.scope("replication"),
|
|
14880
16256
|
logScope.scope("log"),
|
|
@@ -14974,17 +16350,23 @@ export class SharedLog<
|
|
|
14974
16350
|
|
|
14975
16351
|
let directoryProviders: string[] = [];
|
|
14976
16352
|
try {
|
|
14977
|
-
|
|
14978
|
-
|
|
14979
|
-
|
|
14980
|
-
|
|
14981
|
-
|
|
14982
|
-
|
|
14983
|
-
|
|
14984
|
-
|
|
14985
|
-
|
|
14986
|
-
|
|
14987
|
-
)
|
|
16353
|
+
const query = (namespace: string) =>
|
|
16354
|
+
fanoutService?.queryProviders(namespace, {
|
|
16355
|
+
want: maxPeers,
|
|
16356
|
+
timeoutMs: 2_000,
|
|
16357
|
+
queryTimeoutMs: 500,
|
|
16358
|
+
bootstrapMaxPeers: 2,
|
|
16359
|
+
signal: opts?.signal,
|
|
16360
|
+
}) ?? Promise.resolve([]);
|
|
16361
|
+
const results = await Promise.allSettled([
|
|
16362
|
+
query(blockProviderNamespace(cid)),
|
|
16363
|
+
query(logProviderNamespace),
|
|
16364
|
+
]);
|
|
16365
|
+
for (const result of results) {
|
|
16366
|
+
if (result.status === "fulfilled") {
|
|
16367
|
+
directoryProviders.push(...result.value);
|
|
16368
|
+
}
|
|
16369
|
+
}
|
|
14988
16370
|
} catch {
|
|
14989
16371
|
// Ignore discovery failures; local evidence remains usable.
|
|
14990
16372
|
}
|
|
@@ -14992,11 +16374,7 @@ export class SharedLog<
|
|
|
14992
16374
|
const selected: string[] = [];
|
|
14993
16375
|
const selectedSet = new Set<string>();
|
|
14994
16376
|
const add = (peer: string | undefined) => {
|
|
14995
|
-
if (
|
|
14996
|
-
!peer ||
|
|
14997
|
-
selectedSet.has(peer) ||
|
|
14998
|
-
selected.length >= maxPeers
|
|
14999
|
-
) {
|
|
16377
|
+
if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
|
|
15000
16378
|
return;
|
|
15001
16379
|
}
|
|
15002
16380
|
selectedSet.add(peer);
|
|
@@ -15014,31 +16392,46 @@ export class SharedLog<
|
|
|
15014
16392
|
return selected;
|
|
15015
16393
|
},
|
|
15016
16394
|
watchProviders: fanoutService
|
|
15017
|
-
? (cid, opts) =>
|
|
15018
|
-
|
|
15019
|
-
|
|
15020
|
-
|
|
15021
|
-
|
|
15022
|
-
|
|
15023
|
-
|
|
15024
|
-
|
|
15025
|
-
|
|
15026
|
-
|
|
15027
|
-
|
|
15028
|
-
|
|
15029
|
-
? async (cid) => {
|
|
15030
|
-
// Best-effort directory announce for "get without remote.from" workflows.
|
|
16395
|
+
? (cid, opts) => {
|
|
16396
|
+
const watch = (namespace: string) =>
|
|
16397
|
+
fanoutService.watchProviders(namespace, {
|
|
16398
|
+
signal: opts.signal,
|
|
16399
|
+
want: 8,
|
|
16400
|
+
ttlMs: 10_000,
|
|
16401
|
+
renewIntervalMs: 5_000,
|
|
16402
|
+
bootstrapMaxPeers: 2,
|
|
16403
|
+
onProviders: (providers) =>
|
|
16404
|
+
opts.onProviders(providers.map((provider) => provider.hash)),
|
|
16405
|
+
});
|
|
16406
|
+
const cidWatch = watch(blockProviderNamespace(cid));
|
|
15031
16407
|
try {
|
|
15032
|
-
|
|
15033
|
-
|
|
15034
|
-
{
|
|
15035
|
-
|
|
15036
|
-
|
|
16408
|
+
const logWatch = watch(logProviderNamespace);
|
|
16409
|
+
return {
|
|
16410
|
+
close: () => {
|
|
16411
|
+
cidWatch.close();
|
|
16412
|
+
logWatch.close();
|
|
15037
16413
|
},
|
|
15038
|
-
|
|
16414
|
+
};
|
|
16415
|
+
} catch (error) {
|
|
16416
|
+
cidWatch.close();
|
|
16417
|
+
throw error;
|
|
16418
|
+
}
|
|
16419
|
+
}
|
|
16420
|
+
: undefined,
|
|
16421
|
+
onPut: fanoutService ? announceBlockProvider : undefined,
|
|
16422
|
+
onPutMany: fanoutService
|
|
16423
|
+
? (cids) => {
|
|
16424
|
+
// A renewable log-wide provider lease makes every CID in a
|
|
16425
|
+
// stored batch discoverable to current readers. Retain the
|
|
16426
|
+
// per-CID directory publications for released readers that only
|
|
16427
|
+
// know the legacy namespace; the bounded workers avoid creating
|
|
16428
|
+
// one in-flight promise per block.
|
|
16429
|
+
try {
|
|
16430
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
15039
16431
|
} catch {
|
|
15040
16432
|
// ignore announce failures
|
|
15041
16433
|
}
|
|
16434
|
+
return announceBlockProviders(cids);
|
|
15042
16435
|
}
|
|
15043
16436
|
: undefined,
|
|
15044
16437
|
});
|
|
@@ -15290,6 +16683,7 @@ export class SharedLog<
|
|
|
15290
16683
|
},
|
|
15291
16684
|
indexer: logIndex,
|
|
15292
16685
|
});
|
|
16686
|
+
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
15293
16687
|
try {
|
|
15294
16688
|
const recovered =
|
|
15295
16689
|
await this.recoverNativeStrictDurableTransactionIntent();
|
|
@@ -16088,13 +17482,12 @@ export class SharedLog<
|
|
|
16088
17482
|
this.topic,
|
|
16089
17483
|
);
|
|
16090
17484
|
// We do this here, because these calls requires this.closed == false
|
|
16091
|
-
void this.pruneOfflineReplicators()
|
|
16092
|
-
|
|
16093
|
-
|
|
16094
|
-
|
|
16095
|
-
|
|
16096
|
-
|
|
16097
|
-
});
|
|
17485
|
+
void this.pruneOfflineReplicators().catch((error) => {
|
|
17486
|
+
if (isNotStartedError(error as Error)) {
|
|
17487
|
+
return;
|
|
17488
|
+
}
|
|
17489
|
+
logger.error(error);
|
|
17490
|
+
});
|
|
16098
17491
|
|
|
16099
17492
|
this._liveness.startReplicatorLivenessSweep();
|
|
16100
17493
|
|
|
@@ -16413,7 +17806,8 @@ export class SharedLog<
|
|
|
16413
17806
|
for (const peer of reachable) {
|
|
16414
17807
|
if (confirmed?.has(peer)) liveEvidenceTiers[0].push(peer);
|
|
16415
17808
|
else if (contacted?.has(peer)) liveEvidenceTiers[1].push(peer);
|
|
16416
|
-
else if (this.uniqueReplicators.has(peer))
|
|
17809
|
+
else if (this.uniqueReplicators.has(peer))
|
|
17810
|
+
liveEvidenceTiers[2].push(peer);
|
|
16417
17811
|
}
|
|
16418
17812
|
const liveKnown: string[] = [];
|
|
16419
17813
|
for (let tier = 0; tier < liveEvidenceTiers.length; tier++) {
|
|
@@ -16983,8 +18377,7 @@ export class SharedLog<
|
|
|
16983
18377
|
): Promise<void> {
|
|
16984
18378
|
if (!gids) return;
|
|
16985
18379
|
const candidates = [...new Set(gids)].filter(
|
|
16986
|
-
(gid) =>
|
|
16987
|
-
history === this._gidPeersHistory && !!gid && history.has(gid),
|
|
18380
|
+
(gid) => history === this._gidPeersHistory && !!gid && history.has(gid),
|
|
16988
18381
|
);
|
|
16989
18382
|
if (candidates.length === 0) return;
|
|
16990
18383
|
const hasHeads = await this.hasAnyHeadForGidSets(
|
|
@@ -17018,9 +18411,7 @@ export class SharedLog<
|
|
|
17018
18411
|
for (const gid of gids) {
|
|
17019
18412
|
if (!gid || !history.has(gid)) continue;
|
|
17020
18413
|
if (state.pending.has(gid)) continue;
|
|
17021
|
-
if (
|
|
17022
|
-
state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY
|
|
17023
|
-
) {
|
|
18414
|
+
if (state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY) {
|
|
17024
18415
|
// History is only a suppression memo. Under sustained synchronous
|
|
17025
18416
|
// production, forgetting an overflow row is the bounded, safe fallback:
|
|
17026
18417
|
// it can cause redundant delivery but cannot affect ownership or data.
|
|
@@ -17819,6 +19210,9 @@ export class SharedLog<
|
|
|
17819
19210
|
this._peerSyncCapabilities?.clear();
|
|
17820
19211
|
this._peerSyncCapabilitySessions?.clear();
|
|
17821
19212
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
19213
|
+
this._persistedReceiptStorage = undefined;
|
|
19214
|
+
this._persistedReceiptRequestsInFlight?.clear();
|
|
19215
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
17822
19216
|
this._v2Receive?.clearForClose();
|
|
17823
19217
|
this._v2Send?.clearForClose();
|
|
17824
19218
|
this._liveRawGossipBatches?.clear();
|
|
@@ -18591,7 +19985,11 @@ export class SharedLog<
|
|
|
18591
19985
|
async onMessage(
|
|
18592
19986
|
msg: TransportMessage,
|
|
18593
19987
|
context: RequestContext,
|
|
18594
|
-
): Promise<void
|
|
19988
|
+
): Promise<void>;
|
|
19989
|
+
async onMessage(
|
|
19990
|
+
msg: TransportMessage,
|
|
19991
|
+
context: RequestContext,
|
|
19992
|
+
): Promise<any> {
|
|
18595
19993
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
18596
19994
|
? msg
|
|
18597
19995
|
: undefined;
|
|
@@ -18667,6 +20065,7 @@ export class SharedLog<
|
|
|
18667
20065
|
if (
|
|
18668
20066
|
!context.from.equals(this.node.identity.publicKey) &&
|
|
18669
20067
|
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
20068
|
+
!(msg instanceof RequestPersistedEntriesV1) &&
|
|
18670
20069
|
!isReplicationInfoV2Message(msg)
|
|
18671
20070
|
) {
|
|
18672
20071
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
@@ -20113,24 +21512,30 @@ export class SharedLog<
|
|
|
20113
21512
|
);
|
|
20114
21513
|
}
|
|
20115
21514
|
if (syncProfile) {
|
|
20116
|
-
emitSyncProfileDuration(
|
|
20117
|
-
|
|
20118
|
-
|
|
20119
|
-
|
|
20120
|
-
|
|
20121
|
-
|
|
20122
|
-
|
|
20123
|
-
|
|
20124
|
-
|
|
21515
|
+
emitSyncProfileDuration(
|
|
21516
|
+
syncProfile,
|
|
21517
|
+
coordinatePersistStartedAt,
|
|
21518
|
+
{
|
|
21519
|
+
name: "sharedLog.receive.coordinatePersist",
|
|
21520
|
+
component: "shared-log",
|
|
21521
|
+
entries: entriesToPersist.length,
|
|
21522
|
+
messages: 1,
|
|
21523
|
+
details: {
|
|
21524
|
+
reusedLeaderPlans: reusableCoordinatePersistItemCount,
|
|
21525
|
+
nativeBackboneOnly:
|
|
21526
|
+
nativeBackboneOnlyPersistedHashes?.size ?? 0,
|
|
21527
|
+
},
|
|
20125
21528
|
},
|
|
20126
|
-
|
|
21529
|
+
);
|
|
20127
21530
|
}
|
|
20128
21531
|
for (const hash of admittedHashes) {
|
|
20129
21532
|
confirmedHashes.add(hash);
|
|
20130
21533
|
}
|
|
20131
21534
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
20132
21535
|
const ownershipChangedDuringReceive =
|
|
20133
|
-
!this.isReceiveOwnershipSnapshotStable(
|
|
21536
|
+
!this.isReceiveOwnershipSnapshotStable(
|
|
21537
|
+
receiveOwnershipRevision,
|
|
21538
|
+
);
|
|
20134
21539
|
if (ownershipChangedDuringReceive) {
|
|
20135
21540
|
const freshAuditRevision =
|
|
20136
21541
|
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
@@ -20307,7 +21712,13 @@ export class SharedLog<
|
|
|
20307
21712
|
};
|
|
20308
21713
|
// The prelude already threw when `context.from` was missing.
|
|
20309
21714
|
const laneRequestContext = context as ReceiveRequestContext;
|
|
20310
|
-
if (msg instanceof
|
|
21715
|
+
if (msg instanceof RequestPersistedEntriesV1) {
|
|
21716
|
+
return await this.handleRequestPersistedEntriesV1(
|
|
21717
|
+
msg,
|
|
21718
|
+
laneRequestContext,
|
|
21719
|
+
lane,
|
|
21720
|
+
);
|
|
21721
|
+
} else if (msg instanceof RequestIPruneV2) {
|
|
20311
21722
|
await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
|
|
20312
21723
|
} else if (msg instanceof ResponseIPruneV2) {
|
|
20313
21724
|
await this.handleResponseIPruneV2(msg, laneRequestContext, lane);
|
|
@@ -20514,6 +21925,322 @@ export class SharedLog<
|
|
|
20514
21925
|
// wire-stash release, lease release, durable-poison recheck).
|
|
20515
21926
|
// -----------------------------------------------------------------
|
|
20516
21927
|
|
|
21928
|
+
private isPersistedReceiptRequestSessionCurrent(
|
|
21929
|
+
request: RequestPersistedEntriesV1,
|
|
21930
|
+
context: ReceiveRequestContext,
|
|
21931
|
+
lane: ReceiveLaneContext,
|
|
21932
|
+
): boolean {
|
|
21933
|
+
const session = lane.session;
|
|
21934
|
+
return (
|
|
21935
|
+
!!this._persistedReceiptStorage &&
|
|
21936
|
+
!context.from.equals(this.node.identity.publicKey) &&
|
|
21937
|
+
request.expectedReceiverSession === this.ownTransportSession() &&
|
|
21938
|
+
session !== null &&
|
|
21939
|
+
session.phase === "open" &&
|
|
21940
|
+
this._peerSessions.isCurrent(lane.fromHash, session) &&
|
|
21941
|
+
!this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
|
|
21942
|
+
this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
|
|
21943
|
+
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
21944
|
+
context.message.header.session &&
|
|
21945
|
+
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
21946
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
21947
|
+
);
|
|
21948
|
+
}
|
|
21949
|
+
|
|
21950
|
+
private admitPersistedReceiptIngress(
|
|
21951
|
+
peer: string,
|
|
21952
|
+
transportSession: bigint,
|
|
21953
|
+
hashCount: number,
|
|
21954
|
+
now = Date.now(),
|
|
21955
|
+
): boolean {
|
|
21956
|
+
// Charge malformed empty/oversized vectors too. The request was already
|
|
21957
|
+
// decoded by the transport, so letting an invalid shape bypass this bucket
|
|
21958
|
+
// would leave an authenticated peer with an unmetered validation/logging
|
|
21959
|
+
// path. Clamp only the accounting cost; shape validation still rejects the
|
|
21960
|
+
// request below.
|
|
21961
|
+
const hashCost = Math.max(
|
|
21962
|
+
1,
|
|
21963
|
+
Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount),
|
|
21964
|
+
);
|
|
21965
|
+
let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
|
|
21966
|
+
if (!nodeBudget) {
|
|
21967
|
+
nodeBudget = {
|
|
21968
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
21969
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
21970
|
+
refilledAt: now,
|
|
21971
|
+
peerSessions: new Map(),
|
|
21972
|
+
};
|
|
21973
|
+
persistedReceiptIngressBudgets.set(this.node, nodeBudget);
|
|
21974
|
+
}
|
|
21975
|
+
refillPersistedReceiptIngressBucket(
|
|
21976
|
+
nodeBudget,
|
|
21977
|
+
now,
|
|
21978
|
+
PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
21979
|
+
PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
21980
|
+
PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND,
|
|
21981
|
+
PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND,
|
|
21982
|
+
);
|
|
21983
|
+
|
|
21984
|
+
const peerSessionKey = `${peer}\0${transportSession}`;
|
|
21985
|
+
let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
|
|
21986
|
+
if (!peerBudget) {
|
|
21987
|
+
while (
|
|
21988
|
+
nodeBudget.peerSessions.size >=
|
|
21989
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
|
|
21990
|
+
) {
|
|
21991
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
21992
|
+
if (oldest === undefined) break;
|
|
21993
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
21994
|
+
}
|
|
21995
|
+
peerBudget = {
|
|
21996
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
21997
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
21998
|
+
refilledAt: now,
|
|
21999
|
+
};
|
|
22000
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
22001
|
+
} else {
|
|
22002
|
+
// Refresh insertion order so bounded eviction prefers inactive sessions.
|
|
22003
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
22004
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
22005
|
+
}
|
|
22006
|
+
refillPersistedReceiptIngressBucket(
|
|
22007
|
+
peerBudget,
|
|
22008
|
+
now,
|
|
22009
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
22010
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
22011
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
|
|
22012
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
|
|
22013
|
+
);
|
|
22014
|
+
|
|
22015
|
+
if (
|
|
22016
|
+
nodeBudget.requestTokens < 1 ||
|
|
22017
|
+
nodeBudget.hashTokens < hashCost ||
|
|
22018
|
+
peerBudget.requestTokens < 1 ||
|
|
22019
|
+
peerBudget.hashTokens < hashCost
|
|
22020
|
+
) {
|
|
22021
|
+
return false;
|
|
22022
|
+
}
|
|
22023
|
+
nodeBudget.requestTokens -= 1;
|
|
22024
|
+
nodeBudget.hashTokens -= hashCost;
|
|
22025
|
+
peerBudget.requestTokens -= 1;
|
|
22026
|
+
peerBudget.hashTokens -= hashCost;
|
|
22027
|
+
return true;
|
|
22028
|
+
}
|
|
22029
|
+
|
|
22030
|
+
private async handleRequestPersistedEntriesV1(
|
|
22031
|
+
request: RequestPersistedEntriesV1,
|
|
22032
|
+
context: ReceiveRequestContext,
|
|
22033
|
+
lane: ReceiveLaneContext,
|
|
22034
|
+
): Promise<ConfirmEntriesMessage | undefined> {
|
|
22035
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
22036
|
+
return undefined;
|
|
22037
|
+
}
|
|
22038
|
+
if (
|
|
22039
|
+
!this.admitPersistedReceiptIngress(
|
|
22040
|
+
lane.fromHash,
|
|
22041
|
+
context.message.header.session,
|
|
22042
|
+
request.hashes.length,
|
|
22043
|
+
)
|
|
22044
|
+
) {
|
|
22045
|
+
return undefined;
|
|
22046
|
+
}
|
|
22047
|
+
// Charge the request before shape validation and attacker-controlled CID
|
|
22048
|
+
// parsing. Invalid shapes are rejected quietly here so they cannot turn the
|
|
22049
|
+
// outer receive error logger into a post-budget work amplifier.
|
|
22050
|
+
try {
|
|
22051
|
+
this.validatePersistedReceiptRequestShape(request);
|
|
22052
|
+
} catch {
|
|
22053
|
+
return undefined;
|
|
22054
|
+
}
|
|
22055
|
+
if (!this.hasValidPersistedReceiptHashes(request)) {
|
|
22056
|
+
return undefined;
|
|
22057
|
+
}
|
|
22058
|
+
const peerInFlight =
|
|
22059
|
+
this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
|
|
22060
|
+
if (
|
|
22061
|
+
peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
|
|
22062
|
+
this._persistedReceiptRequestsInFlightTotal >=
|
|
22063
|
+
MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL
|
|
22064
|
+
) {
|
|
22065
|
+
return undefined;
|
|
22066
|
+
}
|
|
22067
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
|
|
22068
|
+
this._persistedReceiptRequestsInFlightTotal++;
|
|
22069
|
+
|
|
22070
|
+
try {
|
|
22071
|
+
// A positive receipt requires the block itself. Reject entirely absent
|
|
22072
|
+
// batches before the serialized mutation lane and, critically, before a
|
|
22073
|
+
// request can force unrelated pending index/coordinate journals durable.
|
|
22074
|
+
// A concurrent put can yield a harmless false negative: the sender retries.
|
|
22075
|
+
const blockPresence = await this.remoteBlocks.localStore.hasMany(
|
|
22076
|
+
request.hashes,
|
|
22077
|
+
);
|
|
22078
|
+
const presentBlockHashes = request.hashes.filter(
|
|
22079
|
+
(_hash, index) => blockPresence[index] === true,
|
|
22080
|
+
);
|
|
22081
|
+
if (
|
|
22082
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22083
|
+
) {
|
|
22084
|
+
return undefined;
|
|
22085
|
+
}
|
|
22086
|
+
if (presentBlockHashes.length === 0) {
|
|
22087
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22088
|
+
}
|
|
22089
|
+
this._liveness.markReplicatorActivity(lane.fromHash);
|
|
22090
|
+
return await this.withReplicationRangeMutationQueue(async () => {
|
|
22091
|
+
if (
|
|
22092
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22093
|
+
) {
|
|
22094
|
+
return undefined;
|
|
22095
|
+
}
|
|
22096
|
+
this.throwIfNativeDurableCommitFailed();
|
|
22097
|
+
const storage = this.resolvePersistedReceiptStorage();
|
|
22098
|
+
if (!storage) {
|
|
22099
|
+
return undefined;
|
|
22100
|
+
}
|
|
22101
|
+
|
|
22102
|
+
await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
|
|
22103
|
+
await this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
22104
|
+
const candidates = new Map(
|
|
22105
|
+
await Promise.all(
|
|
22106
|
+
presentBlockHashes.map(async (hash) => {
|
|
22107
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
22108
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22109
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22110
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(
|
|
22111
|
+
hash,
|
|
22112
|
+
),
|
|
22113
|
+
]);
|
|
22114
|
+
return [hash, { blockPresent, lowerRow, coordinate }] as const;
|
|
22115
|
+
}),
|
|
22116
|
+
),
|
|
22117
|
+
);
|
|
22118
|
+
const presentHashes = presentBlockHashes.filter((hash) => {
|
|
22119
|
+
const candidate = candidates.get(hash)!;
|
|
22120
|
+
return (
|
|
22121
|
+
candidate.blockPresent &&
|
|
22122
|
+
candidate.lowerRow != null &&
|
|
22123
|
+
candidate.coordinate != null
|
|
22124
|
+
);
|
|
22125
|
+
});
|
|
22126
|
+
if (presentHashes.length === 0) {
|
|
22127
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22128
|
+
}
|
|
22129
|
+
await Promise.all(
|
|
22130
|
+
[...new Set([storage.block, storage.lower, storage.coordinate])].map(
|
|
22131
|
+
(store) => store.barrier(),
|
|
22132
|
+
),
|
|
22133
|
+
);
|
|
22134
|
+
this.throwIfNativeDurableCommitFailed();
|
|
22135
|
+
if (
|
|
22136
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22137
|
+
) {
|
|
22138
|
+
return undefined;
|
|
22139
|
+
}
|
|
22140
|
+
|
|
22141
|
+
const ownershipRevision =
|
|
22142
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
22143
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
22144
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22145
|
+
}
|
|
22146
|
+
|
|
22147
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
22148
|
+
const confirmed: string[] = [];
|
|
22149
|
+
for (const hash of presentHashes) {
|
|
22150
|
+
const candidate = candidates.get(hash)!;
|
|
22151
|
+
if (
|
|
22152
|
+
!candidate.blockPresent ||
|
|
22153
|
+
!candidate.lowerRow ||
|
|
22154
|
+
!candidate.coordinate ||
|
|
22155
|
+
this._checkedPrune.hasActiveWork(hash) ||
|
|
22156
|
+
!this.isPersistedReceiptRequestSessionCurrent(
|
|
22157
|
+
request,
|
|
22158
|
+
context,
|
|
22159
|
+
lane,
|
|
22160
|
+
) ||
|
|
22161
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
22162
|
+
) {
|
|
22163
|
+
continue;
|
|
22164
|
+
}
|
|
22165
|
+
|
|
22166
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
22167
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22168
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22169
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
22170
|
+
]);
|
|
22171
|
+
if (!blockPresent || !lowerRow || !coordinate) {
|
|
22172
|
+
continue;
|
|
22173
|
+
}
|
|
22174
|
+
|
|
22175
|
+
const replicas = decodeReplicas(coordinate).getValue(this);
|
|
22176
|
+
const leaders = await this.findLeadersFromEntry(
|
|
22177
|
+
coordinate,
|
|
22178
|
+
replicas,
|
|
22179
|
+
{ freshLeaderPlan: true },
|
|
22180
|
+
lane.ownershipLifecycleController,
|
|
22181
|
+
);
|
|
22182
|
+
if (!leaders.has(selfHash)) {
|
|
22183
|
+
continue;
|
|
22184
|
+
}
|
|
22185
|
+
if (
|
|
22186
|
+
!this._checkedPrune.hasActiveWork(hash) &&
|
|
22187
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
|
|
22188
|
+
this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22189
|
+
) {
|
|
22190
|
+
confirmed.push(hash);
|
|
22191
|
+
}
|
|
22192
|
+
}
|
|
22193
|
+
|
|
22194
|
+
if (
|
|
22195
|
+
confirmed.length === 0 ||
|
|
22196
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
22197
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22198
|
+
) {
|
|
22199
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22200
|
+
}
|
|
22201
|
+
const finalRows = await Promise.all(
|
|
22202
|
+
confirmed.map(async (hash) => {
|
|
22203
|
+
const [block, lower, coordinate] = await Promise.all([
|
|
22204
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22205
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22206
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
22207
|
+
]);
|
|
22208
|
+
return { hash, block, lower, coordinate };
|
|
22209
|
+
}),
|
|
22210
|
+
);
|
|
22211
|
+
if (
|
|
22212
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
22213
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22214
|
+
) {
|
|
22215
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22216
|
+
}
|
|
22217
|
+
return new ConfirmEntriesMessage({
|
|
22218
|
+
hashes: finalRows
|
|
22219
|
+
.filter(
|
|
22220
|
+
(row) =>
|
|
22221
|
+
row.block &&
|
|
22222
|
+
row.lower &&
|
|
22223
|
+
row.coordinate &&
|
|
22224
|
+
!this._checkedPrune.hasActiveWork(row.hash),
|
|
22225
|
+
)
|
|
22226
|
+
.map((row) => row.hash),
|
|
22227
|
+
});
|
|
22228
|
+
}, lane.ownershipLifecycleController);
|
|
22229
|
+
} finally {
|
|
22230
|
+
const remaining =
|
|
22231
|
+
(this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
|
|
22232
|
+
if (remaining <= 0) {
|
|
22233
|
+
this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
|
|
22234
|
+
} else {
|
|
22235
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
|
|
22236
|
+
}
|
|
22237
|
+
this._persistedReceiptRequestsInFlightTotal = Math.max(
|
|
22238
|
+
0,
|
|
22239
|
+
this._persistedReceiptRequestsInFlightTotal - 1,
|
|
22240
|
+
);
|
|
22241
|
+
}
|
|
22242
|
+
}
|
|
22243
|
+
|
|
20517
22244
|
private async handleRequestIPruneV2(
|
|
20518
22245
|
msg: RequestIPruneV2,
|
|
20519
22246
|
context: ReceiveRequestContext,
|