@peerbit/shared-log 16.0.13 → 16.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/src/coordinate-persistence.d.ts +10 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -1
- package/dist/src/coordinate-persistence.js +18 -0
- package/dist/src/coordinate-persistence.js.map +1 -1
- package/dist/src/errors.d.ts +11 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +23 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +6 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +6 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +60 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1325 -144
- package/dist/src/index.js.map +1 -1
- package/dist/src/native-write-through-block-store.d.ts +4 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -1
- package/dist/src/native-write-through-block-store.js +13 -0
- package/dist/src/native-write-through-block-store.js.map +1 -1
- package/dist/src/sync/simple.d.ts +15 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +44 -0
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +18 -18
- package/src/coordinate-persistence.ts +21 -0
- package/src/errors.ts +35 -0
- package/src/exchange-heads.ts +7 -0
- package/src/index.ts +2002 -264
- package/src/native-write-through-block-store.ts +14 -0
- package/src/sync/simple.ts +22 -0
package/src/index.ts
CHANGED
|
@@ -115,6 +115,7 @@ import {
|
|
|
115
115
|
AbortError,
|
|
116
116
|
TimeoutError,
|
|
117
117
|
debounceFixedInterval,
|
|
118
|
+
delay,
|
|
118
119
|
waitFor,
|
|
119
120
|
} from "@peerbit/time";
|
|
120
121
|
import pDefer, { type DeferredPromise } from "p-defer";
|
|
@@ -147,6 +148,7 @@ import {
|
|
|
147
148
|
CompatibilityModeRetiredError,
|
|
148
149
|
NativeDurableCommitError,
|
|
149
150
|
NoPeersError,
|
|
151
|
+
PersistedDeliveryError,
|
|
150
152
|
isNotStartedError,
|
|
151
153
|
} from "./errors.js";
|
|
152
154
|
import {
|
|
@@ -162,6 +164,7 @@ import {
|
|
|
162
164
|
RequestIPruneV2,
|
|
163
165
|
ResponseIPrune,
|
|
164
166
|
ResponseIPruneV2,
|
|
167
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS,
|
|
165
168
|
SYNC_CAPABILITY_RAW_EXCHANGE_HEADS,
|
|
166
169
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY,
|
|
167
170
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM,
|
|
@@ -306,6 +309,7 @@ import {
|
|
|
306
309
|
import {
|
|
307
310
|
ConfirmEntriesMessage,
|
|
308
311
|
RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS,
|
|
312
|
+
RequestPersistedEntriesV1,
|
|
309
313
|
SYNC_MESSAGE_PRIORITY,
|
|
310
314
|
SimpleSyncronizer,
|
|
311
315
|
} from "./sync/simple.js";
|
|
@@ -449,6 +453,7 @@ export {
|
|
|
449
453
|
CompatibilityModeRetiredError,
|
|
450
454
|
NativeDurableCommitError,
|
|
451
455
|
NoPeersError,
|
|
456
|
+
PersistedDeliveryError,
|
|
452
457
|
};
|
|
453
458
|
export { MAX_U32, MAX_U64, type NumberFromType };
|
|
454
459
|
export type {
|
|
@@ -498,25 +503,36 @@ const joinNativeCoordinateDirectory = (
|
|
|
498
503
|
`${nodeDirectory.replace(/[/\\]+$/, "")}/coordinates/${fsSafeLogId}`;
|
|
499
504
|
|
|
500
505
|
type DurableBlockSublevelStore = {
|
|
506
|
+
readonly supportsCrashSafeJournalCheckpoint?: boolean;
|
|
501
507
|
sublevel(
|
|
502
508
|
name: string,
|
|
503
509
|
options?: {
|
|
504
510
|
compactOnClose?: boolean;
|
|
505
511
|
compactOnCloseMinJournalBytes?: number;
|
|
512
|
+
compactMaxJournalBytes?: number;
|
|
506
513
|
durability?: "normal" | "strict";
|
|
507
514
|
},
|
|
508
515
|
): MaybePromise<AnyStore>;
|
|
509
516
|
};
|
|
510
517
|
|
|
518
|
+
const defaultNativeEntryBlockCompactMaxJournalBytes = 64 * 1024 * 1024;
|
|
519
|
+
|
|
511
520
|
const createNativeDurableBlockStore = async (
|
|
512
521
|
storage: DurableBlockSublevelStore,
|
|
513
522
|
): Promise<AnyBlockStore> =>
|
|
514
523
|
new AnyBlockStore(
|
|
515
524
|
await storage.sublevel("blocks", {
|
|
516
|
-
// Strict mirrors remain WAL-backed
|
|
517
|
-
//
|
|
518
|
-
//
|
|
525
|
+
// Strict mirrors remain WAL-backed. On POSIX Node, checkpoint only the
|
|
526
|
+
// historical suffix through the Rust store's fsync + atomic-rename path;
|
|
527
|
+
// browsers/custom backends and Windows retain the prior unbounded WAL until
|
|
528
|
+
// they expose an equally strong directory durability barrier.
|
|
519
529
|
compactOnClose: false,
|
|
530
|
+
...(storage.supportsCrashSafeJournalCheckpoint === true
|
|
531
|
+
? {
|
|
532
|
+
compactMaxJournalBytes:
|
|
533
|
+
defaultNativeEntryBlockCompactMaxJournalBytes,
|
|
534
|
+
}
|
|
535
|
+
: {}),
|
|
520
536
|
// A native append is acknowledged only after this mirror resolves. The
|
|
521
537
|
// Rust store's normal immutable fast path may resolve before its WAL write;
|
|
522
538
|
// strict mode waits for the journal write and sync, closing the SIGKILL gap.
|
|
@@ -540,6 +556,7 @@ type LeaderMap = Map<string, { intersecting: boolean }>;
|
|
|
540
556
|
type LeaderSelectionOptions<R extends "u32" | "u64"> = {
|
|
541
557
|
roleAge?: number;
|
|
542
558
|
candidates?: Iterable<string>;
|
|
559
|
+
freshLeaderPlan?: boolean;
|
|
543
560
|
onLeader?: (key: string) => void;
|
|
544
561
|
persist?:
|
|
545
562
|
| {
|
|
@@ -687,6 +704,10 @@ type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
|
|
|
687
704
|
};
|
|
688
705
|
};
|
|
689
706
|
|
|
707
|
+
type PersistedDeliveryPlanningEntry<T, R extends "u32" | "u64"> =
|
|
708
|
+
| ShallowOrFullEntry<T>
|
|
709
|
+
| EntryReplicated<R>;
|
|
710
|
+
|
|
690
711
|
type NativeBackboneSimpleDocumentProjectionPlan = {
|
|
691
712
|
documentVariantType?: "u8" | "string";
|
|
692
713
|
documentVariantValue?: string;
|
|
@@ -897,10 +918,15 @@ const nativeStrictDurableTransactionJournalRecordBytes = (
|
|
|
897
918
|
return new TextEncoder().encode(JSON.stringify(record));
|
|
898
919
|
};
|
|
899
920
|
|
|
921
|
+
type TrustedLocalCommitEvidence = {
|
|
922
|
+
committedHashes: Set<string>;
|
|
923
|
+
};
|
|
924
|
+
|
|
900
925
|
type PreparedPayloadCommitOnlyProperties =
|
|
901
926
|
NativeBackboneDocumentCommitOptions & {
|
|
902
927
|
skipMissingNextJoin?: boolean;
|
|
903
928
|
resolveTrimmedEntries?: boolean;
|
|
929
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
904
930
|
};
|
|
905
931
|
|
|
906
932
|
type PreparedPayloadsManyIndependentProperties<T> = {
|
|
@@ -909,6 +935,7 @@ type PreparedPayloadsManyIndependentProperties<T> = {
|
|
|
909
935
|
nexts?: ShallowOrFullEntry<T>[][];
|
|
910
936
|
nativeBackboneDocumentIndexes?: NativeBackboneDocumentIndexCommitInput[];
|
|
911
937
|
retainMaterializationBytes?: boolean;
|
|
938
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
912
939
|
};
|
|
913
940
|
|
|
914
941
|
type PreparedPayloadCommitOnlyResult<T, R extends "u32" | "u64"> = {
|
|
@@ -1670,10 +1697,23 @@ export type Args<
|
|
|
1670
1697
|
: "u32",
|
|
1671
1698
|
> = LogProperties<T> & LogEvents<T> & SharedLogOptions<T, D, R>;
|
|
1672
1699
|
|
|
1673
|
-
|
|
1700
|
+
/**
|
|
1701
|
+
* `persisted` waits, after the local commit, for `minAcks` distinct remote
|
|
1702
|
+
* leaders that advertised crash-safe receipt support. Each receipt proves that
|
|
1703
|
+
* the exact block, lower-log row, and replication-coordinate row crossed that
|
|
1704
|
+
* peer's storage barriers at the receipt instant. It is cooperative-peer
|
|
1705
|
+
* evidence, not a Byzantine proof or a promise that the peer will retain the
|
|
1706
|
+
* entry forever. Only current capable leaders count.
|
|
1707
|
+
*/
|
|
1708
|
+
export type DeliveryReliability = "ack" | "best-effort" | "persisted";
|
|
1674
1709
|
|
|
1675
1710
|
export type DeliveryOptions = {
|
|
1676
1711
|
reliability?: DeliveryReliability;
|
|
1712
|
+
/**
|
|
1713
|
+
* Required for persisted delivery; counts distinct current remote leaders.
|
|
1714
|
+
* This does not increase the entry's replication degree, so the configured
|
|
1715
|
+
* replication must make at least this many capable remote leaders eligible.
|
|
1716
|
+
*/
|
|
1677
1717
|
minAcks?: number;
|
|
1678
1718
|
requireRecipients?: boolean;
|
|
1679
1719
|
/**
|
|
@@ -1681,10 +1721,131 @@ export type DeliveryOptions = {
|
|
|
1681
1721
|
* its control lane, so this only changes the direct/fallback RPC path.
|
|
1682
1722
|
*/
|
|
1683
1723
|
priority?: number;
|
|
1724
|
+
/**
|
|
1725
|
+
* Overall delivery deadline in milliseconds. For persisted delivery it
|
|
1726
|
+
* starts after the local append has returned and includes leader planning,
|
|
1727
|
+
* transfer, receipt requests, and final ownership/session validation. The
|
|
1728
|
+
* omitted persisted default is 10 seconds plus one admission-attempt budget
|
|
1729
|
+
* per transfer chunk and the minimum receipt sender-pacing time implied by
|
|
1730
|
+
* the batch size. An explicit timeout remains exact.
|
|
1731
|
+
*/
|
|
1684
1732
|
timeout?: number;
|
|
1685
1733
|
signal?: AbortSignal;
|
|
1686
1734
|
};
|
|
1687
1735
|
|
|
1736
|
+
type CrashSafeStorageBarrier = {
|
|
1737
|
+
readonly crashSafe: true;
|
|
1738
|
+
barrier(): MaybePromise<void>;
|
|
1739
|
+
};
|
|
1740
|
+
|
|
1741
|
+
type PersistedReceiptStorage = {
|
|
1742
|
+
block: CrashSafeStorageBarrier;
|
|
1743
|
+
lower: CrashSafeStorageBarrier;
|
|
1744
|
+
coordinate: CrashSafeStorageBarrier;
|
|
1745
|
+
};
|
|
1746
|
+
|
|
1747
|
+
type PersistedDeliveryDeadline = {
|
|
1748
|
+
deadline: number;
|
|
1749
|
+
signal: AbortSignal;
|
|
1750
|
+
dispose(): void;
|
|
1751
|
+
};
|
|
1752
|
+
|
|
1753
|
+
const MAX_PERSISTED_RECEIPT_HASHES = 1_024;
|
|
1754
|
+
const MAX_PERSISTED_RECEIPT_HASH_BYTES = 128 * 1_024;
|
|
1755
|
+
const PERSISTED_RECEIPT_CHUNK_SIZE = 512;
|
|
1756
|
+
const PERSISTED_TRANSFER_CHUNK_SIZE = 256;
|
|
1757
|
+
const DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS = 10_000;
|
|
1758
|
+
const MAX_PERSISTED_DELIVERY_TIMEOUT_MS = 2_147_483_647;
|
|
1759
|
+
const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
1760
|
+
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
1761
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
1762
|
+
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
1763
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
1764
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
1765
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
1766
|
+
const PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND = 4_096;
|
|
1767
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY = 32;
|
|
1768
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY = 16_384;
|
|
1769
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND = 16;
|
|
1770
|
+
const PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND = 8_192;
|
|
1771
|
+
const MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS = 256;
|
|
1772
|
+
|
|
1773
|
+
type PersistedReceiptIngressBucket = {
|
|
1774
|
+
requestTokens: number;
|
|
1775
|
+
hashTokens: number;
|
|
1776
|
+
refilledAt: number;
|
|
1777
|
+
};
|
|
1778
|
+
|
|
1779
|
+
type PersistedReceiptNodeIngressBudget = PersistedReceiptIngressBucket & {
|
|
1780
|
+
peerSessions: Map<string, PersistedReceiptIngressBucket>;
|
|
1781
|
+
};
|
|
1782
|
+
|
|
1783
|
+
type PersistedReceiptNodeEgressBudget = {
|
|
1784
|
+
peerSessions: Map<string, PersistedReceiptIngressBucket>;
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
const persistedReceiptIngressBudgets = new WeakMap<
|
|
1788
|
+
object,
|
|
1789
|
+
PersistedReceiptNodeIngressBudget
|
|
1790
|
+
>();
|
|
1791
|
+
|
|
1792
|
+
// Sender pacing mirrors the receiver's per-peer/session allowance. Keeping it
|
|
1793
|
+
// on the Peerbit node (rather than one SharedLog) prevents independent programs
|
|
1794
|
+
// from silently overrunning the same remote receiver together.
|
|
1795
|
+
const persistedReceiptEgressBudgets = new WeakMap<
|
|
1796
|
+
object,
|
|
1797
|
+
PersistedReceiptNodeEgressBudget
|
|
1798
|
+
>();
|
|
1799
|
+
|
|
1800
|
+
const refillPersistedReceiptIngressBucket = (
|
|
1801
|
+
bucket: PersistedReceiptIngressBucket,
|
|
1802
|
+
now: number,
|
|
1803
|
+
requestCapacity: number,
|
|
1804
|
+
hashCapacity: number,
|
|
1805
|
+
requestsPerSecond: number,
|
|
1806
|
+
hashesPerSecond: number,
|
|
1807
|
+
) => {
|
|
1808
|
+
// Never move the refill watermark backwards. Date.now() can jump after a
|
|
1809
|
+
// clock correction; accepting that earlier timestamp would grant the same
|
|
1810
|
+
// elapsed interval again on the next request.
|
|
1811
|
+
if (now <= bucket.refilledAt) {
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
const elapsedSeconds = Math.max(0, now - bucket.refilledAt) / 1_000;
|
|
1815
|
+
if (elapsedSeconds > 0) {
|
|
1816
|
+
bucket.requestTokens = Math.min(
|
|
1817
|
+
requestCapacity,
|
|
1818
|
+
bucket.requestTokens + elapsedSeconds * requestsPerSecond,
|
|
1819
|
+
);
|
|
1820
|
+
bucket.hashTokens = Math.min(
|
|
1821
|
+
hashCapacity,
|
|
1822
|
+
bucket.hashTokens + elapsedSeconds * hashesPerSecond,
|
|
1823
|
+
);
|
|
1824
|
+
}
|
|
1825
|
+
bucket.refilledAt = now;
|
|
1826
|
+
};
|
|
1827
|
+
|
|
1828
|
+
const persistedReceiptPacingFloorMs = (hashCount: number): number => {
|
|
1829
|
+
const boundedHashCount = Math.max(0, Math.floor(hashCount));
|
|
1830
|
+
const requestCount = Math.ceil(
|
|
1831
|
+
boundedHashCount / PERSISTED_RECEIPT_CHUNK_SIZE,
|
|
1832
|
+
);
|
|
1833
|
+
return Math.max(
|
|
1834
|
+
0,
|
|
1835
|
+
((requestCount - PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY) /
|
|
1836
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
1837
|
+
1_000,
|
|
1838
|
+
((boundedHashCount - PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY) /
|
|
1839
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
1840
|
+
1_000,
|
|
1841
|
+
);
|
|
1842
|
+
};
|
|
1843
|
+
|
|
1844
|
+
const persistedTransferAdmissionBudgetMs = (hashCount: number): number =>
|
|
1845
|
+
Math.ceil(
|
|
1846
|
+
Math.max(0, Math.floor(hashCount)) / PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
1847
|
+
) * MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
1848
|
+
|
|
1688
1849
|
export type SharedLogFanoutOptions = {
|
|
1689
1850
|
root?: string;
|
|
1690
1851
|
channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
|
|
@@ -1698,6 +1859,19 @@ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
|
|
|
1698
1859
|
|
|
1699
1860
|
type TrustedLogAppendOptions<T> = AppendOptions<T> & {
|
|
1700
1861
|
__peerbitCanAppendAlreadyValidated?: boolean;
|
|
1862
|
+
__peerbitOnLocalCommit?: (hashes: readonly string[]) => void;
|
|
1863
|
+
};
|
|
1864
|
+
|
|
1865
|
+
const attachTrustedLocalCommitEvidence = <T>(
|
|
1866
|
+
options: AppendOptions<T>,
|
|
1867
|
+
evidence: TrustedLocalCommitEvidence | undefined,
|
|
1868
|
+
): void => {
|
|
1869
|
+
if (!evidence) return;
|
|
1870
|
+
(options as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (hashes) => {
|
|
1871
|
+
for (const hash of hashes) {
|
|
1872
|
+
evidence.committedHashes.add(hash);
|
|
1873
|
+
}
|
|
1874
|
+
};
|
|
1701
1875
|
};
|
|
1702
1876
|
|
|
1703
1877
|
export type SharedAppendOptions<T> =
|
|
@@ -3008,6 +3182,37 @@ export class SharedLog<
|
|
|
3008
3182
|
}
|
|
3009
3183
|
}
|
|
3010
3184
|
|
|
3185
|
+
private async finishCommittedNativeStrictDurableTransaction<TValue>(
|
|
3186
|
+
handle: NativeStrictDurableTransactionHandle | undefined,
|
|
3187
|
+
finish: () => MaybePromise<TValue>,
|
|
3188
|
+
shouldWarnOnRetirementFailure: () => boolean = () => true,
|
|
3189
|
+
): Promise<TValue> {
|
|
3190
|
+
let finishResult: TValue | undefined;
|
|
3191
|
+
let finishError: unknown;
|
|
3192
|
+
let finishFailed = false;
|
|
3193
|
+
try {
|
|
3194
|
+
finishResult = await finish();
|
|
3195
|
+
} catch (error) {
|
|
3196
|
+
finishError = error;
|
|
3197
|
+
finishFailed = true;
|
|
3198
|
+
}
|
|
3199
|
+
try {
|
|
3200
|
+
await this.completeNativeStrictDurableTransaction(handle);
|
|
3201
|
+
} catch (error) {
|
|
3202
|
+
// The acknowledged lower commit is already final. Completion poisons and
|
|
3203
|
+
// releases the in-memory transaction before throwing, so preserve an
|
|
3204
|
+
// earlier post-commit failure while leaving recovery to retire the intent.
|
|
3205
|
+
if (!finishFailed && !shouldWarnOnRetirementFailure()) {
|
|
3206
|
+
throw error;
|
|
3207
|
+
}
|
|
3208
|
+
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
3209
|
+
}
|
|
3210
|
+
if (finishFailed) {
|
|
3211
|
+
throw finishError;
|
|
3212
|
+
}
|
|
3213
|
+
return finishResult as TValue;
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3011
3216
|
private releaseNativeStrictDurableTransaction(
|
|
3012
3217
|
handle: NativeStrictDurableTransactionHandle | undefined,
|
|
3013
3218
|
cause: unknown = new Error(
|
|
@@ -3493,6 +3698,9 @@ export class SharedLog<
|
|
|
3493
3698
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
3494
3699
|
private _peerSyncCapabilitySessions!: Map<string, bigint>;
|
|
3495
3700
|
private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
|
|
3701
|
+
private _persistedReceiptStorage?: PersistedReceiptStorage;
|
|
3702
|
+
private _persistedReceiptRequestsInFlight!: Map<string, number>;
|
|
3703
|
+
private _persistedReceiptRequestsInFlightTotal!: number;
|
|
3496
3704
|
// Pending live raw exchange-head gossip, coalesced per recipient set and
|
|
3497
3705
|
// flushed at the end of the current event-loop turn (or when a batch cap
|
|
3498
3706
|
// is hit). Only used when every recipient advertised raw capability.
|
|
@@ -3567,10 +3775,7 @@ export class SharedLog<
|
|
|
3567
3775
|
return new ReplicationInfoV2SendCoordinator<R>({
|
|
3568
3776
|
getRpc: () => this.rpc,
|
|
3569
3777
|
getSelfKey: () => this.node.identity.publicKey,
|
|
3570
|
-
getSenderTransportSession: () =>
|
|
3571
|
-
BigInt(
|
|
3572
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3573
|
-
),
|
|
3778
|
+
getSenderTransportSession: () => this.ownTransportSession(),
|
|
3574
3779
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
3575
3780
|
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3576
3781
|
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
@@ -3595,12 +3800,37 @@ export class SharedLog<
|
|
|
3595
3800
|
});
|
|
3596
3801
|
}
|
|
3597
3802
|
|
|
3803
|
+
private resolvePersistedReceiptStorage():
|
|
3804
|
+
| PersistedReceiptStorage
|
|
3805
|
+
| undefined {
|
|
3806
|
+
if (this.log.appendDurability !== "strict") {
|
|
3807
|
+
return undefined;
|
|
3808
|
+
}
|
|
3809
|
+
const block = this.remoteBlocks.crashSafeDurability;
|
|
3810
|
+
const lower = this.log.entryIndex.properties.index.crashSafeDurability;
|
|
3811
|
+
const coordinate = this.entryCoordinatesIndex.crashSafeDurability;
|
|
3812
|
+
if (!block || !lower || !coordinate) {
|
|
3813
|
+
return undefined;
|
|
3814
|
+
}
|
|
3815
|
+
return { block, lower, coordinate };
|
|
3816
|
+
}
|
|
3817
|
+
|
|
3818
|
+
private ownTransportSession(): bigint {
|
|
3819
|
+
return BigInt(
|
|
3820
|
+
(this.node.services.pubsub as unknown as { session: number | bigint })
|
|
3821
|
+
.session,
|
|
3822
|
+
);
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3598
3825
|
private replicationInfoV2ReceiveCapabilities(): number {
|
|
3599
3826
|
return (
|
|
3600
3827
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
3601
3828
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
3602
3829
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
3603
3830
|
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM |
|
|
3831
|
+
(this._persistedReceiptStorage
|
|
3832
|
+
? SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS
|
|
3833
|
+
: 0) |
|
|
3604
3834
|
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
3605
3835
|
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
3606
3836
|
: 0)
|
|
@@ -3616,9 +3846,7 @@ export class SharedLog<
|
|
|
3616
3846
|
{ receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
|
|
3617
3847
|
> {
|
|
3618
3848
|
const peerHash = properties.target.hashcode();
|
|
3619
|
-
const receiverTransportSession =
|
|
3620
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3621
|
-
);
|
|
3849
|
+
const receiverTransportSession = this.ownTransportSession();
|
|
3622
3850
|
await this.rpc.send(
|
|
3623
3851
|
new SyncCapabilitiesMessage({
|
|
3624
3852
|
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
@@ -3644,9 +3872,7 @@ export class SharedLog<
|
|
|
3644
3872
|
) ||
|
|
3645
3873
|
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
3646
3874
|
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
3647
|
-
|
|
3648
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3649
|
-
) !== receiverTransportSession
|
|
3875
|
+
this.ownTransportSession() !== receiverTransportSession
|
|
3650
3876
|
) {
|
|
3651
3877
|
return undefined;
|
|
3652
3878
|
}
|
|
@@ -3659,10 +3885,7 @@ export class SharedLog<
|
|
|
3659
3885
|
private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
|
|
3660
3886
|
return new ReplicationInfoV2ReceiveCoordinator({
|
|
3661
3887
|
getSelfKey: () => this.node.identity.publicKey,
|
|
3662
|
-
getReceiverTransportSession: () =>
|
|
3663
|
-
BigInt(
|
|
3664
|
-
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3665
|
-
),
|
|
3888
|
+
getReceiverTransportSession: () => this.ownTransportSession(),
|
|
3666
3889
|
isClosed: () => this.closed,
|
|
3667
3890
|
isPeerSessionCurrent: (peerHash, peerSession) =>
|
|
3668
3891
|
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
@@ -3832,6 +4055,9 @@ export class SharedLog<
|
|
|
3832
4055
|
this._peerSyncCapabilities = new Map();
|
|
3833
4056
|
this._peerSyncCapabilitySessions = new Map();
|
|
3834
4057
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
4058
|
+
this._persistedReceiptStorage = undefined;
|
|
4059
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
4060
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
3835
4061
|
this._liveRawGossipBatches = new Map();
|
|
3836
4062
|
this._liveRawGossipFlushScheduled = false;
|
|
3837
4063
|
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
@@ -4001,6 +4227,14 @@ export class SharedLog<
|
|
|
4001
4227
|
this._fanoutChannel = undefined;
|
|
4002
4228
|
}
|
|
4003
4229
|
|
|
4230
|
+
private ensureLogProviderHandle(fanoutService: FanoutTree): void {
|
|
4231
|
+
if (this._providerHandle || this._closeController.signal.aborted) return;
|
|
4232
|
+
this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
|
|
4233
|
+
ttlMs: 120_000,
|
|
4234
|
+
announceIntervalMs: 60_000,
|
|
4235
|
+
});
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4004
4238
|
private async _onFanoutData(detail: FanoutTreeDataEvent) {
|
|
4005
4239
|
let envelope: FanoutEnvelope;
|
|
4006
4240
|
try {
|
|
@@ -4127,11 +4361,23 @@ export class SharedLog<
|
|
|
4127
4361
|
const reliability: DeliveryReliability = delivery.reliability ?? "ack";
|
|
4128
4362
|
const deliveryTimeout = delivery.timeout;
|
|
4129
4363
|
const deliverySignal = delivery.signal;
|
|
4130
|
-
const requireRecipients =
|
|
4364
|
+
const requireRecipients =
|
|
4365
|
+
reliability === "persisted" || delivery.requireRecipients === true;
|
|
4131
4366
|
const minAcks =
|
|
4132
4367
|
delivery.minAcks != null && Number.isFinite(delivery.minAcks)
|
|
4133
4368
|
? Math.max(0, Math.floor(delivery.minAcks))
|
|
4134
4369
|
: undefined;
|
|
4370
|
+
if (reliability === "persisted") {
|
|
4371
|
+
if (
|
|
4372
|
+
delivery.minAcks == null ||
|
|
4373
|
+
!Number.isSafeInteger(delivery.minAcks) ||
|
|
4374
|
+
delivery.minAcks <= 0
|
|
4375
|
+
) {
|
|
4376
|
+
throw new Error(
|
|
4377
|
+
'persisted delivery requires a positive explicit "minAcks"',
|
|
4378
|
+
);
|
|
4379
|
+
}
|
|
4380
|
+
}
|
|
4135
4381
|
|
|
4136
4382
|
const wrap =
|
|
4137
4383
|
deliveryTimeout == null && deliverySignal == null
|
|
@@ -4206,6 +4452,82 @@ export class SharedLog<
|
|
|
4206
4452
|
};
|
|
4207
4453
|
}
|
|
4208
4454
|
|
|
4455
|
+
private validatePersistedReceiptRequestShape(
|
|
4456
|
+
request: RequestPersistedEntriesV1,
|
|
4457
|
+
): void {
|
|
4458
|
+
if (
|
|
4459
|
+
request.hashes.length === 0 ||
|
|
4460
|
+
request.hashes.length > MAX_PERSISTED_RECEIPT_HASHES
|
|
4461
|
+
) {
|
|
4462
|
+
throw new Error(
|
|
4463
|
+
`Persisted receipt requests require 1-${MAX_PERSISTED_RECEIPT_HASHES} hashes`,
|
|
4464
|
+
);
|
|
4465
|
+
}
|
|
4466
|
+
let bytes = 0;
|
|
4467
|
+
const encoder = new TextEncoder();
|
|
4468
|
+
for (const hash of request.hashes) {
|
|
4469
|
+
if (hash.length === 0 || hash.length > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
4470
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
4471
|
+
}
|
|
4472
|
+
bytes += encoder.encode(hash).byteLength;
|
|
4473
|
+
if (bytes > MAX_PERSISTED_RECEIPT_HASH_BYTES) {
|
|
4474
|
+
throw new Error("Invalid persisted receipt hash batch");
|
|
4475
|
+
}
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
|
|
4479
|
+
private hasValidPersistedReceiptHashes(
|
|
4480
|
+
request: RequestPersistedEntriesV1,
|
|
4481
|
+
): boolean {
|
|
4482
|
+
const seen = new Set<string>();
|
|
4483
|
+
try {
|
|
4484
|
+
for (const hash of request.hashes) {
|
|
4485
|
+
if (seen.has(hash)) return false;
|
|
4486
|
+
cidifyString(hash);
|
|
4487
|
+
seen.add(hash);
|
|
4488
|
+
}
|
|
4489
|
+
} catch {
|
|
4490
|
+
return false;
|
|
4491
|
+
}
|
|
4492
|
+
return true;
|
|
4493
|
+
}
|
|
4494
|
+
|
|
4495
|
+
private getPersistedDeliveryOptions(
|
|
4496
|
+
options?: SharedAppendOptions<T>,
|
|
4497
|
+
): DeliveryOptions | undefined {
|
|
4498
|
+
const delivery = options?.delivery;
|
|
4499
|
+
if (typeof delivery !== "object" || delivery.reliability !== "persisted") {
|
|
4500
|
+
return undefined;
|
|
4501
|
+
}
|
|
4502
|
+
const parsed = this._parseDeliveryOptions(options?.delivery);
|
|
4503
|
+
const target = (options as { target?: string } | undefined)?.target;
|
|
4504
|
+
if (target !== undefined && target !== "replicators") {
|
|
4505
|
+
throw new Error(
|
|
4506
|
+
'persisted delivery requires target="replicators" (or an omitted target)',
|
|
4507
|
+
);
|
|
4508
|
+
}
|
|
4509
|
+
if (
|
|
4510
|
+
parsed.delivery?.timeout != null &&
|
|
4511
|
+
(!Number.isFinite(parsed.delivery.timeout) ||
|
|
4512
|
+
parsed.delivery.timeout <= 0 ||
|
|
4513
|
+
parsed.delivery.timeout > MAX_PERSISTED_DELIVERY_TIMEOUT_MS)
|
|
4514
|
+
) {
|
|
4515
|
+
throw new Error(
|
|
4516
|
+
`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
|
|
4517
|
+
);
|
|
4518
|
+
}
|
|
4519
|
+
if (parsed.delivery?.signal?.aborted) {
|
|
4520
|
+
throw parsed.delivery.signal.reason ?? new AbortError();
|
|
4521
|
+
}
|
|
4522
|
+
return parsed.delivery!;
|
|
4523
|
+
}
|
|
4524
|
+
|
|
4525
|
+
private assertPersistedDeliveryOptions(
|
|
4526
|
+
options?: SharedAppendOptions<T>,
|
|
4527
|
+
): void {
|
|
4528
|
+
this.getPersistedDeliveryOptions(options);
|
|
4529
|
+
}
|
|
4530
|
+
|
|
4209
4531
|
private async _getSortedRouteHints(targetHash: string): Promise<RouteHint[]> {
|
|
4210
4532
|
const pubsub: any = this.node.services.pubsub as any;
|
|
4211
4533
|
const maybeHints = await pubsub?.getUnifiedRouteHints?.(
|
|
@@ -4620,6 +4942,7 @@ export class SharedLog<
|
|
|
4620
4942
|
plan: RawExchangeHeadSendPlan,
|
|
4621
4943
|
to: string[] | Set<string>,
|
|
4622
4944
|
options?: {
|
|
4945
|
+
acknowledge?: boolean;
|
|
4623
4946
|
priority?: number;
|
|
4624
4947
|
reserved?: Uint8Array;
|
|
4625
4948
|
signal?: AbortSignal;
|
|
@@ -4634,7 +4957,7 @@ export class SharedLog<
|
|
|
4634
4957
|
payload: Uint8Array,
|
|
4635
4958
|
properties: { topics: string[] },
|
|
4636
4959
|
options: {
|
|
4637
|
-
mode: SilentDelivery;
|
|
4960
|
+
mode: SilentDelivery | AcknowledgeDelivery;
|
|
4638
4961
|
priority?: number;
|
|
4639
4962
|
signal?: AbortSignal;
|
|
4640
4963
|
},
|
|
@@ -4722,7 +5045,9 @@ export class SharedLog<
|
|
|
4722
5045
|
item.payload,
|
|
4723
5046
|
{ topics: [topic] },
|
|
4724
5047
|
{
|
|
4725
|
-
mode:
|
|
5048
|
+
mode: options?.acknowledge
|
|
5049
|
+
? new AcknowledgeDelivery({ redundancy: 1, to: [...to] })
|
|
5050
|
+
: new SilentDelivery({ redundancy: 1, to: [...to] }),
|
|
4726
5051
|
priority: options?.priority,
|
|
4727
5052
|
signal: options?.signal,
|
|
4728
5053
|
},
|
|
@@ -4749,34 +5074,762 @@ export class SharedLog<
|
|
|
4749
5074
|
});
|
|
4750
5075
|
}
|
|
4751
5076
|
}
|
|
4752
|
-
return sentMessages;
|
|
4753
|
-
}
|
|
4754
|
-
|
|
4755
|
-
/**
|
|
4756
|
-
* `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
|
|
4757
|
-
* responses: resolves the head/reference plan like the TS raw path and
|
|
4758
|
-
* ships it fused when possible.
|
|
4759
|
-
*/
|
|
4760
|
-
private async trySendFusedRawExchangeHeads(
|
|
4761
|
-
hashes: string[],
|
|
4762
|
-
to: string[],
|
|
4763
|
-
options?: {
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
4777
|
-
|
|
4778
|
-
|
|
4779
|
-
|
|
5077
|
+
return sentMessages;
|
|
5078
|
+
}
|
|
5079
|
+
|
|
5080
|
+
/**
|
|
5081
|
+
* `RawExchangeHeadsSender` seam handed to the synchronizer for bulk sync
|
|
5082
|
+
* responses: resolves the head/reference plan like the TS raw path and
|
|
5083
|
+
* ships it fused when possible.
|
|
5084
|
+
*/
|
|
5085
|
+
private async trySendFusedRawExchangeHeads(
|
|
5086
|
+
hashes: string[],
|
|
5087
|
+
to: string[],
|
|
5088
|
+
options?: {
|
|
5089
|
+
acknowledge?: boolean;
|
|
5090
|
+
priority?: number;
|
|
5091
|
+
reserved?: Uint8Array;
|
|
5092
|
+
signal?: AbortSignal;
|
|
5093
|
+
},
|
|
5094
|
+
): Promise<number | undefined> {
|
|
5095
|
+
if (!this._nativeBackbone?.encodeRawExchangeSyncPayload) {
|
|
5096
|
+
return undefined;
|
|
5097
|
+
}
|
|
5098
|
+
const plan = collectRawExchangeHeadSendPlan(this.log, hashes);
|
|
5099
|
+
if (!plan) {
|
|
5100
|
+
return undefined;
|
|
5101
|
+
}
|
|
5102
|
+
if (plan.hashes.length === 0) {
|
|
5103
|
+
return 0;
|
|
5104
|
+
}
|
|
5105
|
+
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
5106
|
+
}
|
|
5107
|
+
|
|
5108
|
+
private persistedReceiptPeerSession(
|
|
5109
|
+
peerHash: string,
|
|
5110
|
+
): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
|
|
5111
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5112
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
5113
|
+
if (
|
|
5114
|
+
capabilitySession == null ||
|
|
5115
|
+
!peerSession ||
|
|
5116
|
+
peerSession.phase !== "open" ||
|
|
5117
|
+
!this._peerSessions.isCurrent(peerHash, peerSession) ||
|
|
5118
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
5119
|
+
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
5120
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) ===
|
|
5121
|
+
0
|
|
5122
|
+
) {
|
|
5123
|
+
return undefined;
|
|
5124
|
+
}
|
|
5125
|
+
return { capabilitySession, peerSession };
|
|
5126
|
+
}
|
|
5127
|
+
|
|
5128
|
+
private async waitPersistedReceiptRetry(
|
|
5129
|
+
signal: AbortSignal,
|
|
5130
|
+
ms: number,
|
|
5131
|
+
): Promise<void> {
|
|
5132
|
+
try {
|
|
5133
|
+
await delay(ms, { signal });
|
|
5134
|
+
} catch (error) {
|
|
5135
|
+
throw signal.aborted ? (signal.reason ?? error) : error;
|
|
5136
|
+
}
|
|
5137
|
+
}
|
|
5138
|
+
|
|
5139
|
+
private reservePersistedReceiptEgress(
|
|
5140
|
+
peer: string,
|
|
5141
|
+
capabilitySession: bigint,
|
|
5142
|
+
hashCount: number,
|
|
5143
|
+
now = Date.now(),
|
|
5144
|
+
): number {
|
|
5145
|
+
let nodeBudget = persistedReceiptEgressBudgets.get(this.node);
|
|
5146
|
+
if (!nodeBudget) {
|
|
5147
|
+
nodeBudget = { peerSessions: new Map() };
|
|
5148
|
+
persistedReceiptEgressBudgets.set(this.node, nodeBudget);
|
|
5149
|
+
}
|
|
5150
|
+
const peerSessionKey = `${peer}\0${capabilitySession}`;
|
|
5151
|
+
let bucket = nodeBudget.peerSessions.get(peerSessionKey);
|
|
5152
|
+
if (!bucket) {
|
|
5153
|
+
while (
|
|
5154
|
+
nodeBudget.peerSessions.size >=
|
|
5155
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
|
|
5156
|
+
) {
|
|
5157
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
5158
|
+
if (oldest === undefined) break;
|
|
5159
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
5160
|
+
}
|
|
5161
|
+
bucket = {
|
|
5162
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
5163
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
5164
|
+
refilledAt: now,
|
|
5165
|
+
};
|
|
5166
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
5167
|
+
} else {
|
|
5168
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
5169
|
+
nodeBudget.peerSessions.set(peerSessionKey, bucket);
|
|
5170
|
+
}
|
|
5171
|
+
refillPersistedReceiptIngressBucket(
|
|
5172
|
+
bucket,
|
|
5173
|
+
now,
|
|
5174
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
5175
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
5176
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
|
|
5177
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
|
|
5178
|
+
);
|
|
5179
|
+
if (bucket.requestTokens >= 1 && bucket.hashTokens >= hashCount) {
|
|
5180
|
+
bucket.requestTokens -= 1;
|
|
5181
|
+
bucket.hashTokens -= hashCount;
|
|
5182
|
+
return 0;
|
|
5183
|
+
}
|
|
5184
|
+
return Math.max(
|
|
5185
|
+
1,
|
|
5186
|
+
Math.ceil(
|
|
5187
|
+
Math.max(
|
|
5188
|
+
((1 - bucket.requestTokens) /
|
|
5189
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND) *
|
|
5190
|
+
1_000,
|
|
5191
|
+
((hashCount - bucket.hashTokens) /
|
|
5192
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND) *
|
|
5193
|
+
1_000,
|
|
5194
|
+
),
|
|
5195
|
+
),
|
|
5196
|
+
);
|
|
5197
|
+
}
|
|
5198
|
+
|
|
5199
|
+
private async waitForPersistedReceiptEgressAdmission(
|
|
5200
|
+
peer: string,
|
|
5201
|
+
capabilitySession: bigint,
|
|
5202
|
+
hashCount: number,
|
|
5203
|
+
signal: AbortSignal,
|
|
5204
|
+
): Promise<void> {
|
|
5205
|
+
while (true) {
|
|
5206
|
+
if (signal.aborted) {
|
|
5207
|
+
throw signal.reason ?? new AbortError();
|
|
5208
|
+
}
|
|
5209
|
+
const waitMs = this.reservePersistedReceiptEgress(
|
|
5210
|
+
peer,
|
|
5211
|
+
capabilitySession,
|
|
5212
|
+
hashCount,
|
|
5213
|
+
);
|
|
5214
|
+
if (waitMs === 0) return;
|
|
5215
|
+
await this.waitPersistedReceiptRetry(signal, waitMs);
|
|
5216
|
+
}
|
|
5217
|
+
}
|
|
5218
|
+
|
|
5219
|
+
private persistedDeliveryTimeoutMs(
|
|
5220
|
+
delivery: DeliveryOptions,
|
|
5221
|
+
hashCount: number,
|
|
5222
|
+
): number {
|
|
5223
|
+
return (
|
|
5224
|
+
delivery.timeout ??
|
|
5225
|
+
Math.min(
|
|
5226
|
+
MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
|
|
5227
|
+
DEFAULT_PERSISTED_RECEIPT_TIMEOUT_MS +
|
|
5228
|
+
persistedTransferAdmissionBudgetMs(hashCount) +
|
|
5229
|
+
Math.ceil(persistedReceiptPacingFloorMs(hashCount)),
|
|
5230
|
+
)
|
|
5231
|
+
);
|
|
5232
|
+
}
|
|
5233
|
+
|
|
5234
|
+
private async waitForPersistedTransferAdmission(
|
|
5235
|
+
peer: string,
|
|
5236
|
+
hashes: readonly string[],
|
|
5237
|
+
captured: { capabilitySession: bigint; peerSession: PeerSession },
|
|
5238
|
+
signal: AbortSignal,
|
|
5239
|
+
isStillCurrent: () => boolean,
|
|
5240
|
+
): Promise<boolean> {
|
|
5241
|
+
const expiresAt = Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
5242
|
+
while (!signal.aborted && isStillCurrent()) {
|
|
5243
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5244
|
+
if (
|
|
5245
|
+
!current ||
|
|
5246
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5247
|
+
current.peerSession !== captured.peerSession
|
|
5248
|
+
) {
|
|
5249
|
+
return false;
|
|
5250
|
+
}
|
|
5251
|
+
if (hashes.every((hash) => this.isEntryKnownByPeer(hash, peer))) {
|
|
5252
|
+
return true;
|
|
5253
|
+
}
|
|
5254
|
+
const remaining = expiresAt - Date.now();
|
|
5255
|
+
if (remaining <= 0) return false;
|
|
5256
|
+
await this.waitPersistedReceiptRetry(
|
|
5257
|
+
signal,
|
|
5258
|
+
Math.min(PERSISTED_RECEIPT_RETRY_MS, remaining),
|
|
5259
|
+
);
|
|
5260
|
+
}
|
|
5261
|
+
if (signal.aborted) throw signal.reason ?? new AbortError();
|
|
5262
|
+
return false;
|
|
5263
|
+
}
|
|
5264
|
+
|
|
5265
|
+
private createPersistedDeliveryDeadline(
|
|
5266
|
+
delivery: DeliveryOptions,
|
|
5267
|
+
ownershipLifecycleController: AbortController,
|
|
5268
|
+
hashCount = 1,
|
|
5269
|
+
): PersistedDeliveryDeadline {
|
|
5270
|
+
const timeoutMs = this.persistedDeliveryTimeoutMs(delivery, hashCount);
|
|
5271
|
+
if (
|
|
5272
|
+
!Number.isFinite(timeoutMs) ||
|
|
5273
|
+
timeoutMs <= 0 ||
|
|
5274
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
|
|
5275
|
+
) {
|
|
5276
|
+
throw new Error(
|
|
5277
|
+
`persisted delivery timeout must be a positive number no greater than ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS}`,
|
|
5278
|
+
);
|
|
5279
|
+
}
|
|
5280
|
+
const minAcks = Math.floor(delivery.minAcks!);
|
|
5281
|
+
const deadlineController = new AbortController();
|
|
5282
|
+
const timeout = setTimeout(
|
|
5283
|
+
() =>
|
|
5284
|
+
deadlineController.abort(
|
|
5285
|
+
new TimeoutError(
|
|
5286
|
+
`Timed out waiting for ${minAcks} persisted remote replicas.`,
|
|
5287
|
+
),
|
|
5288
|
+
),
|
|
5289
|
+
timeoutMs,
|
|
5290
|
+
);
|
|
5291
|
+
timeout.unref?.();
|
|
5292
|
+
return {
|
|
5293
|
+
deadline: Date.now() + timeoutMs,
|
|
5294
|
+
signal: AbortSignal.any(
|
|
5295
|
+
[
|
|
5296
|
+
delivery.signal,
|
|
5297
|
+
this._closeController.signal,
|
|
5298
|
+
ownershipLifecycleController.signal,
|
|
5299
|
+
deadlineController.signal,
|
|
5300
|
+
].filter((value): value is AbortSignal => !!value),
|
|
5301
|
+
),
|
|
5302
|
+
dispose: () => clearTimeout(timeout),
|
|
5303
|
+
};
|
|
5304
|
+
}
|
|
5305
|
+
|
|
5306
|
+
private async planPersistedDeliveryLeaders(
|
|
5307
|
+
entries: PersistedDeliveryPlanningEntry<T, R>[],
|
|
5308
|
+
replicas: number,
|
|
5309
|
+
ownershipLifecycleController: AbortController,
|
|
5310
|
+
): Promise<LeaderMap[]> {
|
|
5311
|
+
if (
|
|
5312
|
+
this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry
|
|
5313
|
+
) {
|
|
5314
|
+
const leaders: LeaderMap[] = [];
|
|
5315
|
+
for (const entry of entries) {
|
|
5316
|
+
leaders.push(
|
|
5317
|
+
await this.findLeadersFromEntry(
|
|
5318
|
+
entry,
|
|
5319
|
+
replicas,
|
|
5320
|
+
{ freshLeaderPlan: true },
|
|
5321
|
+
ownershipLifecycleController,
|
|
5322
|
+
),
|
|
5323
|
+
);
|
|
5324
|
+
}
|
|
5325
|
+
return leaders;
|
|
5326
|
+
}
|
|
5327
|
+
const items: EntryLeaderBatchItem<R>[] = entries.map((entry) => ({
|
|
5328
|
+
entry,
|
|
5329
|
+
replicas,
|
|
5330
|
+
options: { freshLeaderPlan: true, persist: false },
|
|
5331
|
+
}));
|
|
5332
|
+
const nativeRoutingPlanner =
|
|
5333
|
+
this._nativeRangePlanner ?? this._nativeBackbone;
|
|
5334
|
+
if (this.canPlanNativeEntryLeaderBatch(items) && nativeRoutingPlanner) {
|
|
5335
|
+
const options = items[0]!.options!;
|
|
5336
|
+
const context = await this.createLeaderSelectionContext(
|
|
5337
|
+
options,
|
|
5338
|
+
ownershipLifecycleController,
|
|
5339
|
+
);
|
|
5340
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5341
|
+
ownershipLifecycleController,
|
|
5342
|
+
);
|
|
5343
|
+
const nativeOptions = this.createNativeLeaderOptions(context, options);
|
|
5344
|
+
const fullReplicaLeaders =
|
|
5345
|
+
nativeRoutingPlanner.getRoutingFullReplicaLeaders?.(
|
|
5346
|
+
replicas,
|
|
5347
|
+
nativeOptions,
|
|
5348
|
+
);
|
|
5349
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5350
|
+
ownershipLifecycleController,
|
|
5351
|
+
);
|
|
5352
|
+
if (fullReplicaLeaders) {
|
|
5353
|
+
// Receipt settlement treats leader maps as read-only. Reusing the one
|
|
5354
|
+
// gid-independent routing result avoids one Map allocation per entry.
|
|
5355
|
+
return new Array<LeaderMap>(items.length).fill(fullReplicaLeaders);
|
|
5356
|
+
}
|
|
5357
|
+
if (nativeRoutingPlanner.planLeaderSamplesForGidsBatch) {
|
|
5358
|
+
const nativeLeaders =
|
|
5359
|
+
nativeRoutingPlanner.planLeaderSamplesForGidsBatch(
|
|
5360
|
+
items.map((item) => ({
|
|
5361
|
+
gid: this.getEntryGid(item.entry),
|
|
5362
|
+
replicas: item.replicas,
|
|
5363
|
+
})),
|
|
5364
|
+
nativeOptions,
|
|
5365
|
+
);
|
|
5366
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5367
|
+
ownershipLifecycleController,
|
|
5368
|
+
);
|
|
5369
|
+
if (nativeLeaders?.length === items.length) {
|
|
5370
|
+
return nativeLeaders;
|
|
5371
|
+
}
|
|
5372
|
+
}
|
|
5373
|
+
}
|
|
5374
|
+
return (
|
|
5375
|
+
await this.planEntryLeaderBatch(items, ownershipLifecycleController)
|
|
5376
|
+
).map((plan) => plan.leaders);
|
|
5377
|
+
}
|
|
5378
|
+
|
|
5379
|
+
private async settlePersistedDelivery(
|
|
5380
|
+
input: PersistedDeliveryPlanningEntry<T, R>[],
|
|
5381
|
+
replicas: number,
|
|
5382
|
+
delivery: DeliveryOptions,
|
|
5383
|
+
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
5384
|
+
persistedDeadline?: PersistedDeliveryDeadline,
|
|
5385
|
+
transferOnFirstRound = false,
|
|
5386
|
+
): Promise<void> {
|
|
5387
|
+
const minAcks = Math.floor(delivery.minAcks!);
|
|
5388
|
+
const entries = new Map(input.map((entry) => [entry.hash, entry]));
|
|
5389
|
+
if (entries.size === 0) return;
|
|
5390
|
+
|
|
5391
|
+
const committedHashes = [...entries.keys()];
|
|
5392
|
+
const ownedDeadline = !persistedDeadline;
|
|
5393
|
+
const deadline =
|
|
5394
|
+
persistedDeadline ??
|
|
5395
|
+
this.createPersistedDeliveryDeadline(
|
|
5396
|
+
delivery,
|
|
5397
|
+
ownershipLifecycleController,
|
|
5398
|
+
entries.size,
|
|
5399
|
+
);
|
|
5400
|
+
const signal = deadline.signal;
|
|
5401
|
+
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
5402
|
+
let initialTransferPending = transferOnFirstRound;
|
|
5403
|
+
let needsInitialLeaderCheck = true;
|
|
5404
|
+
const carriedAcknowledgements = new Map<
|
|
5405
|
+
string,
|
|
5406
|
+
Map<string, { capabilitySession: bigint; peerSession: PeerSession }>
|
|
5407
|
+
>(committedHashes.map((hash) => [hash, new Map()]));
|
|
5408
|
+
const repairsByPeer = new Map<
|
|
5409
|
+
string,
|
|
5410
|
+
{
|
|
5411
|
+
capabilitySession: bigint;
|
|
5412
|
+
peerSession: PeerSession;
|
|
5413
|
+
hashes: Set<string>;
|
|
5414
|
+
}
|
|
5415
|
+
>();
|
|
5416
|
+
let acknowledgementOwnershipRevision: number | undefined;
|
|
5417
|
+
const purgePeerDeliveryState = (peer: string) => {
|
|
5418
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5419
|
+
acknowledgements.delete(peer);
|
|
5420
|
+
}
|
|
5421
|
+
repairsByPeer.delete(peer);
|
|
5422
|
+
};
|
|
5423
|
+
try {
|
|
5424
|
+
while (true) {
|
|
5425
|
+
if (signal.aborted) {
|
|
5426
|
+
throw signal.reason ?? new AbortError();
|
|
5427
|
+
}
|
|
5428
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5429
|
+
ownershipLifecycleController,
|
|
5430
|
+
);
|
|
5431
|
+
const ownershipRevision =
|
|
5432
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
5433
|
+
const isRoundOwnershipCurrent = () =>
|
|
5434
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
5435
|
+
if (acknowledgementOwnershipRevision !== ownershipRevision) {
|
|
5436
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5437
|
+
acknowledgements.clear();
|
|
5438
|
+
}
|
|
5439
|
+
repairsByPeer.clear();
|
|
5440
|
+
acknowledgementOwnershipRevision = ownershipRevision;
|
|
5441
|
+
} else {
|
|
5442
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5443
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5444
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5445
|
+
if (
|
|
5446
|
+
!current ||
|
|
5447
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5448
|
+
current.peerSession !== captured.peerSession
|
|
5449
|
+
) {
|
|
5450
|
+
purgePeerDeliveryState(peer);
|
|
5451
|
+
}
|
|
5452
|
+
}
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
if (!isRoundOwnershipCurrent()) {
|
|
5456
|
+
await this.waitPersistedReceiptRetry(
|
|
5457
|
+
signal,
|
|
5458
|
+
PERSISTED_RECEIPT_RETRY_MS,
|
|
5459
|
+
);
|
|
5460
|
+
continue;
|
|
5461
|
+
}
|
|
5462
|
+
|
|
5463
|
+
// Carry receipts only across retries in the exact same ownership and
|
|
5464
|
+
// transport epoch. A revision/session change purges them before they can
|
|
5465
|
+
// survive an away-and-back leader transition or combine with a later peer.
|
|
5466
|
+
const hashesByPeer = new Map<string, string[]>();
|
|
5467
|
+
const entryArray = [...entries.values()];
|
|
5468
|
+
const leadersByEntry = await this.planPersistedDeliveryLeaders(
|
|
5469
|
+
entryArray,
|
|
5470
|
+
replicas,
|
|
5471
|
+
ownershipLifecycleController,
|
|
5472
|
+
);
|
|
5473
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5474
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
5475
|
+
if (needsInitialLeaderCheck) {
|
|
5476
|
+
needsInitialLeaderCheck = false;
|
|
5477
|
+
if (
|
|
5478
|
+
leadersByEntry.some(
|
|
5479
|
+
(leaders) =>
|
|
5480
|
+
leaders.size === 0 ||
|
|
5481
|
+
(leaders.size === 1 && leaders.has(selfHash)),
|
|
5482
|
+
)
|
|
5483
|
+
) {
|
|
5484
|
+
throw new NoPeersError(this.rpc.topic);
|
|
5485
|
+
}
|
|
5486
|
+
}
|
|
5487
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
5488
|
+
const hash = entryArray[index]!.hash;
|
|
5489
|
+
const leaders = leadersByEntry[index]!;
|
|
5490
|
+
if (signal.aborted) {
|
|
5491
|
+
throw signal.reason ?? new AbortError();
|
|
5492
|
+
}
|
|
5493
|
+
if (!isRoundOwnershipCurrent()) break;
|
|
5494
|
+
const acknowledgements = carriedAcknowledgements.get(hash)!;
|
|
5495
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5496
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5497
|
+
if (
|
|
5498
|
+
!leaders.has(peer) ||
|
|
5499
|
+
!current ||
|
|
5500
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5501
|
+
current.peerSession !== captured.peerSession
|
|
5502
|
+
) {
|
|
5503
|
+
acknowledgements.delete(peer);
|
|
5504
|
+
}
|
|
5505
|
+
}
|
|
5506
|
+
if (acknowledgements.size >= minAcks) continue;
|
|
5507
|
+
for (const peer of leaders.keys()) {
|
|
5508
|
+
if (peer === selfHash) continue;
|
|
5509
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5510
|
+
if (!current) continue;
|
|
5511
|
+
if (acknowledgements.has(peer)) continue;
|
|
5512
|
+
const hashes = hashesByPeer.get(peer) ?? [];
|
|
5513
|
+
hashes.push(hash);
|
|
5514
|
+
hashesByPeer.set(peer, hashes);
|
|
5515
|
+
}
|
|
5516
|
+
}
|
|
5517
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5518
|
+
|
|
5519
|
+
const operationQueue = new PQueue({
|
|
5520
|
+
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
5521
|
+
});
|
|
5522
|
+
const candidateWaves = Math.max(
|
|
5523
|
+
1,
|
|
5524
|
+
Math.ceil(hashesByPeer.size / MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL),
|
|
5525
|
+
);
|
|
5526
|
+
const roundController = new AbortController();
|
|
5527
|
+
const roundSignal = AbortSignal.any([signal, roundController.signal]);
|
|
5528
|
+
const getAttemptTimeout = () =>
|
|
5529
|
+
Math.max(
|
|
5530
|
+
1,
|
|
5531
|
+
Math.min(
|
|
5532
|
+
maxAttemptMs,
|
|
5533
|
+
Math.floor((deadline.deadline - Date.now()) / candidateWaves),
|
|
5534
|
+
),
|
|
5535
|
+
);
|
|
5536
|
+
const requests = new Set<Promise<void>>();
|
|
5537
|
+
const transferAllOnRound = initialTransferPending;
|
|
5538
|
+
try {
|
|
5539
|
+
for (const [peer, hashes] of hashesByPeer) {
|
|
5540
|
+
let request!: Promise<void>;
|
|
5541
|
+
request = (async () => {
|
|
5542
|
+
const captured = this.persistedReceiptPeerSession(peer);
|
|
5543
|
+
if (!captured) return;
|
|
5544
|
+
const previousRepair = repairsByPeer.get(peer);
|
|
5545
|
+
if (
|
|
5546
|
+
previousRepair &&
|
|
5547
|
+
(previousRepair.capabilitySession !==
|
|
5548
|
+
captured.capabilitySession ||
|
|
5549
|
+
previousRepair.peerSession !== captured.peerSession)
|
|
5550
|
+
) {
|
|
5551
|
+
repairsByPeer.delete(peer);
|
|
5552
|
+
}
|
|
5553
|
+
const ensureRepairState = () => {
|
|
5554
|
+
let state = repairsByPeer.get(peer);
|
|
5555
|
+
if (!state) {
|
|
5556
|
+
state = {
|
|
5557
|
+
capabilitySession: captured.capabilitySession,
|
|
5558
|
+
peerSession: captured.peerSession,
|
|
5559
|
+
hashes: new Set<string>(),
|
|
5560
|
+
};
|
|
5561
|
+
repairsByPeer.set(peer, state);
|
|
5562
|
+
}
|
|
5563
|
+
return state;
|
|
5564
|
+
};
|
|
5565
|
+
const isPeerRoundCurrent = () => {
|
|
5566
|
+
if (roundSignal.aborted || !isRoundOwnershipCurrent()) {
|
|
5567
|
+
return false;
|
|
5568
|
+
}
|
|
5569
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5570
|
+
return (
|
|
5571
|
+
!!current &&
|
|
5572
|
+
current.capabilitySession === captured.capabilitySession &&
|
|
5573
|
+
current.peerSession === captured.peerSession
|
|
5574
|
+
);
|
|
5575
|
+
};
|
|
5576
|
+
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
5577
|
+
const transferHashes = transferAllOnRound
|
|
5578
|
+
? hashes
|
|
5579
|
+
: repairs
|
|
5580
|
+
? hashes.filter((hash) => repairs.has(hash))
|
|
5581
|
+
: [];
|
|
5582
|
+
let attemptedHashCount = 0;
|
|
5583
|
+
if (transferHashes.length > 0) {
|
|
5584
|
+
try {
|
|
5585
|
+
await this.pushEntryHashes(peer, transferHashes, {
|
|
5586
|
+
acknowledge: false,
|
|
5587
|
+
chunkTimeout: getAttemptTimeout,
|
|
5588
|
+
chunkSize: PERSISTED_TRANSFER_CHUNK_SIZE,
|
|
5589
|
+
isStillCurrent: isPeerRoundCurrent,
|
|
5590
|
+
onChunkAttempted: (chunk) => {
|
|
5591
|
+
attemptedHashCount += chunk.length;
|
|
5592
|
+
if (!transferAllOnRound) {
|
|
5593
|
+
const state = repairsByPeer.get(peer);
|
|
5594
|
+
for (const hash of chunk) state?.hashes.delete(hash);
|
|
5595
|
+
}
|
|
5596
|
+
},
|
|
5597
|
+
onChunkSent: async (chunk) => {
|
|
5598
|
+
return this.waitForPersistedTransferAdmission(
|
|
5599
|
+
peer,
|
|
5600
|
+
chunk,
|
|
5601
|
+
captured,
|
|
5602
|
+
roundSignal,
|
|
5603
|
+
isPeerRoundCurrent,
|
|
5604
|
+
);
|
|
5605
|
+
},
|
|
5606
|
+
operationQueue,
|
|
5607
|
+
priority: delivery.priority,
|
|
5608
|
+
repairHint: !transferAllOnRound,
|
|
5609
|
+
signal: roundSignal,
|
|
5610
|
+
});
|
|
5611
|
+
} catch {
|
|
5612
|
+
// A send can fail after remote admission. The attempted prefix
|
|
5613
|
+
// includes that uncertain chunk for authoritative receipt probing.
|
|
5614
|
+
}
|
|
5615
|
+
if (
|
|
5616
|
+
transferAllOnRound &&
|
|
5617
|
+
attemptedHashCount < transferHashes.length
|
|
5618
|
+
) {
|
|
5619
|
+
const state = ensureRepairState();
|
|
5620
|
+
for (const hash of transferHashes.slice(attemptedHashCount)) {
|
|
5621
|
+
state.hashes.add(hash);
|
|
5622
|
+
}
|
|
5623
|
+
}
|
|
5624
|
+
}
|
|
5625
|
+
const receiptHashes = transferAllOnRound
|
|
5626
|
+
? hashes.slice(0, attemptedHashCount)
|
|
5627
|
+
: hashes;
|
|
5628
|
+
if (!isPeerRoundCurrent()) {
|
|
5629
|
+
purgePeerDeliveryState(peer);
|
|
5630
|
+
return;
|
|
5631
|
+
}
|
|
5632
|
+
// Keep receipt chunks sequential per peer. The shared operation
|
|
5633
|
+
// queue bounds only active sends/requests; admission waits never
|
|
5634
|
+
// occupy its slots and cannot starve a later healthy candidate.
|
|
5635
|
+
for (
|
|
5636
|
+
let offset = 0;
|
|
5637
|
+
offset < receiptHashes.length;
|
|
5638
|
+
offset += PERSISTED_RECEIPT_CHUNK_SIZE
|
|
5639
|
+
) {
|
|
5640
|
+
const requestedHashes = receiptHashes.slice(
|
|
5641
|
+
offset,
|
|
5642
|
+
offset + PERSISTED_RECEIPT_CHUNK_SIZE,
|
|
5643
|
+
);
|
|
5644
|
+
if (roundSignal.aborted || !isPeerRoundCurrent()) {
|
|
5645
|
+
if (!isRoundOwnershipCurrent()) roundController.abort();
|
|
5646
|
+
break;
|
|
5647
|
+
}
|
|
5648
|
+
let responses;
|
|
5649
|
+
try {
|
|
5650
|
+
await this.waitForPersistedReceiptEgressAdmission(
|
|
5651
|
+
peer,
|
|
5652
|
+
captured.capabilitySession,
|
|
5653
|
+
requestedHashes.length,
|
|
5654
|
+
roundSignal,
|
|
5655
|
+
);
|
|
5656
|
+
if (!isPeerRoundCurrent()) break;
|
|
5657
|
+
const attemptTimeout = getAttemptTimeout();
|
|
5658
|
+
responses =
|
|
5659
|
+
(await operationQueue.add(async () => {
|
|
5660
|
+
if (!isPeerRoundCurrent()) return [];
|
|
5661
|
+
return this.rpc.request(
|
|
5662
|
+
new RequestPersistedEntriesV1({
|
|
5663
|
+
expectedReceiverSession: captured.capabilitySession,
|
|
5664
|
+
hashes: requestedHashes,
|
|
5665
|
+
}),
|
|
5666
|
+
{
|
|
5667
|
+
mode: new SilentDelivery({
|
|
5668
|
+
to: [peer],
|
|
5669
|
+
redundancy: 1,
|
|
5670
|
+
}),
|
|
5671
|
+
amount: 1,
|
|
5672
|
+
priority: delivery.priority,
|
|
5673
|
+
timeout: attemptTimeout,
|
|
5674
|
+
signal: roundSignal,
|
|
5675
|
+
},
|
|
5676
|
+
);
|
|
5677
|
+
})) ?? [];
|
|
5678
|
+
} catch {
|
|
5679
|
+
if (roundSignal.aborted) break;
|
|
5680
|
+
// A peer can disconnect or miss this retry while the overall
|
|
5681
|
+
// quorum deadline remains active. Replan on the next round.
|
|
5682
|
+
break;
|
|
5683
|
+
}
|
|
5684
|
+
if (!isRoundOwnershipCurrent()) {
|
|
5685
|
+
roundController.abort();
|
|
5686
|
+
break;
|
|
5687
|
+
}
|
|
5688
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5689
|
+
if (
|
|
5690
|
+
!current ||
|
|
5691
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5692
|
+
current.peerSession !== captured.peerSession
|
|
5693
|
+
) {
|
|
5694
|
+
purgePeerDeliveryState(peer);
|
|
5695
|
+
break;
|
|
5696
|
+
}
|
|
5697
|
+
const requested = new Set(requestedHashes);
|
|
5698
|
+
const confirmed = new Set<string>();
|
|
5699
|
+
let receivedValidConfirmation = false;
|
|
5700
|
+
for (const result of responses) {
|
|
5701
|
+
if (
|
|
5702
|
+
!(result.response instanceof ConfirmEntriesMessage) ||
|
|
5703
|
+
result.from?.hashcode() !== peer ||
|
|
5704
|
+
result.message.header.session !== captured.capabilitySession
|
|
5705
|
+
) {
|
|
5706
|
+
continue;
|
|
5707
|
+
}
|
|
5708
|
+
const unique = new Set(result.response.hashes);
|
|
5709
|
+
if (
|
|
5710
|
+
unique.size !== result.response.hashes.length ||
|
|
5711
|
+
[...unique].some((hash) => !requested.has(hash))
|
|
5712
|
+
) {
|
|
5713
|
+
continue;
|
|
5714
|
+
}
|
|
5715
|
+
receivedValidConfirmation = true;
|
|
5716
|
+
for (const hash of unique) {
|
|
5717
|
+
confirmed.add(hash);
|
|
5718
|
+
carriedAcknowledgements.get(hash)?.set(peer, captured);
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
if (receivedValidConfirmation) {
|
|
5722
|
+
const state = ensureRepairState();
|
|
5723
|
+
for (const hash of requested) {
|
|
5724
|
+
if (confirmed.has(hash)) {
|
|
5725
|
+
state.hashes.delete(hash);
|
|
5726
|
+
} else {
|
|
5727
|
+
state.hashes.add(hash);
|
|
5728
|
+
}
|
|
5729
|
+
}
|
|
5730
|
+
}
|
|
5731
|
+
}
|
|
5732
|
+
})()
|
|
5733
|
+
.then(() => undefined)
|
|
5734
|
+
.catch(() => undefined)
|
|
5735
|
+
.finally(() => requests.delete(request));
|
|
5736
|
+
requests.add(request);
|
|
5737
|
+
}
|
|
5738
|
+
|
|
5739
|
+
const roundComplete = async (): Promise<boolean> => {
|
|
5740
|
+
if (signal.aborted) {
|
|
5741
|
+
throw signal.reason ?? new AbortError();
|
|
5742
|
+
}
|
|
5743
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
5744
|
+
for (const acknowledgements of carriedAcknowledgements.values()) {
|
|
5745
|
+
if (acknowledgements.size < minAcks) return false;
|
|
5746
|
+
}
|
|
5747
|
+
const validatedLeaders = await this.planPersistedDeliveryLeaders(
|
|
5748
|
+
entryArray,
|
|
5749
|
+
replicas,
|
|
5750
|
+
ownershipLifecycleController,
|
|
5751
|
+
);
|
|
5752
|
+
for (let index = 0; index < entryArray.length; index++) {
|
|
5753
|
+
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
5754
|
+
if (signal.aborted) {
|
|
5755
|
+
throw signal.reason ?? new AbortError();
|
|
5756
|
+
}
|
|
5757
|
+
return false;
|
|
5758
|
+
}
|
|
5759
|
+
const hash = entryArray[index]!.hash;
|
|
5760
|
+
const leaders = validatedLeaders[index]!;
|
|
5761
|
+
let valid = 0;
|
|
5762
|
+
const acknowledgements = carriedAcknowledgements.get(hash)!;
|
|
5763
|
+
for (const [peer, captured] of acknowledgements) {
|
|
5764
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
5765
|
+
if (
|
|
5766
|
+
!current ||
|
|
5767
|
+
current.capabilitySession !== captured.capabilitySession ||
|
|
5768
|
+
current.peerSession !== captured.peerSession
|
|
5769
|
+
) {
|
|
5770
|
+
purgePeerDeliveryState(peer);
|
|
5771
|
+
continue;
|
|
5772
|
+
}
|
|
5773
|
+
if (!leaders.has(peer)) {
|
|
5774
|
+
acknowledgements.delete(peer);
|
|
5775
|
+
continue;
|
|
5776
|
+
}
|
|
5777
|
+
if (++valid >= minAcks) {
|
|
5778
|
+
break;
|
|
5779
|
+
}
|
|
5780
|
+
}
|
|
5781
|
+
if (valid < minAcks) return false;
|
|
5782
|
+
}
|
|
5783
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
5784
|
+
ownershipLifecycleController,
|
|
5785
|
+
);
|
|
5786
|
+
if (signal.aborted) {
|
|
5787
|
+
throw signal.reason ?? new AbortError();
|
|
5788
|
+
}
|
|
5789
|
+
return isRoundOwnershipCurrent();
|
|
5790
|
+
};
|
|
5791
|
+
|
|
5792
|
+
while (requests.size > 0) {
|
|
5793
|
+
await Promise.race(requests);
|
|
5794
|
+
if (await roundComplete()) {
|
|
5795
|
+
return;
|
|
5796
|
+
}
|
|
5797
|
+
}
|
|
5798
|
+
if (await roundComplete()) {
|
|
5799
|
+
return;
|
|
5800
|
+
}
|
|
5801
|
+
} finally {
|
|
5802
|
+
roundController.abort();
|
|
5803
|
+
void Promise.allSettled([...requests]);
|
|
5804
|
+
}
|
|
5805
|
+
if (isRoundOwnershipCurrent()) {
|
|
5806
|
+
initialTransferPending = false;
|
|
5807
|
+
}
|
|
5808
|
+
// Keep early retries fair and responsive, then let a caller's longer
|
|
5809
|
+
// overall deadline accommodate a genuinely slow durability barrier.
|
|
5810
|
+
maxAttemptMs = Math.min(
|
|
5811
|
+
MAX_PERSISTED_DELIVERY_TIMEOUT_MS,
|
|
5812
|
+
maxAttemptMs * 2,
|
|
5813
|
+
);
|
|
5814
|
+
await this.waitPersistedReceiptRetry(
|
|
5815
|
+
signal,
|
|
5816
|
+
Math.max(
|
|
5817
|
+
0,
|
|
5818
|
+
Math.min(
|
|
5819
|
+
PERSISTED_RECEIPT_RETRY_MS,
|
|
5820
|
+
deadline.deadline - Date.now(),
|
|
5821
|
+
),
|
|
5822
|
+
),
|
|
5823
|
+
);
|
|
5824
|
+
}
|
|
5825
|
+
} catch (error) {
|
|
5826
|
+
if (error instanceof PersistedDeliveryError) {
|
|
5827
|
+
throw error;
|
|
5828
|
+
}
|
|
5829
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
5830
|
+
} finally {
|
|
5831
|
+
if (ownedDeadline) deadline.dispose();
|
|
5832
|
+
}
|
|
4780
5833
|
}
|
|
4781
5834
|
|
|
4782
5835
|
private async _appendDeliverToReplicators(
|
|
@@ -7839,13 +8892,7 @@ export class SharedLog<
|
|
|
7839
8892
|
try {
|
|
7840
8893
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
7841
8894
|
if (fanoutService?.provide && !this._providerHandle) {
|
|
7842
|
-
this.
|
|
7843
|
-
`shared-log|${this.topic}`,
|
|
7844
|
-
{
|
|
7845
|
-
ttlMs: 120_000,
|
|
7846
|
-
announceIntervalMs: 60_000,
|
|
7847
|
-
},
|
|
7848
|
-
);
|
|
8895
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
7849
8896
|
}
|
|
7850
8897
|
} catch {
|
|
7851
8898
|
// Best-effort only.
|
|
@@ -8357,63 +9404,135 @@ export class SharedLog<
|
|
|
8357
9404
|
}
|
|
8358
9405
|
}
|
|
8359
9406
|
|
|
8360
|
-
private async
|
|
9407
|
+
private async pushEntryHashChunk(
|
|
8361
9408
|
target: string,
|
|
8362
|
-
|
|
8363
|
-
|
|
8364
|
-
|
|
8365
|
-
|
|
8366
|
-
|
|
8367
|
-
|
|
8368
|
-
}
|
|
8369
|
-
|
|
8370
|
-
|
|
9409
|
+
chunk: string[],
|
|
9410
|
+
options: {
|
|
9411
|
+
acknowledge?: boolean;
|
|
9412
|
+
priority?: number;
|
|
9413
|
+
repairHint?: boolean;
|
|
9414
|
+
signal?: AbortSignal;
|
|
9415
|
+
},
|
|
9416
|
+
isStillCurrent: () => boolean,
|
|
9417
|
+
): Promise<boolean> {
|
|
9418
|
+
if (!isStillCurrent()) return false;
|
|
9419
|
+
const useRaw =
|
|
8371
9420
|
this._logProperties?.sync?.rawExchangeHeads === true &&
|
|
8372
|
-
this.peerSupportsRawExchangeHeads(target)
|
|
8373
|
-
) {
|
|
8374
|
-
const reserved = new Uint8Array(4);
|
|
8375
|
-
reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
9421
|
+
this.peerSupportsRawExchangeHeads(target);
|
|
9422
|
+
if (useRaw) {
|
|
9423
|
+
const reserved = options.repairHint ? new Uint8Array(4) : undefined;
|
|
9424
|
+
if (reserved) reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8376
9425
|
const sentMessages = await this.trySendFusedRawExchangeHeads(
|
|
8377
|
-
|
|
9426
|
+
chunk,
|
|
8378
9427
|
[target],
|
|
8379
|
-
{
|
|
9428
|
+
{
|
|
9429
|
+
acknowledge: options.acknowledge,
|
|
9430
|
+
priority: options.priority,
|
|
9431
|
+
reserved,
|
|
9432
|
+
signal: options.signal,
|
|
9433
|
+
},
|
|
8380
9434
|
);
|
|
8381
|
-
if (!isStillCurrent())
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
9435
|
+
if (!isStillCurrent()) return false;
|
|
9436
|
+
if (sentMessages === undefined) {
|
|
9437
|
+
for await (const message of createRawExchangeHeadsMessages(
|
|
9438
|
+
this.log,
|
|
9439
|
+
chunk,
|
|
9440
|
+
this._logProperties?.sync?.profile,
|
|
9441
|
+
)) {
|
|
9442
|
+
if (!isStillCurrent()) return false;
|
|
9443
|
+
if (options.repairHint) {
|
|
9444
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
9445
|
+
}
|
|
9446
|
+
await this.rpc.send(message, {
|
|
9447
|
+
priority: options.priority,
|
|
9448
|
+
mode: options.acknowledge
|
|
9449
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
9450
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
9451
|
+
signal: options.signal,
|
|
9452
|
+
});
|
|
9453
|
+
}
|
|
8386
9454
|
}
|
|
8387
|
-
|
|
9455
|
+
} else {
|
|
9456
|
+
for await (const message of createExchangeHeadsMessages(
|
|
8388
9457
|
this.log,
|
|
8389
|
-
|
|
8390
|
-
this._logProperties?.sync?.profile,
|
|
9458
|
+
chunk,
|
|
8391
9459
|
)) {
|
|
8392
|
-
if (!isStillCurrent())
|
|
8393
|
-
|
|
9460
|
+
if (!isStillCurrent()) return false;
|
|
9461
|
+
if (options.repairHint) {
|
|
9462
|
+
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8394
9463
|
}
|
|
8395
|
-
message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
|
|
8396
9464
|
await this.rpc.send(message, {
|
|
8397
|
-
priority:
|
|
8398
|
-
mode:
|
|
8399
|
-
|
|
9465
|
+
priority: options.priority,
|
|
9466
|
+
mode: options.acknowledge
|
|
9467
|
+
? new AcknowledgeDelivery({ to: [target], redundancy: 1 })
|
|
9468
|
+
: new SilentDelivery({ to: [target], redundancy: 1 }),
|
|
9469
|
+
signal: options.signal,
|
|
8400
9470
|
});
|
|
8401
9471
|
}
|
|
8402
|
-
return;
|
|
8403
9472
|
}
|
|
8404
|
-
|
|
8405
|
-
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
|
|
8410
|
-
|
|
8411
|
-
|
|
8412
|
-
|
|
8413
|
-
|
|
9473
|
+
return isStillCurrent();
|
|
9474
|
+
}
|
|
9475
|
+
|
|
9476
|
+
private async pushEntryHashes(
|
|
9477
|
+
target: string,
|
|
9478
|
+
hashes: string[],
|
|
9479
|
+
options: {
|
|
9480
|
+
acknowledge?: boolean;
|
|
9481
|
+
chunkTimeout?: () => number;
|
|
9482
|
+
chunkSize?: number;
|
|
9483
|
+
isStillCurrent?: () => boolean;
|
|
9484
|
+
onChunkAttempted?: (hashes: readonly string[]) => void;
|
|
9485
|
+
onChunkSent?: (hashes: readonly string[]) => Promise<boolean>;
|
|
9486
|
+
operationQueue?: PQueue;
|
|
9487
|
+
priority?: number;
|
|
9488
|
+
repairHint?: boolean;
|
|
9489
|
+
signal?: AbortSignal;
|
|
9490
|
+
},
|
|
9491
|
+
) {
|
|
9492
|
+
const isStillCurrent = options.isStillCurrent ?? (() => true);
|
|
9493
|
+
if (!isStillCurrent()) return;
|
|
9494
|
+
const chunkSize = Math.max(1, options.chunkSize ?? hashes.length);
|
|
9495
|
+
for (let offset = 0; offset < hashes.length; offset += chunkSize) {
|
|
9496
|
+
const chunk = hashes.slice(offset, offset + chunkSize);
|
|
9497
|
+
const pushChunk = () => {
|
|
9498
|
+
options.onChunkAttempted?.(chunk);
|
|
9499
|
+
const chunkTimeout = options.chunkTimeout?.();
|
|
9500
|
+
const chunkSignal =
|
|
9501
|
+
chunkTimeout == null
|
|
9502
|
+
? options.signal
|
|
9503
|
+
: AbortSignal.any([
|
|
9504
|
+
...(options.signal ? [options.signal] : []),
|
|
9505
|
+
AbortSignal.timeout(Math.max(1, chunkTimeout)),
|
|
9506
|
+
]);
|
|
9507
|
+
return this.pushEntryHashChunk(
|
|
9508
|
+
target,
|
|
9509
|
+
chunk,
|
|
9510
|
+
{ ...options, signal: chunkSignal },
|
|
9511
|
+
isStillCurrent,
|
|
9512
|
+
);
|
|
9513
|
+
};
|
|
9514
|
+
const pushed = options.operationQueue
|
|
9515
|
+
? await options.operationQueue.add(pushChunk)
|
|
9516
|
+
: await pushChunk();
|
|
9517
|
+
if (pushed !== true) return;
|
|
9518
|
+
if (options.onChunkSent && !(await options.onChunkSent(chunk))) return;
|
|
8414
9519
|
}
|
|
8415
9520
|
}
|
|
8416
9521
|
|
|
9522
|
+
private async pushRepairEntries(
|
|
9523
|
+
target: string,
|
|
9524
|
+
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
9525
|
+
isStillCurrent: () => boolean = () => true,
|
|
9526
|
+
signal?: AbortSignal,
|
|
9527
|
+
) {
|
|
9528
|
+
return this.pushEntryHashes(target, [...entries.keys()], {
|
|
9529
|
+
isStillCurrent,
|
|
9530
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
9531
|
+
repairHint: true,
|
|
9532
|
+
signal,
|
|
9533
|
+
});
|
|
9534
|
+
}
|
|
9535
|
+
|
|
8417
9536
|
private async sendRepairEntriesWithTransport(
|
|
8418
9537
|
target: string,
|
|
8419
9538
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
@@ -10384,6 +11503,7 @@ export class SharedLog<
|
|
|
10384
11503
|
removed: ShallowOrFullEntry<T>[];
|
|
10385
11504
|
}> {
|
|
10386
11505
|
this.throwIfNativeDurableCommitFailed();
|
|
11506
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
10387
11507
|
const ownershipLifecycleController =
|
|
10388
11508
|
this.captureReplicationOwnershipLifecycle();
|
|
10389
11509
|
if (this._isAdaptiveReplicating) {
|
|
@@ -10394,18 +11514,73 @@ export class SharedLog<
|
|
|
10394
11514
|
options,
|
|
10395
11515
|
ownershipLifecycleController,
|
|
10396
11516
|
);
|
|
10397
|
-
|
|
10398
|
-
|
|
10399
|
-
|
|
10400
|
-
|
|
10401
|
-
|
|
10402
|
-
|
|
10403
|
-
|
|
10404
|
-
|
|
10405
|
-
|
|
10406
|
-
|
|
10407
|
-
|
|
10408
|
-
|
|
11517
|
+
let committedHashes: readonly string[] | undefined;
|
|
11518
|
+
if (persistedDelivery) {
|
|
11519
|
+
(appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
|
|
11520
|
+
hashes,
|
|
11521
|
+
) => {
|
|
11522
|
+
// The lower log reports the exact hashes immediately after their
|
|
11523
|
+
// irreversible local mutation and before entry initialization, trim,
|
|
11524
|
+
// or change callbacks can reject the append.
|
|
11525
|
+
committedHashes = hashes;
|
|
11526
|
+
};
|
|
11527
|
+
}
|
|
11528
|
+
let persistedDeadline: PersistedDeliveryDeadline | undefined;
|
|
11529
|
+
const throwIfDeliveryAborted = () => {
|
|
11530
|
+
if (persistedDeadline?.signal.aborted) {
|
|
11531
|
+
throw persistedDeadline.signal.reason ?? new AbortError();
|
|
11532
|
+
}
|
|
11533
|
+
};
|
|
11534
|
+
try {
|
|
11535
|
+
const result = await this.log.append(data, appendOptions);
|
|
11536
|
+
committedHashes ??= [result.entry.hash];
|
|
11537
|
+
persistedDeadline = persistedDelivery
|
|
11538
|
+
? this.createPersistedDeliveryDeadline(
|
|
11539
|
+
persistedDelivery,
|
|
11540
|
+
ownershipLifecycleController,
|
|
11541
|
+
1,
|
|
11542
|
+
)
|
|
11543
|
+
: undefined;
|
|
11544
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
11545
|
+
ownershipLifecycleController,
|
|
11546
|
+
);
|
|
11547
|
+
throwIfDeliveryAborted();
|
|
11548
|
+
await this.processLocalAppend(result.entry, result.removed, options, {
|
|
11549
|
+
minReplicasValue,
|
|
11550
|
+
ownershipLifecycleController,
|
|
11551
|
+
});
|
|
11552
|
+
throwIfDeliveryAborted();
|
|
11553
|
+
if (persistedDelivery && persistedDeadline) {
|
|
11554
|
+
await this.settlePersistedDelivery(
|
|
11555
|
+
[result.entry],
|
|
11556
|
+
minReplicasValue,
|
|
11557
|
+
persistedDelivery,
|
|
11558
|
+
ownershipLifecycleController,
|
|
11559
|
+
persistedDeadline,
|
|
11560
|
+
);
|
|
11561
|
+
}
|
|
11562
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
11563
|
+
ownershipLifecycleController,
|
|
11564
|
+
);
|
|
11565
|
+
return result;
|
|
11566
|
+
} catch (error) {
|
|
11567
|
+
if (persistedDelivery && committedHashes) {
|
|
11568
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
11569
|
+
}
|
|
11570
|
+
throw error;
|
|
11571
|
+
} finally {
|
|
11572
|
+
persistedDeadline?.dispose();
|
|
11573
|
+
}
|
|
11574
|
+
}
|
|
11575
|
+
|
|
11576
|
+
private rejectPersistedDeliveryOnTrustedLocalAppend(
|
|
11577
|
+
options?: SharedAppendOptions<T>,
|
|
11578
|
+
): void {
|
|
11579
|
+
if (this.getPersistedDeliveryOptions(options)) {
|
|
11580
|
+
throw new Error(
|
|
11581
|
+
"trusted local append paths require delivery=false; call deliverPersistedEntries after the local commit",
|
|
11582
|
+
);
|
|
11583
|
+
}
|
|
10409
11584
|
}
|
|
10410
11585
|
|
|
10411
11586
|
// Trusted local append path for callers that already validated the entry.
|
|
@@ -10417,6 +11592,7 @@ export class SharedLog<
|
|
|
10417
11592
|
removed: ShallowOrFullEntry<T>[];
|
|
10418
11593
|
}> {
|
|
10419
11594
|
this.throwIfNativeDurableCommitFailed();
|
|
11595
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10420
11596
|
const ownershipLifecycleController =
|
|
10421
11597
|
this.captureReplicationOwnershipLifecycle();
|
|
10422
11598
|
if (options?.canAppend || options?.onChange) {
|
|
@@ -10457,6 +11633,7 @@ export class SharedLog<
|
|
|
10457
11633
|
skipMissingNextJoin?: boolean;
|
|
10458
11634
|
resolveTrimmedEntries?: boolean;
|
|
10459
11635
|
payloadData?: Uint8Array;
|
|
11636
|
+
localCommitEvidence?: TrustedLocalCommitEvidence;
|
|
10460
11637
|
},
|
|
10461
11638
|
): Promise<{
|
|
10462
11639
|
entry: Entry<T>;
|
|
@@ -10466,6 +11643,7 @@ export class SharedLog<
|
|
|
10466
11643
|
appendCommit: PreparedLocalAppendCommit<R>;
|
|
10467
11644
|
}> {
|
|
10468
11645
|
this.throwIfNativeDurableCommitFailed();
|
|
11646
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10469
11647
|
const ownershipLifecycleController =
|
|
10470
11648
|
this.captureReplicationOwnershipLifecycle();
|
|
10471
11649
|
if (options?.canAppend || options?.onChange) {
|
|
@@ -10480,6 +11658,10 @@ export class SharedLog<
|
|
|
10480
11658
|
const { appendOptions, minReplicasValue } =
|
|
10481
11659
|
this.createLogAppendOptions(options);
|
|
10482
11660
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
11661
|
+
attachTrustedLocalCommitEvidence(
|
|
11662
|
+
appendOptions,
|
|
11663
|
+
properties?.localCommitEvidence,
|
|
11664
|
+
);
|
|
10483
11665
|
const result = await asTrustedLowerLog(this.log).appendLocallyPrepared(
|
|
10484
11666
|
data,
|
|
10485
11667
|
appendOptions,
|
|
@@ -10489,6 +11671,11 @@ export class SharedLog<
|
|
|
10489
11671
|
payloadData: properties?.payloadData,
|
|
10490
11672
|
},
|
|
10491
11673
|
);
|
|
11674
|
+
if (properties?.localCommitEvidence) {
|
|
11675
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
11676
|
+
result.appendFacts.hash,
|
|
11677
|
+
);
|
|
11678
|
+
}
|
|
10492
11679
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
10493
11680
|
ownershipLifecycleController,
|
|
10494
11681
|
);
|
|
@@ -10737,6 +11924,9 @@ export class SharedLog<
|
|
|
10737
11924
|
skipMissingNextJoin: properties?.skipMissingNextJoin,
|
|
10738
11925
|
resolveTrimmedEntries: properties?.resolveTrimmedEntries,
|
|
10739
11926
|
payloadData,
|
|
11927
|
+
...(properties?.localCommitEvidence
|
|
11928
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
11929
|
+
: undefined),
|
|
10740
11930
|
});
|
|
10741
11931
|
}
|
|
10742
11932
|
|
|
@@ -10747,6 +11937,7 @@ export class SharedLog<
|
|
|
10747
11937
|
properties?: PreparedPayloadCommitOnlyProperties,
|
|
10748
11938
|
): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
|
|
10749
11939
|
this.throwIfNativeDurableCommitFailed();
|
|
11940
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10750
11941
|
if (options?.canAppend || options?.onChange) {
|
|
10751
11942
|
throw new Error(
|
|
10752
11943
|
"appendLocallyPreparedPayloadCommitOnly does not accept canAppend or onChange hooks",
|
|
@@ -10770,6 +11961,10 @@ export class SharedLog<
|
|
|
10770
11961
|
const { appendOptions, minReplicasValue } =
|
|
10771
11962
|
this.createLogAppendOptions(options);
|
|
10772
11963
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
11964
|
+
attachTrustedLocalCommitEvidence(
|
|
11965
|
+
appendOptions,
|
|
11966
|
+
properties?.localCommitEvidence,
|
|
11967
|
+
);
|
|
10773
11968
|
const deferHeadCoordinatePersistence =
|
|
10774
11969
|
this.shouldDeferHeadCoordinatePersistence(options);
|
|
10775
11970
|
const nativeBackboneResult =
|
|
@@ -10819,6 +12014,7 @@ export class SharedLog<
|
|
|
10819
12014
|
properties?: PreparedPayloadCommitOnlyProperties,
|
|
10820
12015
|
): MaybePromise<PreparedPayloadCommitOnlyResult<T, R> | undefined> {
|
|
10821
12016
|
this.throwIfNativeDurableCommitFailed();
|
|
12017
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
10822
12018
|
if (options?.canAppend || options?.onChange) {
|
|
10823
12019
|
throw new Error(
|
|
10824
12020
|
"appendStrictNativeDocumentPayloadCommitOnly does not accept canAppend or onChange hooks",
|
|
@@ -10842,6 +12038,10 @@ export class SharedLog<
|
|
|
10842
12038
|
const { appendOptions, minReplicasValue } =
|
|
10843
12039
|
this.createLogAppendOptions(options);
|
|
10844
12040
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
12041
|
+
attachTrustedLocalCommitEvidence(
|
|
12042
|
+
appendOptions,
|
|
12043
|
+
properties?.localCommitEvidence,
|
|
12044
|
+
);
|
|
10845
12045
|
const result = this.appendLocallyPreparedPayloadNativeBackboneCommitOnly(
|
|
10846
12046
|
payloadData,
|
|
10847
12047
|
appendOptions,
|
|
@@ -10881,6 +12081,11 @@ export class SharedLog<
|
|
|
10881
12081
|
includeAppendFactsBytes: !deferHeadCoordinatePersistence,
|
|
10882
12082
|
});
|
|
10883
12083
|
return mapMaybePromise(resultMaybe, (result) => {
|
|
12084
|
+
if (result && properties?.localCommitEvidence) {
|
|
12085
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
12086
|
+
result.appendFacts.hash,
|
|
12087
|
+
);
|
|
12088
|
+
}
|
|
10884
12089
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
10885
12090
|
ownershipLifecycleController,
|
|
10886
12091
|
);
|
|
@@ -11461,25 +12666,28 @@ export class SharedLog<
|
|
|
11461
12666
|
// boundary separates the catch above from this statement and
|
|
11462
12667
|
// `rollback` can no longer fire. Nothing downstream rolls back
|
|
11463
12668
|
// (the retire below only warns), so the token is terminal here.
|
|
11464
|
-
|
|
11465
|
-
|
|
11466
|
-
|
|
11467
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
11468
|
-
prepared.appendFacts,
|
|
11469
|
-
prepared.removed,
|
|
11470
|
-
prepared.materializeEntry,
|
|
11471
|
-
{
|
|
11472
|
-
removedHashes: prepared.removedHashes,
|
|
11473
|
-
removedGids: prepared.removedGids,
|
|
11474
|
-
},
|
|
11475
|
-
);
|
|
11476
|
-
try {
|
|
11477
|
-
await this.completeNativeStrictDurableTransaction(
|
|
11478
|
-
nativeStrictTransaction,
|
|
12669
|
+
if (properties?.localCommitEvidence) {
|
|
12670
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
12671
|
+
prepared.appendFacts.hash,
|
|
11479
12672
|
);
|
|
11480
|
-
} catch (error) {
|
|
11481
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
11482
12673
|
}
|
|
12674
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
12675
|
+
nativeStrictTransaction,
|
|
12676
|
+
() => {
|
|
12677
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
12678
|
+
lowerPublicationRollback?.coordinateEntries,
|
|
12679
|
+
);
|
|
12680
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
12681
|
+
prepared.appendFacts,
|
|
12682
|
+
prepared.removed,
|
|
12683
|
+
prepared.materializeEntry,
|
|
12684
|
+
{
|
|
12685
|
+
removedHashes: prepared.removedHashes,
|
|
12686
|
+
removedGids: prepared.removedGids,
|
|
12687
|
+
},
|
|
12688
|
+
);
|
|
12689
|
+
},
|
|
12690
|
+
);
|
|
11483
12691
|
return finishResult;
|
|
11484
12692
|
});
|
|
11485
12693
|
}
|
|
@@ -12112,26 +13320,29 @@ export class SharedLog<
|
|
|
12112
13320
|
// Success seam: the finalizer acknowledge above is the last
|
|
12113
13321
|
// await inside the protected try, so `rollback` can no
|
|
12114
13322
|
// longer fire and the retire below only warns.
|
|
12115
|
-
|
|
12116
|
-
|
|
12117
|
-
|
|
12118
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
12119
|
-
prepared.appendFacts,
|
|
12120
|
-
prepared.removed,
|
|
12121
|
-
prepared.materializeEntry,
|
|
12122
|
-
{
|
|
12123
|
-
forgetNativeCoordinates: false,
|
|
12124
|
-
removedHashes: prepared.removedHashes,
|
|
12125
|
-
removedGids: prepared.removedGids,
|
|
12126
|
-
},
|
|
12127
|
-
);
|
|
12128
|
-
try {
|
|
12129
|
-
await this.completeNativeStrictDurableTransaction(
|
|
12130
|
-
nativeStrictTransaction,
|
|
13323
|
+
if (properties?.localCommitEvidence) {
|
|
13324
|
+
properties.localCommitEvidence.committedHashes.add(
|
|
13325
|
+
prepared.appendFacts.hash,
|
|
12131
13326
|
);
|
|
12132
|
-
} catch (error) {
|
|
12133
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
12134
13327
|
}
|
|
13328
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
13329
|
+
nativeStrictTransaction,
|
|
13330
|
+
() => {
|
|
13331
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
13332
|
+
rollbackCoordinateEntries,
|
|
13333
|
+
);
|
|
13334
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
13335
|
+
prepared.appendFacts,
|
|
13336
|
+
prepared.removed,
|
|
13337
|
+
prepared.materializeEntry,
|
|
13338
|
+
{
|
|
13339
|
+
forgetNativeCoordinates: false,
|
|
13340
|
+
removedHashes: prepared.removedHashes,
|
|
13341
|
+
removedGids: prepared.removedGids,
|
|
13342
|
+
},
|
|
13343
|
+
);
|
|
13344
|
+
},
|
|
13345
|
+
);
|
|
12135
13346
|
if (
|
|
12136
13347
|
commitBlocksInBackbone &&
|
|
12137
13348
|
!runtimeOnlyCoordinates &&
|
|
@@ -13224,73 +14435,73 @@ export class SharedLog<
|
|
|
13224
14435
|
nativeStrictTransaction,
|
|
13225
14436
|
);
|
|
13226
14437
|
});
|
|
13227
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13228
|
-
ownershipLifecycleController,
|
|
13229
|
-
);
|
|
13230
14438
|
} catch (error) {
|
|
13231
14439
|
return rollbackBatch(error);
|
|
13232
14440
|
}
|
|
13233
14441
|
// Success seam: `rollbackBatch` has exactly one call site (the catch
|
|
13234
14442
|
// above), and everything from here on escapes without any rollback.
|
|
13235
|
-
|
|
13236
|
-
|
|
13237
|
-
|
|
13238
|
-
|
|
13239
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13240
|
-
ownershipLifecycleController,
|
|
13241
|
-
);
|
|
13242
|
-
const appendCommits: PreparedLocalAppendCommit<R>[] = [];
|
|
13243
|
-
for (let i = 0; i < coordinateRows.length; i++) {
|
|
13244
|
-
const {
|
|
13245
|
-
facts,
|
|
13246
|
-
backboneAppend,
|
|
13247
|
-
coordinateFields,
|
|
13248
|
-
plannedCoordinateDeleteHashes,
|
|
13249
|
-
} = coordinateRows[i]!;
|
|
13250
|
-
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
13251
|
-
facts,
|
|
13252
|
-
[],
|
|
13253
|
-
appended.materializeEntries[i]!,
|
|
13254
|
-
{
|
|
13255
|
-
forgetNativeCoordinates: false,
|
|
13256
|
-
removedHashes: plannedCoordinateDeleteHashes,
|
|
13257
|
-
removedGids:
|
|
13258
|
-
backboneAppend.trimmedGids ??
|
|
13259
|
-
(backboneAppend.trimmed.length > 0
|
|
13260
|
-
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
13261
|
-
: undefined),
|
|
13262
|
-
},
|
|
13263
|
-
);
|
|
13264
|
-
if (!runtimeOnlyCoordinates && this.remoteBlocks.hasNotifyStoredHook()) {
|
|
13265
|
-
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
14443
|
+
if (properties?.localCommitEvidence) {
|
|
14444
|
+
for (const facts of appended.appendFacts) {
|
|
14445
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
13266
14446
|
}
|
|
13267
|
-
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
|
|
13268
|
-
facts,
|
|
13269
|
-
{
|
|
13270
|
-
hashNumber: backboneAppend.coordinate.hashNumber as NumberFromType<R>,
|
|
13271
|
-
coordinateFields,
|
|
13272
|
-
},
|
|
13273
|
-
);
|
|
13274
|
-
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
13275
|
-
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
13276
|
-
appended.documentTrimmedHeadsProcessed?.[i];
|
|
13277
|
-
appendCommit.documentPreviousContext =
|
|
13278
|
-
backboneAppend.documentPreviousContext;
|
|
13279
|
-
appendCommits.push(appendCommit);
|
|
13280
14447
|
}
|
|
13281
|
-
|
|
13282
|
-
await this.
|
|
14448
|
+
const appendCommits =
|
|
14449
|
+
await this.finishCommittedNativeStrictDurableTransaction(
|
|
13283
14450
|
nativeStrictTransaction,
|
|
14451
|
+
() => {
|
|
14452
|
+
this._coordinates.settleResidentCoordinateSnapshot(
|
|
14453
|
+
batchCoordinateRollback,
|
|
14454
|
+
);
|
|
14455
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
14456
|
+
ownershipLifecycleController,
|
|
14457
|
+
);
|
|
14458
|
+
const commits: PreparedLocalAppendCommit<R>[] = [];
|
|
14459
|
+
for (let i = 0; i < coordinateRows.length; i++) {
|
|
14460
|
+
const {
|
|
14461
|
+
facts,
|
|
14462
|
+
backboneAppend,
|
|
14463
|
+
coordinateFields,
|
|
14464
|
+
plannedCoordinateDeleteHashes,
|
|
14465
|
+
} = coordinateRows[i]!;
|
|
14466
|
+
this.applyPreparedAppendFactsWithDeferredCoordinateDeletes(
|
|
14467
|
+
facts,
|
|
14468
|
+
[],
|
|
14469
|
+
appended.materializeEntries[i]!,
|
|
14470
|
+
{
|
|
14471
|
+
forgetNativeCoordinates: false,
|
|
14472
|
+
removedHashes: plannedCoordinateDeleteHashes,
|
|
14473
|
+
removedGids:
|
|
14474
|
+
backboneAppend.trimmedGids ??
|
|
14475
|
+
(backboneAppend.trimmed.length > 0
|
|
14476
|
+
? backboneAppend.trimmed.map((entry) => entry.gid)
|
|
14477
|
+
: undefined),
|
|
14478
|
+
},
|
|
14479
|
+
);
|
|
14480
|
+
if (
|
|
14481
|
+
!runtimeOnlyCoordinates &&
|
|
14482
|
+
this.remoteBlocks.hasNotifyStoredHook()
|
|
14483
|
+
) {
|
|
14484
|
+
this.remoteBlocks.notifyStoredDeferred(facts.hash);
|
|
14485
|
+
}
|
|
14486
|
+
const appendCommit = this.createPreparedLocalAppendCommitFromFacts(
|
|
14487
|
+
facts,
|
|
14488
|
+
{
|
|
14489
|
+
hashNumber: backboneAppend.coordinate
|
|
14490
|
+
.hashNumber as NumberFromType<R>,
|
|
14491
|
+
coordinateFields,
|
|
14492
|
+
},
|
|
14493
|
+
);
|
|
14494
|
+
appendCommit.nativeBackboneDocumentIndexCommitted = true;
|
|
14495
|
+
appendCommit.nativeBackboneDocumentIndexTrimmedHeadsProcessed =
|
|
14496
|
+
appended.documentTrimmedHeadsProcessed?.[i];
|
|
14497
|
+
appendCommit.documentPreviousContext =
|
|
14498
|
+
backboneAppend.documentPreviousContext;
|
|
14499
|
+
commits.push(appendCommit);
|
|
14500
|
+
}
|
|
14501
|
+
return commits;
|
|
14502
|
+
},
|
|
14503
|
+
() => this.isRepairLifecycleActive(ownershipLifecycleController),
|
|
13284
14504
|
);
|
|
13285
|
-
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13286
|
-
ownershipLifecycleController,
|
|
13287
|
-
);
|
|
13288
|
-
} catch (error) {
|
|
13289
|
-
if (!this.isRepairLifecycleActive(ownershipLifecycleController)) {
|
|
13290
|
-
throw error;
|
|
13291
|
-
}
|
|
13292
|
-
warn(`Failed to retire committed native intent: ${String(error)}`);
|
|
13293
|
-
}
|
|
13294
14505
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13295
14506
|
ownershipLifecycleController,
|
|
13296
14507
|
);
|
|
@@ -13322,6 +14533,7 @@ export class SharedLog<
|
|
|
13322
14533
|
| undefined
|
|
13323
14534
|
> {
|
|
13324
14535
|
this.throwIfNativeDurableCommitFailed();
|
|
14536
|
+
this.rejectPersistedDeliveryOnTrustedLocalAppend(options);
|
|
13325
14537
|
if (data.length === 0) {
|
|
13326
14538
|
return { entries: [], removed: [], appendCommits: [] };
|
|
13327
14539
|
}
|
|
@@ -13339,6 +14551,10 @@ export class SharedLog<
|
|
|
13339
14551
|
const { appendOptions, minReplicasValue } =
|
|
13340
14552
|
this.createLogAppendOptions(options);
|
|
13341
14553
|
appendOptions.__peerbitCanAppendAlreadyValidated = true;
|
|
14554
|
+
attachTrustedLocalCommitEvidence(
|
|
14555
|
+
appendOptions,
|
|
14556
|
+
properties?.localCommitEvidence,
|
|
14557
|
+
);
|
|
13342
14558
|
const nativeBackboneBatch =
|
|
13343
14559
|
await this.appendLocallyPreparedPayloadsManyNativeBackboneDocumentIndexBatch(
|
|
13344
14560
|
data,
|
|
@@ -13361,6 +14577,11 @@ export class SharedLog<
|
|
|
13361
14577
|
payloadDatas: properties?.payloadDatas,
|
|
13362
14578
|
nexts: properties?.nexts,
|
|
13363
14579
|
});
|
|
14580
|
+
if (result && properties?.localCommitEvidence) {
|
|
14581
|
+
for (const facts of result.appendFacts) {
|
|
14582
|
+
properties.localCommitEvidence.committedHashes.add(facts.hash);
|
|
14583
|
+
}
|
|
14584
|
+
}
|
|
13364
14585
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
13365
14586
|
ownershipLifecycleController,
|
|
13366
14587
|
);
|
|
@@ -13490,6 +14711,9 @@ export class SharedLog<
|
|
|
13490
14711
|
nativeBackboneDocumentIndexes:
|
|
13491
14712
|
properties?.nativeBackboneDocumentIndexes,
|
|
13492
14713
|
retainMaterializationBytes: properties?.retainMaterializationBytes,
|
|
14714
|
+
...(properties?.localCommitEvidence
|
|
14715
|
+
? { localCommitEvidence: properties.localCommitEvidence }
|
|
14716
|
+
: undefined),
|
|
13493
14717
|
},
|
|
13494
14718
|
);
|
|
13495
14719
|
}
|
|
@@ -13502,9 +14726,15 @@ export class SharedLog<
|
|
|
13502
14726
|
removed: ShallowOrFullEntry<T>[];
|
|
13503
14727
|
}> {
|
|
13504
14728
|
this.throwIfNativeDurableCommitFailed();
|
|
14729
|
+
const persistedDelivery = this.getPersistedDeliveryOptions(options);
|
|
13505
14730
|
if (data.length === 0) {
|
|
13506
14731
|
return { entries: [], removed: [] };
|
|
13507
14732
|
}
|
|
14733
|
+
if (persistedDelivery) {
|
|
14734
|
+
throw new Error(
|
|
14735
|
+
"persisted delivery is not supported for chained appendMany; use independent document puts",
|
|
14736
|
+
);
|
|
14737
|
+
}
|
|
13508
14738
|
const ownershipLifecycleController =
|
|
13509
14739
|
this.captureReplicationOwnershipLifecycle();
|
|
13510
14740
|
if (this._isAdaptiveReplicating) {
|
|
@@ -13581,6 +14811,123 @@ export class SharedLog<
|
|
|
13581
14811
|
return result;
|
|
13582
14812
|
}
|
|
13583
14813
|
|
|
14814
|
+
/**
|
|
14815
|
+
* Deliver entries that were already committed locally, then wait for the
|
|
14816
|
+
* requested persisted remote quorum. This is the post-commit seam used by
|
|
14817
|
+
* higher-level transactional/batched writers so a receipt timeout never
|
|
14818
|
+
* rolls back or hides their successful local commit.
|
|
14819
|
+
*/
|
|
14820
|
+
private async deliverPersistedPlanningEntries(
|
|
14821
|
+
resolveEntries: () => PersistedDeliveryPlanningEntry<T, R>[],
|
|
14822
|
+
committedHashes: string[],
|
|
14823
|
+
options: SharedAppendOptions<T>,
|
|
14824
|
+
): Promise<void> {
|
|
14825
|
+
let persistedDeadline: PersistedDeliveryDeadline | undefined;
|
|
14826
|
+
try {
|
|
14827
|
+
const delivery = this.getPersistedDeliveryOptions(options);
|
|
14828
|
+
if (!delivery) {
|
|
14829
|
+
throw new Error(
|
|
14830
|
+
'deliverPersistedEntries requires reliability="persisted"',
|
|
14831
|
+
);
|
|
14832
|
+
}
|
|
14833
|
+
const ownershipLifecycleController =
|
|
14834
|
+
this.captureReplicationOwnershipLifecycle();
|
|
14835
|
+
const deadline = this.createPersistedDeliveryDeadline(
|
|
14836
|
+
delivery,
|
|
14837
|
+
ownershipLifecycleController,
|
|
14838
|
+
committedHashes.length,
|
|
14839
|
+
);
|
|
14840
|
+
persistedDeadline = deadline;
|
|
14841
|
+
const throwIfDeliveryAborted = () => {
|
|
14842
|
+
if (deadline.signal.aborted) {
|
|
14843
|
+
throw deadline.signal.reason ?? new AbortError();
|
|
14844
|
+
}
|
|
14845
|
+
if (Date.now() >= deadline.deadline) {
|
|
14846
|
+
throw new TimeoutError(
|
|
14847
|
+
`Timed out waiting for ${Math.floor(delivery.minAcks!)} persisted remote replicas.`,
|
|
14848
|
+
);
|
|
14849
|
+
}
|
|
14850
|
+
};
|
|
14851
|
+
const entries = resolveEntries();
|
|
14852
|
+
throwIfDeliveryAborted();
|
|
14853
|
+
if (entries.length === 0) return;
|
|
14854
|
+
const { minReplicasValue } = this.createLogAppendOptions(
|
|
14855
|
+
options,
|
|
14856
|
+
ownershipLifecycleController,
|
|
14857
|
+
);
|
|
14858
|
+
throwIfDeliveryAborted();
|
|
14859
|
+
await this.settlePersistedDelivery(
|
|
14860
|
+
entries,
|
|
14861
|
+
minReplicasValue,
|
|
14862
|
+
delivery,
|
|
14863
|
+
ownershipLifecycleController,
|
|
14864
|
+
deadline,
|
|
14865
|
+
true,
|
|
14866
|
+
);
|
|
14867
|
+
} catch (error) {
|
|
14868
|
+
throw new PersistedDeliveryError(error, committedHashes);
|
|
14869
|
+
} finally {
|
|
14870
|
+
persistedDeadline?.dispose();
|
|
14871
|
+
}
|
|
14872
|
+
}
|
|
14873
|
+
|
|
14874
|
+
private createPersistedDeliveryPlanningEntries(
|
|
14875
|
+
appendCommits: PreparedLocalAppendCommit<R>[],
|
|
14876
|
+
materializeEntries: () => Entry<T>[],
|
|
14877
|
+
): PersistedDeliveryPlanningEntry<T, R>[] {
|
|
14878
|
+
let materializedEntries: Entry<T>[] | undefined;
|
|
14879
|
+
const requiresFullEntries =
|
|
14880
|
+
this.findLeadersFromEntry !== SharedLog.prototype.findLeadersFromEntry;
|
|
14881
|
+
const getMaterializedEntry = (
|
|
14882
|
+
appendCommit: PreparedLocalAppendCommit<R>,
|
|
14883
|
+
index: number,
|
|
14884
|
+
) => {
|
|
14885
|
+
materializedEntries ??= materializeEntries();
|
|
14886
|
+
const entry = materializedEntries[index];
|
|
14887
|
+
if (!entry || entry.hash !== appendCommit.hash) {
|
|
14888
|
+
throw new Error(
|
|
14889
|
+
`Persisted delivery materializer did not return committed entry ${appendCommit.hash}`,
|
|
14890
|
+
);
|
|
14891
|
+
}
|
|
14892
|
+
return entry;
|
|
14893
|
+
};
|
|
14894
|
+
return appendCommits.map((appendCommit, index) => {
|
|
14895
|
+
if (!requiresFullEntries && appendCommit.coordinateFields) {
|
|
14896
|
+
return this._coordinates.materializeResidentCoordinateEntry(
|
|
14897
|
+
appendCommit.coordinateFields,
|
|
14898
|
+
);
|
|
14899
|
+
}
|
|
14900
|
+
return getMaterializedEntry(appendCommit, index);
|
|
14901
|
+
});
|
|
14902
|
+
}
|
|
14903
|
+
|
|
14904
|
+
private deliverPersistedAppendCommits(
|
|
14905
|
+
appendCommits: PreparedLocalAppendCommit<R>[],
|
|
14906
|
+
materializeEntries: () => Entry<T>[],
|
|
14907
|
+
options: SharedAppendOptions<T>,
|
|
14908
|
+
): Promise<void> {
|
|
14909
|
+
return this.deliverPersistedPlanningEntries(
|
|
14910
|
+
() =>
|
|
14911
|
+
this.createPersistedDeliveryPlanningEntries(
|
|
14912
|
+
appendCommits,
|
|
14913
|
+
materializeEntries,
|
|
14914
|
+
),
|
|
14915
|
+
appendCommits.map((appendCommit) => appendCommit.hash),
|
|
14916
|
+
options,
|
|
14917
|
+
);
|
|
14918
|
+
}
|
|
14919
|
+
|
|
14920
|
+
async deliverPersistedEntries(
|
|
14921
|
+
entries: Entry<T>[],
|
|
14922
|
+
options: SharedAppendOptions<T>,
|
|
14923
|
+
): Promise<void> {
|
|
14924
|
+
return this.deliverPersistedPlanningEntries(
|
|
14925
|
+
() => entries,
|
|
14926
|
+
entries.map((entry) => entry.hash),
|
|
14927
|
+
options,
|
|
14928
|
+
);
|
|
14929
|
+
}
|
|
14930
|
+
|
|
13584
14931
|
private canCoalesceLocalAppendMany(
|
|
13585
14932
|
entries: Entry<T>[],
|
|
13586
14933
|
options?: SharedAppendOptions<T>,
|
|
@@ -14722,6 +16069,9 @@ export class SharedLog<
|
|
|
14722
16069
|
this._peerSyncCapabilities = new Map();
|
|
14723
16070
|
this._peerSyncCapabilitySessions = new Map();
|
|
14724
16071
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
16072
|
+
this._persistedReceiptStorage = undefined;
|
|
16073
|
+
this._persistedReceiptRequestsInFlight = new Map();
|
|
16074
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
14725
16075
|
this._liveRawGossipBatches = new Map();
|
|
14726
16076
|
this._liveRawGossipFlushScheduled = false;
|
|
14727
16077
|
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
@@ -14864,6 +16214,43 @@ export class SharedLog<
|
|
|
14864
16214
|
|
|
14865
16215
|
const fanoutService = getSharedLogFanoutService(this.node.services);
|
|
14866
16216
|
const blockProviderNamespace = (cid: string) => `cid:${cid}`;
|
|
16217
|
+
const logProviderNamespace = `shared-log|${this.topic}`;
|
|
16218
|
+
const announceBlockProvider = async (cid: string): Promise<void> => {
|
|
16219
|
+
try {
|
|
16220
|
+
await fanoutService?.announceProvider(blockProviderNamespace(cid), {
|
|
16221
|
+
ttlMs: 120_000,
|
|
16222
|
+
bootstrapMaxPeers: 2,
|
|
16223
|
+
});
|
|
16224
|
+
} catch {
|
|
16225
|
+
// Provider publication is best-effort.
|
|
16226
|
+
}
|
|
16227
|
+
};
|
|
16228
|
+
const announceBlockProviders = async (cids: string[]): Promise<void> => {
|
|
16229
|
+
const batchedAnnounce = fanoutService?.announceProviders;
|
|
16230
|
+
if (typeof batchedAnnounce === "function") {
|
|
16231
|
+
const namespaces = function* () {
|
|
16232
|
+
for (const cid of cids) yield blockProviderNamespace(cid);
|
|
16233
|
+
};
|
|
16234
|
+
await batchedAnnounce.call(fanoutService, namespaces(), {
|
|
16235
|
+
ttlMs: 120_000,
|
|
16236
|
+
bootstrapMaxPeers: 2,
|
|
16237
|
+
});
|
|
16238
|
+
return;
|
|
16239
|
+
}
|
|
16240
|
+
|
|
16241
|
+
// Tolerate a skewed runtime that supplied an older FanoutTree service.
|
|
16242
|
+
let nextIndex = 0;
|
|
16243
|
+
const worker = async () => {
|
|
16244
|
+
for (;;) {
|
|
16245
|
+
const index = nextIndex++;
|
|
16246
|
+
if (index >= cids.length) return;
|
|
16247
|
+
await announceBlockProvider(cids[index]!);
|
|
16248
|
+
}
|
|
16249
|
+
};
|
|
16250
|
+
await Promise.all(
|
|
16251
|
+
Array.from({ length: Math.min(8, cids.length) }, () => worker()),
|
|
16252
|
+
);
|
|
16253
|
+
};
|
|
14867
16254
|
const [replicationIndex, logIndex] = await Promise.all([
|
|
14868
16255
|
logScope.scope("replication"),
|
|
14869
16256
|
logScope.scope("log"),
|
|
@@ -14963,17 +16350,23 @@ export class SharedLog<
|
|
|
14963
16350
|
|
|
14964
16351
|
let directoryProviders: string[] = [];
|
|
14965
16352
|
try {
|
|
14966
|
-
|
|
14967
|
-
|
|
14968
|
-
|
|
14969
|
-
|
|
14970
|
-
|
|
14971
|
-
|
|
14972
|
-
|
|
14973
|
-
|
|
14974
|
-
|
|
14975
|
-
|
|
14976
|
-
)
|
|
16353
|
+
const query = (namespace: string) =>
|
|
16354
|
+
fanoutService?.queryProviders(namespace, {
|
|
16355
|
+
want: maxPeers,
|
|
16356
|
+
timeoutMs: 2_000,
|
|
16357
|
+
queryTimeoutMs: 500,
|
|
16358
|
+
bootstrapMaxPeers: 2,
|
|
16359
|
+
signal: opts?.signal,
|
|
16360
|
+
}) ?? Promise.resolve([]);
|
|
16361
|
+
const results = await Promise.allSettled([
|
|
16362
|
+
query(blockProviderNamespace(cid)),
|
|
16363
|
+
query(logProviderNamespace),
|
|
16364
|
+
]);
|
|
16365
|
+
for (const result of results) {
|
|
16366
|
+
if (result.status === "fulfilled") {
|
|
16367
|
+
directoryProviders.push(...result.value);
|
|
16368
|
+
}
|
|
16369
|
+
}
|
|
14977
16370
|
} catch {
|
|
14978
16371
|
// Ignore discovery failures; local evidence remains usable.
|
|
14979
16372
|
}
|
|
@@ -14981,11 +16374,7 @@ export class SharedLog<
|
|
|
14981
16374
|
const selected: string[] = [];
|
|
14982
16375
|
const selectedSet = new Set<string>();
|
|
14983
16376
|
const add = (peer: string | undefined) => {
|
|
14984
|
-
if (
|
|
14985
|
-
!peer ||
|
|
14986
|
-
selectedSet.has(peer) ||
|
|
14987
|
-
selected.length >= maxPeers
|
|
14988
|
-
) {
|
|
16377
|
+
if (!peer || selectedSet.has(peer) || selected.length >= maxPeers) {
|
|
14989
16378
|
return;
|
|
14990
16379
|
}
|
|
14991
16380
|
selectedSet.add(peer);
|
|
@@ -15003,31 +16392,46 @@ export class SharedLog<
|
|
|
15003
16392
|
return selected;
|
|
15004
16393
|
},
|
|
15005
16394
|
watchProviders: fanoutService
|
|
15006
|
-
? (cid, opts) =>
|
|
15007
|
-
|
|
15008
|
-
|
|
15009
|
-
|
|
15010
|
-
|
|
15011
|
-
|
|
15012
|
-
|
|
15013
|
-
|
|
15014
|
-
|
|
15015
|
-
|
|
15016
|
-
|
|
15017
|
-
|
|
15018
|
-
? async (cid) => {
|
|
15019
|
-
// Best-effort directory announce for "get without remote.from" workflows.
|
|
16395
|
+
? (cid, opts) => {
|
|
16396
|
+
const watch = (namespace: string) =>
|
|
16397
|
+
fanoutService.watchProviders(namespace, {
|
|
16398
|
+
signal: opts.signal,
|
|
16399
|
+
want: 8,
|
|
16400
|
+
ttlMs: 10_000,
|
|
16401
|
+
renewIntervalMs: 5_000,
|
|
16402
|
+
bootstrapMaxPeers: 2,
|
|
16403
|
+
onProviders: (providers) =>
|
|
16404
|
+
opts.onProviders(providers.map((provider) => provider.hash)),
|
|
16405
|
+
});
|
|
16406
|
+
const cidWatch = watch(blockProviderNamespace(cid));
|
|
15020
16407
|
try {
|
|
15021
|
-
|
|
15022
|
-
|
|
15023
|
-
{
|
|
15024
|
-
|
|
15025
|
-
|
|
16408
|
+
const logWatch = watch(logProviderNamespace);
|
|
16409
|
+
return {
|
|
16410
|
+
close: () => {
|
|
16411
|
+
cidWatch.close();
|
|
16412
|
+
logWatch.close();
|
|
15026
16413
|
},
|
|
15027
|
-
|
|
16414
|
+
};
|
|
16415
|
+
} catch (error) {
|
|
16416
|
+
cidWatch.close();
|
|
16417
|
+
throw error;
|
|
16418
|
+
}
|
|
16419
|
+
}
|
|
16420
|
+
: undefined,
|
|
16421
|
+
onPut: fanoutService ? announceBlockProvider : undefined,
|
|
16422
|
+
onPutMany: fanoutService
|
|
16423
|
+
? (cids) => {
|
|
16424
|
+
// A renewable log-wide provider lease makes every CID in a
|
|
16425
|
+
// stored batch discoverable to current readers. Retain the
|
|
16426
|
+
// per-CID directory publications for released readers that only
|
|
16427
|
+
// know the legacy namespace; the bounded workers avoid creating
|
|
16428
|
+
// one in-flight promise per block.
|
|
16429
|
+
try {
|
|
16430
|
+
this.ensureLogProviderHandle(fanoutService);
|
|
15028
16431
|
} catch {
|
|
15029
16432
|
// ignore announce failures
|
|
15030
16433
|
}
|
|
16434
|
+
return announceBlockProviders(cids);
|
|
15031
16435
|
}
|
|
15032
16436
|
: undefined,
|
|
15033
16437
|
});
|
|
@@ -15279,6 +16683,7 @@ export class SharedLog<
|
|
|
15279
16683
|
},
|
|
15280
16684
|
indexer: logIndex,
|
|
15281
16685
|
});
|
|
16686
|
+
this._persistedReceiptStorage = this.resolvePersistedReceiptStorage();
|
|
15282
16687
|
try {
|
|
15283
16688
|
const recovered =
|
|
15284
16689
|
await this.recoverNativeStrictDurableTransactionIntent();
|
|
@@ -16077,13 +17482,12 @@ export class SharedLog<
|
|
|
16077
17482
|
this.topic,
|
|
16078
17483
|
);
|
|
16079
17484
|
// We do this here, because these calls requires this.closed == false
|
|
16080
|
-
void this.pruneOfflineReplicators()
|
|
16081
|
-
|
|
16082
|
-
|
|
16083
|
-
|
|
16084
|
-
|
|
16085
|
-
|
|
16086
|
-
});
|
|
17485
|
+
void this.pruneOfflineReplicators().catch((error) => {
|
|
17486
|
+
if (isNotStartedError(error as Error)) {
|
|
17487
|
+
return;
|
|
17488
|
+
}
|
|
17489
|
+
logger.error(error);
|
|
17490
|
+
});
|
|
16087
17491
|
|
|
16088
17492
|
this._liveness.startReplicatorLivenessSweep();
|
|
16089
17493
|
|
|
@@ -16402,7 +17806,8 @@ export class SharedLog<
|
|
|
16402
17806
|
for (const peer of reachable) {
|
|
16403
17807
|
if (confirmed?.has(peer)) liveEvidenceTiers[0].push(peer);
|
|
16404
17808
|
else if (contacted?.has(peer)) liveEvidenceTiers[1].push(peer);
|
|
16405
|
-
else if (this.uniqueReplicators.has(peer))
|
|
17809
|
+
else if (this.uniqueReplicators.has(peer))
|
|
17810
|
+
liveEvidenceTiers[2].push(peer);
|
|
16406
17811
|
}
|
|
16407
17812
|
const liveKnown: string[] = [];
|
|
16408
17813
|
for (let tier = 0; tier < liveEvidenceTiers.length; tier++) {
|
|
@@ -16972,8 +18377,7 @@ export class SharedLog<
|
|
|
16972
18377
|
): Promise<void> {
|
|
16973
18378
|
if (!gids) return;
|
|
16974
18379
|
const candidates = [...new Set(gids)].filter(
|
|
16975
|
-
(gid) =>
|
|
16976
|
-
history === this._gidPeersHistory && !!gid && history.has(gid),
|
|
18380
|
+
(gid) => history === this._gidPeersHistory && !!gid && history.has(gid),
|
|
16977
18381
|
);
|
|
16978
18382
|
if (candidates.length === 0) return;
|
|
16979
18383
|
const hasHeads = await this.hasAnyHeadForGidSets(
|
|
@@ -17007,9 +18411,7 @@ export class SharedLog<
|
|
|
17007
18411
|
for (const gid of gids) {
|
|
17008
18412
|
if (!gid || !history.has(gid)) continue;
|
|
17009
18413
|
if (state.pending.has(gid)) continue;
|
|
17010
|
-
if (
|
|
17011
|
-
state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY
|
|
17012
|
-
) {
|
|
18414
|
+
if (state.pending.size >= GID_PEER_HISTORY_CLEANUP_PENDING_CAPACITY) {
|
|
17013
18415
|
// History is only a suppression memo. Under sustained synchronous
|
|
17014
18416
|
// production, forgetting an overflow row is the bounded, safe fallback:
|
|
17015
18417
|
// it can cause redundant delivery but cannot affect ownership or data.
|
|
@@ -17808,6 +19210,9 @@ export class SharedLog<
|
|
|
17808
19210
|
this._peerSyncCapabilities?.clear();
|
|
17809
19211
|
this._peerSyncCapabilitySessions?.clear();
|
|
17810
19212
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
19213
|
+
this._persistedReceiptStorage = undefined;
|
|
19214
|
+
this._persistedReceiptRequestsInFlight?.clear();
|
|
19215
|
+
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
17811
19216
|
this._v2Receive?.clearForClose();
|
|
17812
19217
|
this._v2Send?.clearForClose();
|
|
17813
19218
|
this._liveRawGossipBatches?.clear();
|
|
@@ -18580,7 +19985,11 @@ export class SharedLog<
|
|
|
18580
19985
|
async onMessage(
|
|
18581
19986
|
msg: TransportMessage,
|
|
18582
19987
|
context: RequestContext,
|
|
18583
|
-
): Promise<void
|
|
19988
|
+
): Promise<void>;
|
|
19989
|
+
async onMessage(
|
|
19990
|
+
msg: TransportMessage,
|
|
19991
|
+
context: RequestContext,
|
|
19992
|
+
): Promise<any> {
|
|
18584
19993
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
18585
19994
|
? msg
|
|
18586
19995
|
: undefined;
|
|
@@ -18656,6 +20065,7 @@ export class SharedLog<
|
|
|
18656
20065
|
if (
|
|
18657
20066
|
!context.from.equals(this.node.identity.publicKey) &&
|
|
18658
20067
|
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
20068
|
+
!(msg instanceof RequestPersistedEntriesV1) &&
|
|
18659
20069
|
!isReplicationInfoV2Message(msg)
|
|
18660
20070
|
) {
|
|
18661
20071
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
@@ -20102,24 +21512,30 @@ export class SharedLog<
|
|
|
20102
21512
|
);
|
|
20103
21513
|
}
|
|
20104
21514
|
if (syncProfile) {
|
|
20105
|
-
emitSyncProfileDuration(
|
|
20106
|
-
|
|
20107
|
-
|
|
20108
|
-
|
|
20109
|
-
|
|
20110
|
-
|
|
20111
|
-
|
|
20112
|
-
|
|
20113
|
-
|
|
21515
|
+
emitSyncProfileDuration(
|
|
21516
|
+
syncProfile,
|
|
21517
|
+
coordinatePersistStartedAt,
|
|
21518
|
+
{
|
|
21519
|
+
name: "sharedLog.receive.coordinatePersist",
|
|
21520
|
+
component: "shared-log",
|
|
21521
|
+
entries: entriesToPersist.length,
|
|
21522
|
+
messages: 1,
|
|
21523
|
+
details: {
|
|
21524
|
+
reusedLeaderPlans: reusableCoordinatePersistItemCount,
|
|
21525
|
+
nativeBackboneOnly:
|
|
21526
|
+
nativeBackboneOnlyPersistedHashes?.size ?? 0,
|
|
21527
|
+
},
|
|
20114
21528
|
},
|
|
20115
|
-
|
|
21529
|
+
);
|
|
20116
21530
|
}
|
|
20117
21531
|
for (const hash of admittedHashes) {
|
|
20118
21532
|
confirmedHashes.add(hash);
|
|
20119
21533
|
}
|
|
20120
21534
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
20121
21535
|
const ownershipChangedDuringReceive =
|
|
20122
|
-
!this.isReceiveOwnershipSnapshotStable(
|
|
21536
|
+
!this.isReceiveOwnershipSnapshotStable(
|
|
21537
|
+
receiveOwnershipRevision,
|
|
21538
|
+
);
|
|
20123
21539
|
if (ownershipChangedDuringReceive) {
|
|
20124
21540
|
const freshAuditRevision =
|
|
20125
21541
|
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
@@ -20296,7 +21712,13 @@ export class SharedLog<
|
|
|
20296
21712
|
};
|
|
20297
21713
|
// The prelude already threw when `context.from` was missing.
|
|
20298
21714
|
const laneRequestContext = context as ReceiveRequestContext;
|
|
20299
|
-
if (msg instanceof
|
|
21715
|
+
if (msg instanceof RequestPersistedEntriesV1) {
|
|
21716
|
+
return await this.handleRequestPersistedEntriesV1(
|
|
21717
|
+
msg,
|
|
21718
|
+
laneRequestContext,
|
|
21719
|
+
lane,
|
|
21720
|
+
);
|
|
21721
|
+
} else if (msg instanceof RequestIPruneV2) {
|
|
20300
21722
|
await this.handleRequestIPruneV2(msg, laneRequestContext, lane);
|
|
20301
21723
|
} else if (msg instanceof ResponseIPruneV2) {
|
|
20302
21724
|
await this.handleResponseIPruneV2(msg, laneRequestContext, lane);
|
|
@@ -20503,6 +21925,322 @@ export class SharedLog<
|
|
|
20503
21925
|
// wire-stash release, lease release, durable-poison recheck).
|
|
20504
21926
|
// -----------------------------------------------------------------
|
|
20505
21927
|
|
|
21928
|
+
private isPersistedReceiptRequestSessionCurrent(
|
|
21929
|
+
request: RequestPersistedEntriesV1,
|
|
21930
|
+
context: ReceiveRequestContext,
|
|
21931
|
+
lane: ReceiveLaneContext,
|
|
21932
|
+
): boolean {
|
|
21933
|
+
const session = lane.session;
|
|
21934
|
+
return (
|
|
21935
|
+
!!this._persistedReceiptStorage &&
|
|
21936
|
+
!context.from.equals(this.node.identity.publicKey) &&
|
|
21937
|
+
request.expectedReceiverSession === this.ownTransportSession() &&
|
|
21938
|
+
session !== null &&
|
|
21939
|
+
session.phase === "open" &&
|
|
21940
|
+
this._peerSessions.isCurrent(lane.fromHash, session) &&
|
|
21941
|
+
!this._peerSessions.isReplicationInfoBlocked(lane.fromHash) &&
|
|
21942
|
+
this._peerSessions.isReceiveCleanupGateOpen(lane.fromHash) &&
|
|
21943
|
+
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
21944
|
+
context.message.header.session &&
|
|
21945
|
+
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
21946
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
21947
|
+
);
|
|
21948
|
+
}
|
|
21949
|
+
|
|
21950
|
+
private admitPersistedReceiptIngress(
|
|
21951
|
+
peer: string,
|
|
21952
|
+
transportSession: bigint,
|
|
21953
|
+
hashCount: number,
|
|
21954
|
+
now = Date.now(),
|
|
21955
|
+
): boolean {
|
|
21956
|
+
// Charge malformed empty/oversized vectors too. The request was already
|
|
21957
|
+
// decoded by the transport, so letting an invalid shape bypass this bucket
|
|
21958
|
+
// would leave an authenticated peer with an unmetered validation/logging
|
|
21959
|
+
// path. Clamp only the accounting cost; shape validation still rejects the
|
|
21960
|
+
// request below.
|
|
21961
|
+
const hashCost = Math.max(
|
|
21962
|
+
1,
|
|
21963
|
+
Math.min(MAX_PERSISTED_RECEIPT_HASHES, hashCount),
|
|
21964
|
+
);
|
|
21965
|
+
let nodeBudget = persistedReceiptIngressBudgets.get(this.node);
|
|
21966
|
+
if (!nodeBudget) {
|
|
21967
|
+
nodeBudget = {
|
|
21968
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
21969
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
21970
|
+
refilledAt: now,
|
|
21971
|
+
peerSessions: new Map(),
|
|
21972
|
+
};
|
|
21973
|
+
persistedReceiptIngressBudgets.set(this.node, nodeBudget);
|
|
21974
|
+
}
|
|
21975
|
+
refillPersistedReceiptIngressBucket(
|
|
21976
|
+
nodeBudget,
|
|
21977
|
+
now,
|
|
21978
|
+
PERSISTED_RECEIPT_INGRESS_NODE_REQUEST_CAPACITY,
|
|
21979
|
+
PERSISTED_RECEIPT_INGRESS_NODE_HASH_CAPACITY,
|
|
21980
|
+
PERSISTED_RECEIPT_INGRESS_NODE_REQUESTS_PER_SECOND,
|
|
21981
|
+
PERSISTED_RECEIPT_INGRESS_NODE_HASHES_PER_SECOND,
|
|
21982
|
+
);
|
|
21983
|
+
|
|
21984
|
+
const peerSessionKey = `${peer}\0${transportSession}`;
|
|
21985
|
+
let peerBudget = nodeBudget.peerSessions.get(peerSessionKey);
|
|
21986
|
+
if (!peerBudget) {
|
|
21987
|
+
while (
|
|
21988
|
+
nodeBudget.peerSessions.size >=
|
|
21989
|
+
MAX_PERSISTED_RECEIPT_INGRESS_PEER_SESSIONS
|
|
21990
|
+
) {
|
|
21991
|
+
const oldest = nodeBudget.peerSessions.keys().next().value;
|
|
21992
|
+
if (oldest === undefined) break;
|
|
21993
|
+
nodeBudget.peerSessions.delete(oldest);
|
|
21994
|
+
}
|
|
21995
|
+
peerBudget = {
|
|
21996
|
+
requestTokens: PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
21997
|
+
hashTokens: PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
21998
|
+
refilledAt: now,
|
|
21999
|
+
};
|
|
22000
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
22001
|
+
} else {
|
|
22002
|
+
// Refresh insertion order so bounded eviction prefers inactive sessions.
|
|
22003
|
+
nodeBudget.peerSessions.delete(peerSessionKey);
|
|
22004
|
+
nodeBudget.peerSessions.set(peerSessionKey, peerBudget);
|
|
22005
|
+
}
|
|
22006
|
+
refillPersistedReceiptIngressBucket(
|
|
22007
|
+
peerBudget,
|
|
22008
|
+
now,
|
|
22009
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY,
|
|
22010
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY,
|
|
22011
|
+
PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND,
|
|
22012
|
+
PERSISTED_RECEIPT_INGRESS_PEER_HASHES_PER_SECOND,
|
|
22013
|
+
);
|
|
22014
|
+
|
|
22015
|
+
if (
|
|
22016
|
+
nodeBudget.requestTokens < 1 ||
|
|
22017
|
+
nodeBudget.hashTokens < hashCost ||
|
|
22018
|
+
peerBudget.requestTokens < 1 ||
|
|
22019
|
+
peerBudget.hashTokens < hashCost
|
|
22020
|
+
) {
|
|
22021
|
+
return false;
|
|
22022
|
+
}
|
|
22023
|
+
nodeBudget.requestTokens -= 1;
|
|
22024
|
+
nodeBudget.hashTokens -= hashCost;
|
|
22025
|
+
peerBudget.requestTokens -= 1;
|
|
22026
|
+
peerBudget.hashTokens -= hashCost;
|
|
22027
|
+
return true;
|
|
22028
|
+
}
|
|
22029
|
+
|
|
22030
|
+
private async handleRequestPersistedEntriesV1(
|
|
22031
|
+
request: RequestPersistedEntriesV1,
|
|
22032
|
+
context: ReceiveRequestContext,
|
|
22033
|
+
lane: ReceiveLaneContext,
|
|
22034
|
+
): Promise<ConfirmEntriesMessage | undefined> {
|
|
22035
|
+
if (!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)) {
|
|
22036
|
+
return undefined;
|
|
22037
|
+
}
|
|
22038
|
+
if (
|
|
22039
|
+
!this.admitPersistedReceiptIngress(
|
|
22040
|
+
lane.fromHash,
|
|
22041
|
+
context.message.header.session,
|
|
22042
|
+
request.hashes.length,
|
|
22043
|
+
)
|
|
22044
|
+
) {
|
|
22045
|
+
return undefined;
|
|
22046
|
+
}
|
|
22047
|
+
// Charge the request before shape validation and attacker-controlled CID
|
|
22048
|
+
// parsing. Invalid shapes are rejected quietly here so they cannot turn the
|
|
22049
|
+
// outer receive error logger into a post-budget work amplifier.
|
|
22050
|
+
try {
|
|
22051
|
+
this.validatePersistedReceiptRequestShape(request);
|
|
22052
|
+
} catch {
|
|
22053
|
+
return undefined;
|
|
22054
|
+
}
|
|
22055
|
+
if (!this.hasValidPersistedReceiptHashes(request)) {
|
|
22056
|
+
return undefined;
|
|
22057
|
+
}
|
|
22058
|
+
const peerInFlight =
|
|
22059
|
+
this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 0;
|
|
22060
|
+
if (
|
|
22061
|
+
peerInFlight >= MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER ||
|
|
22062
|
+
this._persistedReceiptRequestsInFlightTotal >=
|
|
22063
|
+
MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL
|
|
22064
|
+
) {
|
|
22065
|
+
return undefined;
|
|
22066
|
+
}
|
|
22067
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, peerInFlight + 1);
|
|
22068
|
+
this._persistedReceiptRequestsInFlightTotal++;
|
|
22069
|
+
|
|
22070
|
+
try {
|
|
22071
|
+
// A positive receipt requires the block itself. Reject entirely absent
|
|
22072
|
+
// batches before the serialized mutation lane and, critically, before a
|
|
22073
|
+
// request can force unrelated pending index/coordinate journals durable.
|
|
22074
|
+
// A concurrent put can yield a harmless false negative: the sender retries.
|
|
22075
|
+
const blockPresence = await this.remoteBlocks.localStore.hasMany(
|
|
22076
|
+
request.hashes,
|
|
22077
|
+
);
|
|
22078
|
+
const presentBlockHashes = request.hashes.filter(
|
|
22079
|
+
(_hash, index) => blockPresence[index] === true,
|
|
22080
|
+
);
|
|
22081
|
+
if (
|
|
22082
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22083
|
+
) {
|
|
22084
|
+
return undefined;
|
|
22085
|
+
}
|
|
22086
|
+
if (presentBlockHashes.length === 0) {
|
|
22087
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22088
|
+
}
|
|
22089
|
+
this._liveness.markReplicatorActivity(lane.fromHash);
|
|
22090
|
+
return await this.withReplicationRangeMutationQueue(async () => {
|
|
22091
|
+
if (
|
|
22092
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22093
|
+
) {
|
|
22094
|
+
return undefined;
|
|
22095
|
+
}
|
|
22096
|
+
this.throwIfNativeDurableCommitFailed();
|
|
22097
|
+
const storage = this.resolvePersistedReceiptStorage();
|
|
22098
|
+
if (!storage) {
|
|
22099
|
+
return undefined;
|
|
22100
|
+
}
|
|
22101
|
+
|
|
22102
|
+
await this.log.entryIndex.flushPendingWrites(presentBlockHashes);
|
|
22103
|
+
await this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
22104
|
+
const candidates = new Map(
|
|
22105
|
+
await Promise.all(
|
|
22106
|
+
presentBlockHashes.map(async (hash) => {
|
|
22107
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
22108
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22109
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22110
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(
|
|
22111
|
+
hash,
|
|
22112
|
+
),
|
|
22113
|
+
]);
|
|
22114
|
+
return [hash, { blockPresent, lowerRow, coordinate }] as const;
|
|
22115
|
+
}),
|
|
22116
|
+
),
|
|
22117
|
+
);
|
|
22118
|
+
const presentHashes = presentBlockHashes.filter((hash) => {
|
|
22119
|
+
const candidate = candidates.get(hash)!;
|
|
22120
|
+
return (
|
|
22121
|
+
candidate.blockPresent &&
|
|
22122
|
+
candidate.lowerRow != null &&
|
|
22123
|
+
candidate.coordinate != null
|
|
22124
|
+
);
|
|
22125
|
+
});
|
|
22126
|
+
if (presentHashes.length === 0) {
|
|
22127
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22128
|
+
}
|
|
22129
|
+
await Promise.all(
|
|
22130
|
+
[...new Set([storage.block, storage.lower, storage.coordinate])].map(
|
|
22131
|
+
(store) => store.barrier(),
|
|
22132
|
+
),
|
|
22133
|
+
);
|
|
22134
|
+
this.throwIfNativeDurableCommitFailed();
|
|
22135
|
+
if (
|
|
22136
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22137
|
+
) {
|
|
22138
|
+
return undefined;
|
|
22139
|
+
}
|
|
22140
|
+
|
|
22141
|
+
const ownershipRevision =
|
|
22142
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
22143
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
22144
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22145
|
+
}
|
|
22146
|
+
|
|
22147
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
22148
|
+
const confirmed: string[] = [];
|
|
22149
|
+
for (const hash of presentHashes) {
|
|
22150
|
+
const candidate = candidates.get(hash)!;
|
|
22151
|
+
if (
|
|
22152
|
+
!candidate.blockPresent ||
|
|
22153
|
+
!candidate.lowerRow ||
|
|
22154
|
+
!candidate.coordinate ||
|
|
22155
|
+
this._checkedPrune.hasActiveWork(hash) ||
|
|
22156
|
+
!this.isPersistedReceiptRequestSessionCurrent(
|
|
22157
|
+
request,
|
|
22158
|
+
context,
|
|
22159
|
+
lane,
|
|
22160
|
+
) ||
|
|
22161
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
22162
|
+
) {
|
|
22163
|
+
continue;
|
|
22164
|
+
}
|
|
22165
|
+
|
|
22166
|
+
const [blockPresent, lowerRow, coordinate] = await Promise.all([
|
|
22167
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22168
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22169
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
22170
|
+
]);
|
|
22171
|
+
if (!blockPresent || !lowerRow || !coordinate) {
|
|
22172
|
+
continue;
|
|
22173
|
+
}
|
|
22174
|
+
|
|
22175
|
+
const replicas = decodeReplicas(coordinate).getValue(this);
|
|
22176
|
+
const leaders = await this.findLeadersFromEntry(
|
|
22177
|
+
coordinate,
|
|
22178
|
+
replicas,
|
|
22179
|
+
{ freshLeaderPlan: true },
|
|
22180
|
+
lane.ownershipLifecycleController,
|
|
22181
|
+
);
|
|
22182
|
+
if (!leaders.has(selfHash)) {
|
|
22183
|
+
continue;
|
|
22184
|
+
}
|
|
22185
|
+
if (
|
|
22186
|
+
!this._checkedPrune.hasActiveWork(hash) &&
|
|
22187
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision) &&
|
|
22188
|
+
this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22189
|
+
) {
|
|
22190
|
+
confirmed.push(hash);
|
|
22191
|
+
}
|
|
22192
|
+
}
|
|
22193
|
+
|
|
22194
|
+
if (
|
|
22195
|
+
confirmed.length === 0 ||
|
|
22196
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
22197
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22198
|
+
) {
|
|
22199
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22200
|
+
}
|
|
22201
|
+
const finalRows = await Promise.all(
|
|
22202
|
+
confirmed.map(async (hash) => {
|
|
22203
|
+
const [block, lower, coordinate] = await Promise.all([
|
|
22204
|
+
this.remoteBlocks.localStore.has(hash),
|
|
22205
|
+
this.log.entryIndex.properties.index.get(toId(hash)),
|
|
22206
|
+
this._coordinates.getAuthoritativeCoordinateEntryForReceipt(hash),
|
|
22207
|
+
]);
|
|
22208
|
+
return { hash, block, lower, coordinate };
|
|
22209
|
+
}),
|
|
22210
|
+
);
|
|
22211
|
+
if (
|
|
22212
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision) ||
|
|
22213
|
+
!this.isPersistedReceiptRequestSessionCurrent(request, context, lane)
|
|
22214
|
+
) {
|
|
22215
|
+
return new ConfirmEntriesMessage({ hashes: [] });
|
|
22216
|
+
}
|
|
22217
|
+
return new ConfirmEntriesMessage({
|
|
22218
|
+
hashes: finalRows
|
|
22219
|
+
.filter(
|
|
22220
|
+
(row) =>
|
|
22221
|
+
row.block &&
|
|
22222
|
+
row.lower &&
|
|
22223
|
+
row.coordinate &&
|
|
22224
|
+
!this._checkedPrune.hasActiveWork(row.hash),
|
|
22225
|
+
)
|
|
22226
|
+
.map((row) => row.hash),
|
|
22227
|
+
});
|
|
22228
|
+
}, lane.ownershipLifecycleController);
|
|
22229
|
+
} finally {
|
|
22230
|
+
const remaining =
|
|
22231
|
+
(this._persistedReceiptRequestsInFlight.get(lane.fromHash) ?? 1) - 1;
|
|
22232
|
+
if (remaining <= 0) {
|
|
22233
|
+
this._persistedReceiptRequestsInFlight.delete(lane.fromHash);
|
|
22234
|
+
} else {
|
|
22235
|
+
this._persistedReceiptRequestsInFlight.set(lane.fromHash, remaining);
|
|
22236
|
+
}
|
|
22237
|
+
this._persistedReceiptRequestsInFlightTotal = Math.max(
|
|
22238
|
+
0,
|
|
22239
|
+
this._persistedReceiptRequestsInFlightTotal - 1,
|
|
22240
|
+
);
|
|
22241
|
+
}
|
|
22242
|
+
}
|
|
22243
|
+
|
|
20506
22244
|
private async handleRequestIPruneV2(
|
|
20507
22245
|
msg: RequestIPruneV2,
|
|
20508
22246
|
context: ReceiveRequestContext,
|