@peerbit/shared-log 16.0.21 → 16.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +80 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +768 -34
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +11 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +24 -5
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +21 -0
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +104 -30
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/package.json +9 -9
- package/src/index.ts +1101 -35
- package/src/replication-info-v2-receive.ts +32 -5
- package/src/replication-info-v2-send.ts +168 -38
package/src/index.ts
CHANGED
|
@@ -719,6 +719,14 @@ type PersistedDeliveryPlanningRecord<T, R extends "u32" | "u64"> = Readonly<{
|
|
|
719
719
|
createFullPlanningSource?: () => Entry<T>;
|
|
720
720
|
}>;
|
|
721
721
|
|
|
722
|
+
type PersistedAppendBackfillSource<T, R extends "u32" | "u64"> = {
|
|
723
|
+
entry: Entry<T>;
|
|
724
|
+
coordinates: NumberFromType<R>[];
|
|
725
|
+
assignmentExtraLeaders: LeaderMap;
|
|
726
|
+
deliveryExtraTargets: Set<string>;
|
|
727
|
+
extrasOwnershipRevision?: number;
|
|
728
|
+
};
|
|
729
|
+
|
|
722
730
|
type NativeBackboneSimpleDocumentProjectionPlan = {
|
|
723
731
|
documentVariantType?: "u8" | "string";
|
|
724
732
|
documentVariantValue?: string;
|
|
@@ -1744,6 +1752,71 @@ export type DeliveryOptions = {
|
|
|
1744
1752
|
signal?: AbortSignal;
|
|
1745
1753
|
};
|
|
1746
1754
|
|
|
1755
|
+
export type PersistedReceiptPeerReadinessPendingReason =
|
|
1756
|
+
| "closed"
|
|
1757
|
+
| "no-current-session"
|
|
1758
|
+
| "session-opening"
|
|
1759
|
+
| "capability-pending"
|
|
1760
|
+
| "replication-state-pending"
|
|
1761
|
+
| "replication-confirmation-pending"
|
|
1762
|
+
| "not-replicating"
|
|
1763
|
+
| "not-entry-leader"
|
|
1764
|
+
| "ownership-changing";
|
|
1765
|
+
|
|
1766
|
+
export type PersistedReceiptPeerReadinessUnsupportedReason =
|
|
1767
|
+
| "persisted-receipts-unsupported"
|
|
1768
|
+
| "replication-confirmation-unsupported";
|
|
1769
|
+
|
|
1770
|
+
/**
|
|
1771
|
+
* Detached view of one public key's current persisted-receipt generation.
|
|
1772
|
+
* `generation` is opaque: callers may compare it for equality, but must not
|
|
1773
|
+
* interpret its contents or use it as a future-session capability. Equal
|
|
1774
|
+
* generations mean the connection/receive/capability binding is unchanged;
|
|
1775
|
+
* leadership and outbound confirmation can still change within a generation.
|
|
1776
|
+
*/
|
|
1777
|
+
export type PersistedReceiptPeerReadiness =
|
|
1778
|
+
| Readonly<{
|
|
1779
|
+
status: "ready";
|
|
1780
|
+
generation: string;
|
|
1781
|
+
}>
|
|
1782
|
+
| Readonly<{
|
|
1783
|
+
status: "pending";
|
|
1784
|
+
reason: PersistedReceiptPeerReadinessPendingReason;
|
|
1785
|
+
generation?: string;
|
|
1786
|
+
}>
|
|
1787
|
+
| Readonly<{
|
|
1788
|
+
status: "unsupported";
|
|
1789
|
+
reason: PersistedReceiptPeerReadinessUnsupportedReason;
|
|
1790
|
+
generation: string;
|
|
1791
|
+
}>;
|
|
1792
|
+
|
|
1793
|
+
export type PersistedReceiptPeerReady = Extract<
|
|
1794
|
+
PersistedReceiptPeerReadiness,
|
|
1795
|
+
{ status: "ready" }
|
|
1796
|
+
>;
|
|
1797
|
+
|
|
1798
|
+
export type PersistedReceiptPeerReadinessOptions<
|
|
1799
|
+
T,
|
|
1800
|
+
R extends "u32" | "u64",
|
|
1801
|
+
> = Readonly<{
|
|
1802
|
+
/** Require this peer to be a freshly planned leader for every entry. */
|
|
1803
|
+
entries?: readonly (ShallowOrFullEntry<T> | EntryReplicated<R>)[];
|
|
1804
|
+
/**
|
|
1805
|
+
* Total leader-plan replica degree used for `entries`; defaults to this log's
|
|
1806
|
+
* configured minimum. This is not the persisted delivery `minAcks` count.
|
|
1807
|
+
*/
|
|
1808
|
+
replicas?: number;
|
|
1809
|
+
}>;
|
|
1810
|
+
|
|
1811
|
+
export type WaitForPersistedReceiptPeerReadinessOptions<
|
|
1812
|
+
T,
|
|
1813
|
+
R extends "u32" | "u64",
|
|
1814
|
+
> = PersistedReceiptPeerReadinessOptions<T, R> &
|
|
1815
|
+
Readonly<{
|
|
1816
|
+
timeout?: number;
|
|
1817
|
+
signal?: AbortSignal;
|
|
1818
|
+
}>;
|
|
1819
|
+
|
|
1747
1820
|
type PersistedDeliveryOptions = Readonly<{
|
|
1748
1821
|
reliability: "persisted";
|
|
1749
1822
|
minAcks: number;
|
|
@@ -1785,6 +1858,7 @@ const PERSISTED_RECEIPT_RETRY_MS = 50;
|
|
|
1785
1858
|
const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
|
|
1786
1859
|
const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
|
|
1787
1860
|
const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
|
|
1861
|
+
const MAX_PERSISTED_RECEIPT_READINESS_WAITERS = 1_024;
|
|
1788
1862
|
const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
|
|
1789
1863
|
const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
|
|
1790
1864
|
const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
|
|
@@ -2159,6 +2233,10 @@ export type ReplicatorLeaveEvent = { publicKey: PublicSignKey };
|
|
|
2159
2233
|
export type ReplicationChangeEvent = { publicKey: PublicSignKey };
|
|
2160
2234
|
export type ReplicatorMatureEvent = { publicKey: PublicSignKey };
|
|
2161
2235
|
export type ReplicationStatusEvent = ReplicationStatus;
|
|
2236
|
+
/** `peerHash` is the result of `PublicSignKey.hashcode()`. */
|
|
2237
|
+
export type PersistedReceiptPeerReadinessEvent = Readonly<{
|
|
2238
|
+
peerHash: string;
|
|
2239
|
+
}>;
|
|
2162
2240
|
|
|
2163
2241
|
class ReplicationStatusSnapshotChangedError extends Error {
|
|
2164
2242
|
constructor() {
|
|
@@ -2181,6 +2259,12 @@ export interface SharedLogEvents extends ProgramEvents {
|
|
|
2181
2259
|
"replication:change": CustomEvent<ReplicationChangeEvent>;
|
|
2182
2260
|
"replicator:mature": CustomEvent<ReplicatorMatureEvent>;
|
|
2183
2261
|
"replication:status": CustomEvent<ReplicationStatusEvent>;
|
|
2262
|
+
/**
|
|
2263
|
+
* Non-exhaustive wake hint that a peer may now produce a new readiness
|
|
2264
|
+
* snapshot. Consumers must re-read the snapshot; this event is deliberately
|
|
2265
|
+
* not a durable transition log and a `ready` result remains advisory.
|
|
2266
|
+
*/
|
|
2267
|
+
"persisted-receipt:readiness": CustomEvent<PersistedReceiptPeerReadinessEvent>;
|
|
2184
2268
|
}
|
|
2185
2269
|
|
|
2186
2270
|
export type SharedLogRuntimeSnapshot = Readonly<{
|
|
@@ -3734,6 +3818,23 @@ export class SharedLog<
|
|
|
3734
3818
|
// parallel map so existing capability-number consumers remain unchanged.
|
|
3735
3819
|
private _peerSyncCapabilitySessions!: Map<string, bigint>;
|
|
3736
3820
|
private _peerSyncCapabilityTimestamps!: Map<string, bigint>;
|
|
3821
|
+
// design-note: these fields cache a stable, public diagnostics token for the
|
|
3822
|
+
// composite of PeerSession identity, receive epoch, and signed capability
|
|
3823
|
+
// session. They are not consulted to admit or fence asynchronous work. A
|
|
3824
|
+
// separate opaque token is necessary because exposing any of those internal
|
|
3825
|
+
// identities would leak protocol/session values, while PeerSession alone does
|
|
3826
|
+
// not change when receive or capability state is replaced.
|
|
3827
|
+
private _persistedReceiptReadinessGenerations!: WeakMap<
|
|
3828
|
+
PeerSession,
|
|
3829
|
+
{
|
|
3830
|
+
receiveEpoch: object | null;
|
|
3831
|
+
capabilitySession?: bigint;
|
|
3832
|
+
generation: string;
|
|
3833
|
+
}
|
|
3834
|
+
>;
|
|
3835
|
+
private _persistedReceiptReadinessGenerationPrefix!: string;
|
|
3836
|
+
private _persistedReceiptReadinessGenerationCounter!: number;
|
|
3837
|
+
private _persistedReceiptReadinessWaiters!: Set<object>;
|
|
3737
3838
|
private _persistedReceiptStorage?: PersistedReceiptStorage;
|
|
3738
3839
|
private _persistedReceiptRequestsInFlight!: Map<string, number>;
|
|
3739
3840
|
private _persistedReceiptRequestsInFlightTotal!: number;
|
|
@@ -4091,6 +4192,12 @@ export class SharedLog<
|
|
|
4091
4192
|
this._peerSyncCapabilities = new Map();
|
|
4092
4193
|
this._peerSyncCapabilitySessions = new Map();
|
|
4093
4194
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
4195
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
4196
|
+
this._persistedReceiptReadinessGenerationPrefix = toHexString(
|
|
4197
|
+
randomBytes(8),
|
|
4198
|
+
);
|
|
4199
|
+
this._persistedReceiptReadinessGenerationCounter = 0;
|
|
4200
|
+
this._persistedReceiptReadinessWaiters = new Set();
|
|
4094
4201
|
this._persistedReceiptStorage = undefined;
|
|
4095
4202
|
this._persistedReceiptRequestsInFlight = new Map();
|
|
4096
4203
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -5053,15 +5160,23 @@ export class SharedLog<
|
|
|
5053
5160
|
) {
|
|
5054
5161
|
return false;
|
|
5055
5162
|
}
|
|
5163
|
+
const nextCapabilities = previous.capabilities | capabilities;
|
|
5164
|
+
const nextTimestamp =
|
|
5165
|
+
previous.timestamp === undefined || timestamp > previous.timestamp
|
|
5166
|
+
? timestamp
|
|
5167
|
+
: previous.timestamp;
|
|
5056
5168
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
5057
5169
|
epoch: openingSession,
|
|
5058
|
-
capabilities:
|
|
5170
|
+
capabilities: nextCapabilities,
|
|
5059
5171
|
transportSession,
|
|
5060
|
-
timestamp:
|
|
5061
|
-
previous.timestamp === undefined || timestamp > previous.timestamp
|
|
5062
|
-
? timestamp
|
|
5063
|
-
: previous.timestamp,
|
|
5172
|
+
timestamp: nextTimestamp,
|
|
5064
5173
|
});
|
|
5174
|
+
if (
|
|
5175
|
+
previous.capabilities !== nextCapabilities ||
|
|
5176
|
+
previous.timestamp === undefined
|
|
5177
|
+
) {
|
|
5178
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5179
|
+
}
|
|
5065
5180
|
return true;
|
|
5066
5181
|
}
|
|
5067
5182
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
@@ -5070,17 +5185,25 @@ export class SharedLog<
|
|
|
5070
5185
|
transportSession,
|
|
5071
5186
|
timestamp,
|
|
5072
5187
|
});
|
|
5188
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5073
5189
|
return true;
|
|
5074
5190
|
}
|
|
5075
5191
|
|
|
5076
5192
|
if (transportSession === undefined || timestamp === undefined) {
|
|
5077
5193
|
// Test/in-process synthetic contexts predate signed envelope captures.
|
|
5078
5194
|
// They may exercise capability-number behavior, but can never authorize V2.
|
|
5195
|
+
const readinessChanged =
|
|
5196
|
+
this._peerSyncCapabilities.get(peerHash) !== capabilities ||
|
|
5197
|
+
this._peerSyncCapabilitySessions.has(peerHash) ||
|
|
5198
|
+
this._peerSyncCapabilityTimestamps.has(peerHash);
|
|
5079
5199
|
this._peerSyncCapabilities.set(peerHash, capabilities);
|
|
5080
5200
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
5081
5201
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
5082
5202
|
this._v2Send.advancePeerCapability(peerHash);
|
|
5083
5203
|
this._v2Receive.revokePeerCapability(peerHash);
|
|
5204
|
+
if (readinessChanged) {
|
|
5205
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5206
|
+
}
|
|
5084
5207
|
return true;
|
|
5085
5208
|
}
|
|
5086
5209
|
|
|
@@ -5105,6 +5228,10 @@ export class SharedLog<
|
|
|
5105
5228
|
!sameTransportSession ||
|
|
5106
5229
|
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
5107
5230
|
(nextCapabilities & senderGrantCapabilityMask);
|
|
5231
|
+
const readinessChanged =
|
|
5232
|
+
!sameTransportSession ||
|
|
5233
|
+
previousTimestamp === undefined ||
|
|
5234
|
+
previousCapabilities !== nextCapabilities;
|
|
5108
5235
|
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
5109
5236
|
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
5110
5237
|
this._peerSyncCapabilityTimestamps.set(
|
|
@@ -5121,6 +5248,9 @@ export class SharedLog<
|
|
|
5121
5248
|
// recovery re-solicitation may restart from the base interval.
|
|
5122
5249
|
this.resetReplicationInfoV2RecoveryEscalation(peerHash);
|
|
5123
5250
|
}
|
|
5251
|
+
if (readinessChanged) {
|
|
5252
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
5253
|
+
}
|
|
5124
5254
|
return true;
|
|
5125
5255
|
}
|
|
5126
5256
|
|
|
@@ -5470,20 +5600,173 @@ export class SharedLog<
|
|
|
5470
5600
|
return this.sendFusedRawExchangeHeadsPlan(plan, to, options);
|
|
5471
5601
|
}
|
|
5472
5602
|
|
|
5603
|
+
private persistedReceiptReadinessGeneration(
|
|
5604
|
+
peerSession: PeerSession,
|
|
5605
|
+
receiveEpoch: object | null,
|
|
5606
|
+
capabilitySession: bigint | undefined,
|
|
5607
|
+
): string {
|
|
5608
|
+
const current = this._persistedReceiptReadinessGenerations.get(peerSession);
|
|
5609
|
+
if (
|
|
5610
|
+
current?.receiveEpoch === receiveEpoch &&
|
|
5611
|
+
current.capabilitySession === capabilitySession
|
|
5612
|
+
) {
|
|
5613
|
+
return current.generation;
|
|
5614
|
+
}
|
|
5615
|
+
const generation = `${this._persistedReceiptReadinessGenerationPrefix}:${(++this
|
|
5616
|
+
._persistedReceiptReadinessGenerationCounter).toString(36)}`;
|
|
5617
|
+
this._persistedReceiptReadinessGenerations.set(peerSession, {
|
|
5618
|
+
receiveEpoch,
|
|
5619
|
+
capabilitySession,
|
|
5620
|
+
generation,
|
|
5621
|
+
});
|
|
5622
|
+
return generation;
|
|
5623
|
+
}
|
|
5624
|
+
|
|
5625
|
+
private pendingPersistedReceiptReadiness(
|
|
5626
|
+
reason: PersistedReceiptPeerReadinessPendingReason,
|
|
5627
|
+
generation?: string,
|
|
5628
|
+
): PersistedReceiptPeerReadiness {
|
|
5629
|
+
return Object.freeze({
|
|
5630
|
+
status: "pending" as const,
|
|
5631
|
+
reason,
|
|
5632
|
+
...(generation === undefined ? {} : { generation }),
|
|
5633
|
+
});
|
|
5634
|
+
}
|
|
5635
|
+
|
|
5636
|
+
private unsupportedPersistedReceiptReadiness(
|
|
5637
|
+
reason: PersistedReceiptPeerReadinessUnsupportedReason,
|
|
5638
|
+
generation: string,
|
|
5639
|
+
): PersistedReceiptPeerReadiness {
|
|
5640
|
+
return Object.freeze({
|
|
5641
|
+
status: "unsupported" as const,
|
|
5642
|
+
reason,
|
|
5643
|
+
generation,
|
|
5644
|
+
});
|
|
5645
|
+
}
|
|
5646
|
+
|
|
5647
|
+
private dispatchPersistedReceiptReadinessChange(peerHash: string): void {
|
|
5648
|
+
this.events.dispatchEvent(
|
|
5649
|
+
new CustomEvent<PersistedReceiptPeerReadinessEvent>(
|
|
5650
|
+
"persisted-receipt:readiness",
|
|
5651
|
+
{ detail: Object.freeze({ peerHash }) },
|
|
5652
|
+
),
|
|
5653
|
+
);
|
|
5654
|
+
}
|
|
5655
|
+
|
|
5656
|
+
private persistedReceiptReadinessCandidate(peerHash: string):
|
|
5657
|
+
| {
|
|
5658
|
+
capabilitySession: bigint;
|
|
5659
|
+
peerSession: PeerSession;
|
|
5660
|
+
receiveEpoch: object | null;
|
|
5661
|
+
generation: string;
|
|
5662
|
+
}
|
|
5663
|
+
| PersistedReceiptPeerReadiness {
|
|
5664
|
+
if (this.closed) {
|
|
5665
|
+
return this.pendingPersistedReceiptReadiness("closed");
|
|
5666
|
+
}
|
|
5667
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
5668
|
+
if (!peerSession) {
|
|
5669
|
+
return this.pendingPersistedReceiptReadiness("no-current-session");
|
|
5670
|
+
}
|
|
5671
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
5672
|
+
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5673
|
+
const generation = this.persistedReceiptReadinessGeneration(
|
|
5674
|
+
peerSession,
|
|
5675
|
+
receiveEpoch,
|
|
5676
|
+
capabilitySession,
|
|
5677
|
+
);
|
|
5678
|
+
if (
|
|
5679
|
+
peerSession.phase !== "open" ||
|
|
5680
|
+
!peerSession.isActive() ||
|
|
5681
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
5682
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash)
|
|
5683
|
+
) {
|
|
5684
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5685
|
+
"session-opening",
|
|
5686
|
+
generation,
|
|
5687
|
+
);
|
|
5688
|
+
}
|
|
5689
|
+
if (
|
|
5690
|
+
capabilitySession === undefined ||
|
|
5691
|
+
!this._peerSyncCapabilityTimestamps.has(peerHash)
|
|
5692
|
+
) {
|
|
5693
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5694
|
+
"capability-pending",
|
|
5695
|
+
generation,
|
|
5696
|
+
);
|
|
5697
|
+
}
|
|
5698
|
+
const capabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
5699
|
+
if ((capabilities & SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) === 0) {
|
|
5700
|
+
return this.unsupportedPersistedReceiptReadiness(
|
|
5701
|
+
"persisted-receipts-unsupported",
|
|
5702
|
+
generation,
|
|
5703
|
+
);
|
|
5704
|
+
}
|
|
5705
|
+
if ((capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) === 0) {
|
|
5706
|
+
return this.unsupportedPersistedReceiptReadiness(
|
|
5707
|
+
"replication-confirmation-unsupported",
|
|
5708
|
+
generation,
|
|
5709
|
+
);
|
|
5710
|
+
}
|
|
5711
|
+
if (
|
|
5712
|
+
!this._v2Receive.isCurrentActive({
|
|
5713
|
+
peerHash,
|
|
5714
|
+
peerSession,
|
|
5715
|
+
receiveEpoch,
|
|
5716
|
+
senderTransportSession: capabilitySession,
|
|
5717
|
+
})
|
|
5718
|
+
) {
|
|
5719
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5720
|
+
"replication-state-pending",
|
|
5721
|
+
generation,
|
|
5722
|
+
);
|
|
5723
|
+
}
|
|
5724
|
+
if (!this.uniqueReplicators.has(peerHash)) {
|
|
5725
|
+
return this.pendingPersistedReceiptReadiness(
|
|
5726
|
+
"not-replicating",
|
|
5727
|
+
generation,
|
|
5728
|
+
);
|
|
5729
|
+
}
|
|
5730
|
+
return {
|
|
5731
|
+
capabilitySession,
|
|
5732
|
+
peerSession,
|
|
5733
|
+
receiveEpoch,
|
|
5734
|
+
generation,
|
|
5735
|
+
};
|
|
5736
|
+
}
|
|
5737
|
+
|
|
5473
5738
|
private persistedReceiptPeerSession(
|
|
5474
5739
|
peerHash: string,
|
|
5475
5740
|
): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
|
|
5741
|
+
// This is a hot receipt/transfer-loop predicate. Keep it allocation-light,
|
|
5742
|
+
// while mirroring every exact-session gate in
|
|
5743
|
+
// persistedReceiptReadinessCandidate (which additionally creates public
|
|
5744
|
+
// reason/generation snapshots).
|
|
5476
5745
|
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5477
5746
|
const peerSession = this._peerSessions.current(peerHash);
|
|
5747
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
5748
|
+
const requiredCapabilities =
|
|
5749
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS |
|
|
5750
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM;
|
|
5478
5751
|
if (
|
|
5752
|
+
this.closed ||
|
|
5479
5753
|
capabilitySession == null ||
|
|
5480
5754
|
!peerSession ||
|
|
5481
5755
|
peerSession.phase !== "open" ||
|
|
5482
|
-
!
|
|
5756
|
+
!peerSession.isActive() ||
|
|
5757
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
5758
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
5759
|
+
!this.uniqueReplicators.has(peerHash) ||
|
|
5483
5760
|
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
5484
5761
|
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
5485
|
-
|
|
5486
|
-
|
|
5762
|
+
requiredCapabilities) !==
|
|
5763
|
+
requiredCapabilities ||
|
|
5764
|
+
!this._v2Receive.isCurrentActive({
|
|
5765
|
+
peerHash,
|
|
5766
|
+
peerSession,
|
|
5767
|
+
receiveEpoch,
|
|
5768
|
+
senderTransportSession: capabilitySession,
|
|
5769
|
+
})
|
|
5487
5770
|
) {
|
|
5488
5771
|
return undefined;
|
|
5489
5772
|
}
|
|
@@ -5754,6 +6037,10 @@ export class SharedLog<
|
|
|
5754
6037
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
5755
6038
|
persistedDeadline?: PersistedDeliveryDeadline,
|
|
5756
6039
|
transferOnFirstRound = false,
|
|
6040
|
+
onFreshLeaderPlan?: (
|
|
6041
|
+
leadersByEntry: readonly LeaderMap[],
|
|
6042
|
+
ownershipRevision: number,
|
|
6043
|
+
) => void,
|
|
5757
6044
|
): Promise<void> {
|
|
5758
6045
|
const minAcks = Math.floor(delivery.minAcks!);
|
|
5759
6046
|
const records = new Map(
|
|
@@ -5844,6 +6131,8 @@ export class SharedLog<
|
|
|
5844
6131
|
ownershipLifecycleController,
|
|
5845
6132
|
);
|
|
5846
6133
|
if (!isRoundOwnershipCurrent()) continue;
|
|
6134
|
+
onFreshLeaderPlan?.(leadersByEntry, ownershipRevision);
|
|
6135
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5847
6136
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
5848
6137
|
if (needsInitialLeaderCheck) {
|
|
5849
6138
|
needsInitialLeaderCheck = false;
|
|
@@ -5946,6 +6235,32 @@ export class SharedLog<
|
|
|
5946
6235
|
current.peerSession === captured.peerSession
|
|
5947
6236
|
);
|
|
5948
6237
|
};
|
|
6238
|
+
try {
|
|
6239
|
+
await this._v2Send.confirmLatestForPeer(
|
|
6240
|
+
{
|
|
6241
|
+
peerHash: peer,
|
|
6242
|
+
peerSession: captured.peerSession,
|
|
6243
|
+
receiverTransportSession: captured.capabilitySession,
|
|
6244
|
+
},
|
|
6245
|
+
{
|
|
6246
|
+
timeout: getAttemptTimeout(),
|
|
6247
|
+
signal: roundSignal,
|
|
6248
|
+
},
|
|
6249
|
+
);
|
|
6250
|
+
} catch {
|
|
6251
|
+
// The exact receiver generation did not prove that it applied
|
|
6252
|
+
// our latest role state. Replan instead of transferring to a
|
|
6253
|
+
// peer that can still make a stale admission decision.
|
|
6254
|
+
if (transferAllOnRound && isPeerRoundCurrent()) {
|
|
6255
|
+
const state = ensureRepairState();
|
|
6256
|
+
for (const hash of hashes) state.hashes.add(hash);
|
|
6257
|
+
}
|
|
6258
|
+
return;
|
|
6259
|
+
}
|
|
6260
|
+
if (!isPeerRoundCurrent()) {
|
|
6261
|
+
purgePeerDeliveryState(peer);
|
|
6262
|
+
return;
|
|
6263
|
+
}
|
|
5949
6264
|
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
5950
6265
|
const transferHashes = transferAllOnRound
|
|
5951
6266
|
? hashes
|
|
@@ -6122,6 +6437,9 @@ export class SharedLog<
|
|
|
6122
6437
|
replicas,
|
|
6123
6438
|
ownershipLifecycleController,
|
|
6124
6439
|
);
|
|
6440
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
6441
|
+
onFreshLeaderPlan?.(validatedLeaders, ownershipRevision);
|
|
6442
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
6125
6443
|
for (let index = 0; index < entryArray.length; index++) {
|
|
6126
6444
|
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
6127
6445
|
if (signal.aborted) {
|
|
@@ -6205,6 +6523,75 @@ export class SharedLog<
|
|
|
6205
6523
|
}
|
|
6206
6524
|
}
|
|
6207
6525
|
|
|
6526
|
+
private async collectDeferredAppendBackfillExtras(
|
|
6527
|
+
entry: Entry<T>,
|
|
6528
|
+
replicas: number,
|
|
6529
|
+
baseLeaders: LeaderMap,
|
|
6530
|
+
nativeDeliveryPlan: AppendDeliveryPlan | undefined,
|
|
6531
|
+
ownershipLifecycleController: AbortController,
|
|
6532
|
+
): Promise<
|
|
6533
|
+
Omit<PersistedAppendBackfillSource<T, R>, "entry" | "coordinates">
|
|
6534
|
+
> {
|
|
6535
|
+
const ownershipRevision =
|
|
6536
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
6537
|
+
const ownershipWasStable =
|
|
6538
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
6539
|
+
const assignmentExtraLeaders: LeaderMap = new Map();
|
|
6540
|
+
const deliveryExtraTargets = new Set<string>();
|
|
6541
|
+
if (nativeDeliveryPlan) {
|
|
6542
|
+
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6543
|
+
// The sampled entry leaders are replaced by settlement's fresh plan.
|
|
6544
|
+
// Retain only native repair additions that were outside that base map.
|
|
6545
|
+
if (!baseLeaders.has(peer)) deliveryExtraTargets.add(peer);
|
|
6546
|
+
}
|
|
6547
|
+
} else {
|
|
6548
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
6549
|
+
const fullReplicaDeliveryCandidates =
|
|
6550
|
+
await this.getNativeFullReplicaDeliveryCandidates(replicas, selfHash);
|
|
6551
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6552
|
+
ownershipLifecycleController,
|
|
6553
|
+
);
|
|
6554
|
+
if (replicas >= Math.max(1, fullReplicaDeliveryCandidates.size)) {
|
|
6555
|
+
for (const peer of fullReplicaDeliveryCandidates) {
|
|
6556
|
+
if (!baseLeaders.has(peer)) {
|
|
6557
|
+
assignmentExtraLeaders.set(peer, { intersecting: true });
|
|
6558
|
+
}
|
|
6559
|
+
}
|
|
6560
|
+
}
|
|
6561
|
+
}
|
|
6562
|
+
|
|
6563
|
+
const referenceLeaders: LeaderMap = new Map();
|
|
6564
|
+
for await (const message of createExchangeHeadsMessages(this.log, [
|
|
6565
|
+
entry,
|
|
6566
|
+
])) {
|
|
6567
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6568
|
+
ownershipLifecycleController,
|
|
6569
|
+
);
|
|
6570
|
+
await this._mergeLeadersFromGidReferences(
|
|
6571
|
+
message,
|
|
6572
|
+
replicas,
|
|
6573
|
+
referenceLeaders,
|
|
6574
|
+
ownershipLifecycleController,
|
|
6575
|
+
{ freshLeaderPlan: true },
|
|
6576
|
+
);
|
|
6577
|
+
}
|
|
6578
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6579
|
+
ownershipLifecycleController,
|
|
6580
|
+
);
|
|
6581
|
+
for (const peer of referenceLeaders.keys()) {
|
|
6582
|
+
deliveryExtraTargets.add(peer);
|
|
6583
|
+
}
|
|
6584
|
+
return {
|
|
6585
|
+
assignmentExtraLeaders,
|
|
6586
|
+
deliveryExtraTargets,
|
|
6587
|
+
extrasOwnershipRevision:
|
|
6588
|
+
ownershipWasStable &&
|
|
6589
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
6590
|
+
? ownershipRevision
|
|
6591
|
+
: undefined,
|
|
6592
|
+
};
|
|
6593
|
+
}
|
|
6594
|
+
|
|
6208
6595
|
private async _appendDeliverToReplicators(
|
|
6209
6596
|
entry: Entry<T>,
|
|
6210
6597
|
coordinates: NumberFromType<R>[],
|
|
@@ -6299,7 +6686,11 @@ export class SharedLog<
|
|
|
6299
6686
|
if (!delivery) {
|
|
6300
6687
|
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6301
6688
|
throwIfInactive();
|
|
6302
|
-
this.queueAppendBackfill(
|
|
6689
|
+
this.queueAppendBackfill(
|
|
6690
|
+
peer,
|
|
6691
|
+
entryReplicatedForRepair,
|
|
6692
|
+
ownershipLifecycleController,
|
|
6693
|
+
);
|
|
6303
6694
|
}
|
|
6304
6695
|
if (nativeDeliveryPlan.defaultSendSilent) {
|
|
6305
6696
|
const rawTargets = this.canUseLiveRawGossip(
|
|
@@ -6372,7 +6763,11 @@ export class SharedLog<
|
|
|
6372
6763
|
}
|
|
6373
6764
|
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6374
6765
|
throwIfInactive();
|
|
6375
|
-
this.queueAppendBackfill(
|
|
6766
|
+
this.queueAppendBackfill(
|
|
6767
|
+
peer,
|
|
6768
|
+
entryReplicatedForRepair,
|
|
6769
|
+
ownershipLifecycleController,
|
|
6770
|
+
);
|
|
6376
6771
|
}
|
|
6377
6772
|
continue;
|
|
6378
6773
|
}
|
|
@@ -6427,7 +6822,11 @@ export class SharedLog<
|
|
|
6427
6822
|
// delivery acks, we still need a targeted backfill source of truth for the
|
|
6428
6823
|
// authoritative recipients or one entry can get stuck at 2/3 replicas
|
|
6429
6824
|
// forever. Best-effort fallback subscribers are not repair-worthy.
|
|
6430
|
-
this.queueAppendBackfill(
|
|
6825
|
+
this.queueAppendBackfill(
|
|
6826
|
+
peer,
|
|
6827
|
+
entryReplicatedForRepair,
|
|
6828
|
+
ownershipLifecycleController,
|
|
6829
|
+
);
|
|
6431
6830
|
}
|
|
6432
6831
|
if (isLeader) {
|
|
6433
6832
|
const rawTargets = this.canUseLiveRawGossip(set, selfHash);
|
|
@@ -6536,7 +6935,11 @@ export class SharedLog<
|
|
|
6536
6935
|
// Direct append delivery is intentionally optimistic. Queue one delayed,
|
|
6537
6936
|
// batched maybe-sync pass for the intended recipients so stable 3-peer
|
|
6538
6937
|
// append workloads do not depend on perfect first-try delivery ordering.
|
|
6539
|
-
this.queueAppendBackfill(
|
|
6938
|
+
this.queueAppendBackfill(
|
|
6939
|
+
peer,
|
|
6940
|
+
entryReplicatedForRepair,
|
|
6941
|
+
ownershipLifecycleController,
|
|
6942
|
+
);
|
|
6540
6943
|
}
|
|
6541
6944
|
}
|
|
6542
6945
|
|
|
@@ -6551,6 +6954,7 @@ export class SharedLog<
|
|
|
6551
6954
|
minReplicasValue: number,
|
|
6552
6955
|
leaders: LeaderMap,
|
|
6553
6956
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
6957
|
+
options?: { freshLeaderPlan?: boolean },
|
|
6554
6958
|
) {
|
|
6555
6959
|
const throwIfInactive = () =>
|
|
6556
6960
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -6576,13 +6980,13 @@ export class SharedLog<
|
|
|
6576
6980
|
found = await this.findLeadersFromEntry(
|
|
6577
6981
|
gidEntry,
|
|
6578
6982
|
minReplicasValue,
|
|
6579
|
-
undefined,
|
|
6983
|
+
options?.freshLeaderPlan ? { freshLeaderPlan: true } : undefined,
|
|
6580
6984
|
ownershipLifecycleController,
|
|
6581
6985
|
);
|
|
6582
6986
|
} else {
|
|
6583
6987
|
found = await this._findLeaders(
|
|
6584
6988
|
coordinates,
|
|
6585
|
-
undefined,
|
|
6989
|
+
options?.freshLeaderPlan ? { freshLeaderPlan: true } : undefined,
|
|
6586
6990
|
ownershipLifecycleController,
|
|
6587
6991
|
);
|
|
6588
6992
|
}
|
|
@@ -6891,8 +7295,13 @@ export class SharedLog<
|
|
|
6891
7295
|
private canCacheLeaderSelectionContext(options?: {
|
|
6892
7296
|
roleAge?: number;
|
|
6893
7297
|
candidates?: Iterable<string>;
|
|
7298
|
+
freshLeaderPlan?: boolean;
|
|
6894
7299
|
}) {
|
|
6895
|
-
return
|
|
7300
|
+
return (
|
|
7301
|
+
options?.roleAge == null &&
|
|
7302
|
+
options?.candidates == null &&
|
|
7303
|
+
options?.freshLeaderPlan !== true
|
|
7304
|
+
);
|
|
6896
7305
|
}
|
|
6897
7306
|
|
|
6898
7307
|
private cloneLeaderSelectionContext(
|
|
@@ -6910,6 +7319,7 @@ export class SharedLog<
|
|
|
6910
7319
|
private getCachedLeaderSelectionContext(options?: {
|
|
6911
7320
|
roleAge?: number;
|
|
6912
7321
|
candidates?: Iterable<string>;
|
|
7322
|
+
freshLeaderPlan?: boolean;
|
|
6913
7323
|
}): LeaderSelectionContext | undefined {
|
|
6914
7324
|
if (!this.canCacheLeaderSelectionContext(options)) {
|
|
6915
7325
|
return;
|
|
@@ -6926,6 +7336,7 @@ export class SharedLog<
|
|
|
6926
7336
|
| {
|
|
6927
7337
|
roleAge?: number;
|
|
6928
7338
|
candidates?: Iterable<string>;
|
|
7339
|
+
freshLeaderPlan?: boolean;
|
|
6929
7340
|
}
|
|
6930
7341
|
| undefined,
|
|
6931
7342
|
context: LeaderSelectionContext,
|
|
@@ -8035,8 +8446,11 @@ export class SharedLog<
|
|
|
8035
8446
|
? checkedPruneCoordinator.fencePeerRemoval(keyHash)
|
|
8036
8447
|
: undefined;
|
|
8037
8448
|
const blockPeerReceiveAdmission = () => {
|
|
8038
|
-
releaseReceiveCleanupGate
|
|
8039
|
-
|
|
8449
|
+
if (!releaseReceiveCleanupGate) {
|
|
8450
|
+
releaseReceiveCleanupGate =
|
|
8451
|
+
this._peerSessions.acquireReceiveCleanupGate(keyHash);
|
|
8452
|
+
this.dispatchPersistedReceiptReadinessChange(keyHash);
|
|
8453
|
+
}
|
|
8040
8454
|
};
|
|
8041
8455
|
if (!isMe && !isSpeculativePeerRemoval) {
|
|
8042
8456
|
// Revoke this peer's receipts synchronously, before this removal can
|
|
@@ -8277,7 +8691,10 @@ export class SharedLog<
|
|
|
8277
8691
|
});
|
|
8278
8692
|
removalCallCompleted = true;
|
|
8279
8693
|
} finally {
|
|
8280
|
-
releaseReceiveCleanupGate
|
|
8694
|
+
if (releaseReceiveCleanupGate) {
|
|
8695
|
+
releaseReceiveCleanupGate();
|
|
8696
|
+
this.dispatchPersistedReceiptReadinessChange(keyHash);
|
|
8697
|
+
}
|
|
8281
8698
|
if (
|
|
8282
8699
|
replicationInfoRecoveryEpochAdvanced &&
|
|
8283
8700
|
ownsReplicationOwnershipLifecycle() &&
|
|
@@ -10191,10 +10608,7 @@ export class SharedLog<
|
|
|
10191
10608
|
});
|
|
10192
10609
|
}
|
|
10193
10610
|
|
|
10194
|
-
private flushAppendBackfill(
|
|
10195
|
-
repairLifecycleController: AbortController = this._instanceLifecycle
|
|
10196
|
-
?.ownershipLifecycleController as AbortController,
|
|
10197
|
-
) {
|
|
10611
|
+
private flushAppendBackfill(repairLifecycleController: AbortController) {
|
|
10198
10612
|
if (
|
|
10199
10613
|
!this.isRepairLifecycleActive(repairLifecycleController) ||
|
|
10200
10614
|
this._appendBackfillPendingByTarget.size === 0
|
|
@@ -10215,9 +10629,11 @@ export class SharedLog<
|
|
|
10215
10629
|
}
|
|
10216
10630
|
}
|
|
10217
10631
|
|
|
10218
|
-
private queueAppendBackfill(
|
|
10219
|
-
|
|
10220
|
-
|
|
10632
|
+
private queueAppendBackfill(
|
|
10633
|
+
target: string,
|
|
10634
|
+
entry: EntryReplicated<R>,
|
|
10635
|
+
repairLifecycleController: AbortController,
|
|
10636
|
+
) {
|
|
10221
10637
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10222
10638
|
return;
|
|
10223
10639
|
}
|
|
@@ -10249,6 +10665,62 @@ export class SharedLog<
|
|
|
10249
10665
|
this._appendBackfillTimer = timer;
|
|
10250
10666
|
}
|
|
10251
10667
|
|
|
10668
|
+
private queuePersistedAppendBackfill(
|
|
10669
|
+
source: PersistedAppendBackfillSource<T, R>,
|
|
10670
|
+
leaders: LeaderMap,
|
|
10671
|
+
replicas: number,
|
|
10672
|
+
repairLifecycleController: AbortController,
|
|
10673
|
+
ownershipRevision: number,
|
|
10674
|
+
): void {
|
|
10675
|
+
try {
|
|
10676
|
+
if (
|
|
10677
|
+
!this.isRepairLifecycleActive(repairLifecycleController) ||
|
|
10678
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
10679
|
+
) {
|
|
10680
|
+
return;
|
|
10681
|
+
}
|
|
10682
|
+
const assignmentLeaders = new Map(leaders);
|
|
10683
|
+
const deliveryTargets = new Set(leaders.keys());
|
|
10684
|
+
if (source.extrasOwnershipRevision === ownershipRevision) {
|
|
10685
|
+
for (const [peer, sample] of source.assignmentExtraLeaders) {
|
|
10686
|
+
assignmentLeaders.set(peer, sample);
|
|
10687
|
+
}
|
|
10688
|
+
for (const peer of source.deliveryExtraTargets) {
|
|
10689
|
+
deliveryTargets.add(peer);
|
|
10690
|
+
}
|
|
10691
|
+
}
|
|
10692
|
+
const repairEntry = this.createEntryReplicatedForRepair({
|
|
10693
|
+
entry: source.entry,
|
|
10694
|
+
coordinates: source.coordinates,
|
|
10695
|
+
leaders: assignmentLeaders,
|
|
10696
|
+
replicas,
|
|
10697
|
+
});
|
|
10698
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
10699
|
+
for (const peer of assignmentLeaders.keys()) {
|
|
10700
|
+
deliveryTargets.add(peer);
|
|
10701
|
+
}
|
|
10702
|
+
for (const peer of deliveryTargets) {
|
|
10703
|
+
if (peer === selfHash) continue;
|
|
10704
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) return;
|
|
10705
|
+
try {
|
|
10706
|
+
this.queueAppendBackfill(
|
|
10707
|
+
peer,
|
|
10708
|
+
repairEntry,
|
|
10709
|
+
repairLifecycleController,
|
|
10710
|
+
);
|
|
10711
|
+
} catch (error) {
|
|
10712
|
+
if (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10713
|
+
logger.error(error);
|
|
10714
|
+
}
|
|
10715
|
+
}
|
|
10716
|
+
}
|
|
10717
|
+
} catch (error) {
|
|
10718
|
+
if (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10719
|
+
logger.error(error);
|
|
10720
|
+
}
|
|
10721
|
+
}
|
|
10722
|
+
}
|
|
10723
|
+
|
|
10252
10724
|
private dispatchMaybeMissingEntries(
|
|
10253
10725
|
target: string,
|
|
10254
10726
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
@@ -11894,6 +12366,12 @@ export class SharedLog<
|
|
|
11894
12366
|
let persistedPlanningRecord:
|
|
11895
12367
|
| PersistedDeliveryPlanningRecord<T, R>
|
|
11896
12368
|
| undefined;
|
|
12369
|
+
let persistedBackfillSource:
|
|
12370
|
+
| PersistedAppendBackfillSource<T, R>
|
|
12371
|
+
| undefined;
|
|
12372
|
+
let persistedBackfillLeaders: LeaderMap | undefined;
|
|
12373
|
+
let persistedBackfillOwnershipRevision: number | undefined;
|
|
12374
|
+
let localAppendProcessed = false;
|
|
11897
12375
|
if (persistedDelivery) {
|
|
11898
12376
|
(appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
|
|
11899
12377
|
hashes,
|
|
@@ -11947,8 +12425,17 @@ export class SharedLog<
|
|
|
11947
12425
|
await this.processLocalAppend(processingEntry, result.removed, options, {
|
|
11948
12426
|
minReplicasValue,
|
|
11949
12427
|
appendFacts: persistedAppendCommit,
|
|
12428
|
+
// Persisted settlement must confirm each exact receiver generation before
|
|
12429
|
+
// using its transfer as receipt evidence. Keep the optimistic append path
|
|
12430
|
+
// out of that ordering decision.
|
|
12431
|
+
captureDeferredBackfillSource: persistedDelivery
|
|
12432
|
+
? (source) => {
|
|
12433
|
+
persistedBackfillSource = source;
|
|
12434
|
+
}
|
|
12435
|
+
: undefined,
|
|
11950
12436
|
ownershipLifecycleController,
|
|
11951
12437
|
});
|
|
12438
|
+
localAppendProcessed = true;
|
|
11952
12439
|
throwIfDeliveryAborted();
|
|
11953
12440
|
if (persistedDelivery && persistedDeadline) {
|
|
11954
12441
|
await this.settlePersistedDelivery(
|
|
@@ -11957,6 +12444,12 @@ export class SharedLog<
|
|
|
11957
12444
|
persistedDelivery,
|
|
11958
12445
|
ownershipLifecycleController,
|
|
11959
12446
|
persistedDeadline,
|
|
12447
|
+
true,
|
|
12448
|
+
(leadersByEntry, ownershipRevision) => {
|
|
12449
|
+
const leaders = leadersByEntry[0];
|
|
12450
|
+
persistedBackfillLeaders = leaders ? new Map(leaders) : undefined;
|
|
12451
|
+
persistedBackfillOwnershipRevision = ownershipRevision;
|
|
12452
|
+
},
|
|
11960
12453
|
);
|
|
11961
12454
|
}
|
|
11962
12455
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -11969,6 +12462,27 @@ export class SharedLog<
|
|
|
11969
12462
|
}
|
|
11970
12463
|
throw error;
|
|
11971
12464
|
} finally {
|
|
12465
|
+
if (
|
|
12466
|
+
persistedBackfillSource &&
|
|
12467
|
+
persistedBackfillLeaders &&
|
|
12468
|
+
persistedBackfillOwnershipRevision !== undefined &&
|
|
12469
|
+
localAppendProcessed
|
|
12470
|
+
) {
|
|
12471
|
+
// A persisted quorum changes the return condition, not the configured
|
|
12472
|
+
// replication degree. Reuse settlement's latest fresh leader plan rather
|
|
12473
|
+
// than carrying an optimistic target or extending the receipt deadline
|
|
12474
|
+
// with another plan. Fence best-effort repair to this append's ownership
|
|
12475
|
+
// generation, and never let it mask the primary result.
|
|
12476
|
+
try {
|
|
12477
|
+
this.queuePersistedAppendBackfill(
|
|
12478
|
+
persistedBackfillSource,
|
|
12479
|
+
persistedBackfillLeaders,
|
|
12480
|
+
minReplicasValue,
|
|
12481
|
+
ownershipLifecycleController,
|
|
12482
|
+
persistedBackfillOwnershipRevision,
|
|
12483
|
+
);
|
|
12484
|
+
} catch {}
|
|
12485
|
+
}
|
|
11972
12486
|
persistedDeadline?.dispose();
|
|
11973
12487
|
}
|
|
11974
12488
|
}
|
|
@@ -16080,6 +16594,9 @@ export class SharedLog<
|
|
|
16080
16594
|
minReplicasValue: number;
|
|
16081
16595
|
appendFacts?: PreparedAppendFacts;
|
|
16082
16596
|
deferHeadCoordinatePersistence?: boolean;
|
|
16597
|
+
captureDeferredBackfillSource?: (
|
|
16598
|
+
source: PersistedAppendBackfillSource<T, R>,
|
|
16599
|
+
) => void;
|
|
16083
16600
|
nativeAppendPlan?: NativeAppendEntryPlan<R>;
|
|
16084
16601
|
extraCoordinateDeleteHashes?: string[];
|
|
16085
16602
|
ownershipLifecycleController?: AbortController;
|
|
@@ -16223,7 +16740,41 @@ export class SharedLog<
|
|
|
16223
16740
|
ownershipLifecycleController,
|
|
16224
16741
|
);
|
|
16225
16742
|
|
|
16226
|
-
if (
|
|
16743
|
+
if (properties.captureDeferredBackfillSource) {
|
|
16744
|
+
let extras: Omit<
|
|
16745
|
+
PersistedAppendBackfillSource<T, R>,
|
|
16746
|
+
"entry" | "coordinates"
|
|
16747
|
+
> = {
|
|
16748
|
+
assignmentExtraLeaders: new Map(),
|
|
16749
|
+
deliveryExtraTargets: new Set(),
|
|
16750
|
+
};
|
|
16751
|
+
try {
|
|
16752
|
+
extras = await this.collectDeferredAppendBackfillExtras(
|
|
16753
|
+
entry,
|
|
16754
|
+
properties.minReplicasValue,
|
|
16755
|
+
leaders!,
|
|
16756
|
+
nativeDeliveryPlan,
|
|
16757
|
+
ownershipLifecycleController,
|
|
16758
|
+
);
|
|
16759
|
+
} catch (error) {
|
|
16760
|
+
if (this.isRepairLifecycleActive(ownershipLifecycleController)) {
|
|
16761
|
+
logger.error(error);
|
|
16762
|
+
}
|
|
16763
|
+
}
|
|
16764
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
16765
|
+
ownershipLifecycleController,
|
|
16766
|
+
);
|
|
16767
|
+
properties.captureDeferredBackfillSource({
|
|
16768
|
+
entry,
|
|
16769
|
+
coordinates: [...coordinates],
|
|
16770
|
+
...extras,
|
|
16771
|
+
});
|
|
16772
|
+
}
|
|
16773
|
+
|
|
16774
|
+
if (
|
|
16775
|
+
options?.target !== "none" &&
|
|
16776
|
+
!properties.captureDeferredBackfillSource
|
|
16777
|
+
) {
|
|
16227
16778
|
const hasDelivery = !(deliveryArg === undefined || deliveryArg === false);
|
|
16228
16779
|
|
|
16229
16780
|
if (target === "all" && hasDelivery) {
|
|
@@ -16624,6 +17175,12 @@ export class SharedLog<
|
|
|
16624
17175
|
this._peerSyncCapabilities = new Map();
|
|
16625
17176
|
this._peerSyncCapabilitySessions = new Map();
|
|
16626
17177
|
this._peerSyncCapabilityTimestamps = new Map();
|
|
17178
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
17179
|
+
this._persistedReceiptReadinessGenerationPrefix = toHexString(
|
|
17180
|
+
randomBytes(8),
|
|
17181
|
+
);
|
|
17182
|
+
this._persistedReceiptReadinessGenerationCounter = 0;
|
|
17183
|
+
this._persistedReceiptReadinessWaiters = new Set();
|
|
16627
17184
|
this._persistedReceiptStorage = undefined;
|
|
16628
17185
|
this._persistedReceiptRequestsInFlight = new Map();
|
|
16629
17186
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -18300,6 +18857,7 @@ export class SharedLog<
|
|
|
18300
18857
|
ownershipLifecycleController,
|
|
18301
18858
|
this._checkedPrune,
|
|
18302
18859
|
);
|
|
18860
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
18303
18861
|
}
|
|
18304
18862
|
|
|
18305
18863
|
private cleanupPendingIHavePeer(peerHash: string) {
|
|
@@ -18324,6 +18882,7 @@ export class SharedLog<
|
|
|
18324
18882
|
receiveEpoch,
|
|
18325
18883
|
});
|
|
18326
18884
|
}
|
|
18885
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
18327
18886
|
}
|
|
18328
18887
|
|
|
18329
18888
|
private async resolveCandidatePeersForHash(
|
|
@@ -19765,6 +20324,7 @@ export class SharedLog<
|
|
|
19765
20324
|
this._peerSyncCapabilities?.clear();
|
|
19766
20325
|
this._peerSyncCapabilitySessions?.clear();
|
|
19767
20326
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
20327
|
+
this._persistedReceiptReadinessGenerations = new WeakMap();
|
|
19768
20328
|
this._persistedReceiptStorage = undefined;
|
|
19769
20329
|
this._persistedReceiptRequestsInFlight?.clear();
|
|
19770
20330
|
this._persistedReceiptRequestsInFlightTotal = 0;
|
|
@@ -22321,10 +22881,14 @@ export class SharedLog<
|
|
|
22321
22881
|
}
|
|
22322
22882
|
return;
|
|
22323
22883
|
} else if (msg instanceof ReplicationInfoV2AppliedMessage) {
|
|
22324
|
-
|
|
22325
|
-
|
|
22326
|
-
|
|
22327
|
-
|
|
22884
|
+
if (
|
|
22885
|
+
this._v2Send.acceptApplied(msg, {
|
|
22886
|
+
from: context.from,
|
|
22887
|
+
receiverTransportSession: context.message.header.session,
|
|
22888
|
+
})
|
|
22889
|
+
) {
|
|
22890
|
+
this.dispatchPersistedReceiptReadinessChange(receiveFromHash);
|
|
22891
|
+
}
|
|
22328
22892
|
return;
|
|
22329
22893
|
} else if (isReplicationInfoV2Message(msg)) {
|
|
22330
22894
|
await this.handleReplicationInfoV2Announcement(
|
|
@@ -22498,6 +23062,12 @@ export class SharedLog<
|
|
|
22498
23062
|
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
22499
23063
|
context.message.header.session &&
|
|
22500
23064
|
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
23065
|
+
this._v2Receive.isCurrentActive({
|
|
23066
|
+
peerHash: lane.fromHash,
|
|
23067
|
+
peerSession: session,
|
|
23068
|
+
receiveEpoch: lane.receiveEpoch,
|
|
23069
|
+
senderTransportSession: context.message.header.session,
|
|
23070
|
+
}) &&
|
|
22501
23071
|
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
22502
23072
|
);
|
|
22503
23073
|
}
|
|
@@ -23134,6 +23704,7 @@ export class SharedLog<
|
|
|
23134
23704
|
// A committed V2 announcement is applied progress: the peer answers,
|
|
23135
23705
|
// so recovery re-solicitation may restart from the base interval.
|
|
23136
23706
|
this.resetReplicationInfoV2RecoveryEscalation(fromHash);
|
|
23707
|
+
this.dispatchPersistedReceiptReadinessChange(fromHash);
|
|
23137
23708
|
});
|
|
23138
23709
|
} finally {
|
|
23139
23710
|
this._v2Receive.release(admission);
|
|
@@ -23488,6 +24059,465 @@ export class SharedLog<
|
|
|
23488
24059
|
throwIfInactive();
|
|
23489
24060
|
}
|
|
23490
24061
|
|
|
24062
|
+
private nudgePersistedReceiptPeerReadiness(publicKey: PublicSignKey): void {
|
|
24063
|
+
if (this.closed) return;
|
|
24064
|
+
const peerHash = publicKey.hashcode();
|
|
24065
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
24066
|
+
if (
|
|
24067
|
+
!peerSession ||
|
|
24068
|
+
peerSession.phase === "departing" ||
|
|
24069
|
+
(peerSession.phase === "opening" &&
|
|
24070
|
+
!peerSession.openingBarrierActive)
|
|
24071
|
+
) {
|
|
24072
|
+
// A barrier rejection deliberately leaves the current session in its
|
|
24073
|
+
// fail-closed opening phase after the barrier window has settled. Ask the
|
|
24074
|
+
// authenticated peer for a fresh subscriber snapshot so the replacement
|
|
24075
|
+
// session can recover; never rotate a barrier that is still in flight.
|
|
24076
|
+
this.requestSubscriberSnapshotForCapability(publicKey);
|
|
24077
|
+
return;
|
|
24078
|
+
}
|
|
24079
|
+
if (peerSession.phase !== "open" || !peerSession.isActive()) {
|
|
24080
|
+
return;
|
|
24081
|
+
}
|
|
24082
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
24083
|
+
this.promoteReplicationInfoV2ReceiveCapability(publicKey, peerSession);
|
|
24084
|
+
this._v2Receive.reAdvertiseLocalCapabilityForRecovery({
|
|
24085
|
+
peerHash,
|
|
24086
|
+
peerSession,
|
|
24087
|
+
receiveEpoch,
|
|
24088
|
+
});
|
|
24089
|
+
this._v2Receive.ensureRequestProgress({
|
|
24090
|
+
peerHash,
|
|
24091
|
+
peerSession,
|
|
24092
|
+
receiveEpoch,
|
|
24093
|
+
});
|
|
24094
|
+
this.scheduleReplicationInfoV2Recovery(publicKey);
|
|
24095
|
+
}
|
|
24096
|
+
|
|
24097
|
+
/**
|
|
24098
|
+
* Inspect whether one public key's exact current connection generation can
|
|
24099
|
+
* supply persisted-receipt evidence. The returned object is frozen and never
|
|
24100
|
+
* exposes the internal PeerSession token. When `entries` are supplied, the
|
|
24101
|
+
* peer must also be present in a fresh leader plan for every entry.
|
|
24102
|
+
*
|
|
24103
|
+
* This is advisory preflight state. Persisted delivery repeats every
|
|
24104
|
+
* generation, leadership, ownership and storage check at receipt time; a
|
|
24105
|
+
* `ready` snapshot is never itself authority to dispose a source copy.
|
|
24106
|
+
*/
|
|
24107
|
+
async getPersistedReceiptPeerReadiness(
|
|
24108
|
+
key: PublicSignKey,
|
|
24109
|
+
options: PersistedReceiptPeerReadinessOptions<T, R> = {},
|
|
24110
|
+
): Promise<PersistedReceiptPeerReadiness> {
|
|
24111
|
+
return this.inspectPersistedReceiptPeerReadiness(key, options);
|
|
24112
|
+
}
|
|
24113
|
+
|
|
24114
|
+
private async inspectPersistedReceiptPeerReadiness(
|
|
24115
|
+
key: PublicSignKey,
|
|
24116
|
+
options: PersistedReceiptPeerReadinessOptions<T, R>,
|
|
24117
|
+
assertContinue?: () => void,
|
|
24118
|
+
): Promise<PersistedReceiptPeerReadiness> {
|
|
24119
|
+
// Capture and validate caller-owned planning input before consulting live
|
|
24120
|
+
// peer state. Invalid options must not appear to work merely because the
|
|
24121
|
+
// peer is currently absent, then fail later when the same session connects.
|
|
24122
|
+
const entries = options.entries ? [...options.entries] : [];
|
|
24123
|
+
const replicas =
|
|
24124
|
+
options.replicas ??
|
|
24125
|
+
(entries.length > 0 ? this.replicas.min.getValue(this) : undefined);
|
|
24126
|
+
if (replicas !== undefined) {
|
|
24127
|
+
if (!Number.isSafeInteger(replicas) || replicas <= 0) {
|
|
24128
|
+
throw new RangeError(
|
|
24129
|
+
"Persisted-receipt readiness replicas must be a positive integer",
|
|
24130
|
+
);
|
|
24131
|
+
}
|
|
24132
|
+
checkMinReplicasLimit(replicas);
|
|
24133
|
+
}
|
|
24134
|
+
|
|
24135
|
+
const peerHash = key.hashcode();
|
|
24136
|
+
const captured = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24137
|
+
if ("status" in captured) {
|
|
24138
|
+
return captured;
|
|
24139
|
+
}
|
|
24140
|
+
assertContinue?.();
|
|
24141
|
+
|
|
24142
|
+
if (entries.length > 0) {
|
|
24143
|
+
const ownershipLifecycleController =
|
|
24144
|
+
this.captureReplicationOwnershipLifecycle();
|
|
24145
|
+
const ownershipRevision =
|
|
24146
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
24147
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
24148
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24149
|
+
"ownership-changing",
|
|
24150
|
+
captured.generation,
|
|
24151
|
+
);
|
|
24152
|
+
}
|
|
24153
|
+
for (const entry of entries) {
|
|
24154
|
+
assertContinue?.();
|
|
24155
|
+
const leaders = await this.findLeadersFromEntry(
|
|
24156
|
+
entry,
|
|
24157
|
+
replicas!,
|
|
24158
|
+
{ freshLeaderPlan: true },
|
|
24159
|
+
ownershipLifecycleController,
|
|
24160
|
+
);
|
|
24161
|
+
assertContinue?.();
|
|
24162
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
|
|
24163
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24164
|
+
"ownership-changing",
|
|
24165
|
+
captured.generation,
|
|
24166
|
+
);
|
|
24167
|
+
}
|
|
24168
|
+
const current = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24169
|
+
if ("status" in current) {
|
|
24170
|
+
return current;
|
|
24171
|
+
}
|
|
24172
|
+
if (
|
|
24173
|
+
current.peerSession !== captured.peerSession ||
|
|
24174
|
+
current.receiveEpoch !== captured.receiveEpoch ||
|
|
24175
|
+
current.capabilitySession !== captured.capabilitySession
|
|
24176
|
+
) {
|
|
24177
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24178
|
+
"replication-state-pending",
|
|
24179
|
+
current.generation,
|
|
24180
|
+
);
|
|
24181
|
+
}
|
|
24182
|
+
if (!leaders.has(peerHash)) {
|
|
24183
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24184
|
+
"not-entry-leader",
|
|
24185
|
+
captured.generation,
|
|
24186
|
+
);
|
|
24187
|
+
}
|
|
24188
|
+
}
|
|
24189
|
+
}
|
|
24190
|
+
|
|
24191
|
+
assertContinue?.();
|
|
24192
|
+
const current = this.persistedReceiptReadinessCandidate(peerHash);
|
|
24193
|
+
if ("status" in current) {
|
|
24194
|
+
return current;
|
|
24195
|
+
}
|
|
24196
|
+
if (
|
|
24197
|
+
current.peerSession !== captured.peerSession ||
|
|
24198
|
+
current.receiveEpoch !== captured.receiveEpoch ||
|
|
24199
|
+
current.capabilitySession !== captured.capabilitySession
|
|
24200
|
+
) {
|
|
24201
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24202
|
+
"replication-state-pending",
|
|
24203
|
+
current.generation,
|
|
24204
|
+
);
|
|
24205
|
+
}
|
|
24206
|
+
if (
|
|
24207
|
+
!this._v2Send.isLatestConfirmedForPeer({
|
|
24208
|
+
peerHash,
|
|
24209
|
+
peerSession: captured.peerSession,
|
|
24210
|
+
receiverTransportSession: captured.capabilitySession,
|
|
24211
|
+
})
|
|
24212
|
+
) {
|
|
24213
|
+
return this.pendingPersistedReceiptReadiness(
|
|
24214
|
+
"replication-confirmation-pending",
|
|
24215
|
+
captured.generation,
|
|
24216
|
+
);
|
|
24217
|
+
}
|
|
24218
|
+
return Object.freeze({
|
|
24219
|
+
status: "ready" as const,
|
|
24220
|
+
generation: captured.generation,
|
|
24221
|
+
});
|
|
24222
|
+
}
|
|
24223
|
+
|
|
24224
|
+
/**
|
|
24225
|
+
* Wait for a public key's current (or replacement) connection generation to
|
|
24226
|
+
* become persisted-receipt ready. Transition listeners are installed before
|
|
24227
|
+
* the first asynchronous inspection, and a bounded recovery tick repairs
|
|
24228
|
+
* missed subscriber/capability wakes without retaining stale PeerSessions.
|
|
24229
|
+
* This waiter is advisory only; the following persisted delivery remains the
|
|
24230
|
+
* operation that proves the requested remote durability quorum.
|
|
24231
|
+
*/
|
|
24232
|
+
async waitForPersistedReceiptPeerReadiness(
|
|
24233
|
+
key: PublicSignKey,
|
|
24234
|
+
options: WaitForPersistedReceiptPeerReadinessOptions<T, R> = {},
|
|
24235
|
+
): Promise<PersistedReceiptPeerReady> {
|
|
24236
|
+
if (this.closed) {
|
|
24237
|
+
throw new ClosedError();
|
|
24238
|
+
}
|
|
24239
|
+
const timeoutMs = options.timeout ?? this.waitForReplicatorTimeout;
|
|
24240
|
+
if (
|
|
24241
|
+
!Number.isSafeInteger(timeoutMs) ||
|
|
24242
|
+
timeoutMs <= 0 ||
|
|
24243
|
+
timeoutMs > MAX_PERSISTED_DELIVERY_TIMEOUT_MS
|
|
24244
|
+
) {
|
|
24245
|
+
throw new RangeError(
|
|
24246
|
+
`Persisted-receipt readiness timeout must be an integer from 1 to ${MAX_PERSISTED_DELIVERY_TIMEOUT_MS} milliseconds`,
|
|
24247
|
+
);
|
|
24248
|
+
}
|
|
24249
|
+
if (options.signal?.aborted) {
|
|
24250
|
+
throw options.signal.reason instanceof Error
|
|
24251
|
+
? options.signal.reason
|
|
24252
|
+
: new AbortError("Persisted-receipt readiness wait aborted");
|
|
24253
|
+
}
|
|
24254
|
+
|
|
24255
|
+
// Capture caller-owned inputs before reserving a waiter slot. A throwing
|
|
24256
|
+
// iterator/key implementation must not strand capacity permanently.
|
|
24257
|
+
const entries = options.entries ? [...options.entries] : undefined;
|
|
24258
|
+
const inspectOptions: PersistedReceiptPeerReadinessOptions<T, R> = {
|
|
24259
|
+
...(entries ? { entries } : {}),
|
|
24260
|
+
...(options.replicas === undefined ? {} : { replicas: options.replicas }),
|
|
24261
|
+
};
|
|
24262
|
+
const peerHash = key.hashcode();
|
|
24263
|
+
const waiterSet = this._persistedReceiptReadinessWaiters;
|
|
24264
|
+
if (waiterSet.size >= MAX_PERSISTED_RECEIPT_READINESS_WAITERS) {
|
|
24265
|
+
throw new RangeError(
|
|
24266
|
+
`Too many pending persisted-receipt readiness waits (maximum ${MAX_PERSISTED_RECEIPT_READINESS_WAITERS})`,
|
|
24267
|
+
);
|
|
24268
|
+
}
|
|
24269
|
+
const waiterToken = {};
|
|
24270
|
+
waiterSet.add(waiterToken);
|
|
24271
|
+
const deadline = Date.now() + timeoutMs;
|
|
24272
|
+
const closeSignal = this._closeController.signal;
|
|
24273
|
+
const operationController = new AbortController();
|
|
24274
|
+
const operationSignal = AbortSignal.any(
|
|
24275
|
+
[options.signal, closeSignal, operationController.signal].filter(
|
|
24276
|
+
(value): value is AbortSignal => value !== undefined,
|
|
24277
|
+
),
|
|
24278
|
+
);
|
|
24279
|
+
const deferred = pDefer<PersistedReceiptPeerReady>();
|
|
24280
|
+
let settled = false;
|
|
24281
|
+
let checkScheduled = false;
|
|
24282
|
+
let checkInFlight = false;
|
|
24283
|
+
let rerun = false;
|
|
24284
|
+
let recoveryTimer: ReturnType<typeof setTimeout> | undefined;
|
|
24285
|
+
let confirmationController: AbortController | undefined;
|
|
24286
|
+
let lastSnapshot: PersistedReceiptPeerReadiness | undefined;
|
|
24287
|
+
const createTimeoutError = () => {
|
|
24288
|
+
const suffix = lastSnapshot
|
|
24289
|
+
? ` (last status: ${lastSnapshot.status}${
|
|
24290
|
+
"reason" in lastSnapshot ? `/${lastSnapshot.reason}` : ""
|
|
24291
|
+
})`
|
|
24292
|
+
: "";
|
|
24293
|
+
return new TimeoutError(
|
|
24294
|
+
`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}`,
|
|
24295
|
+
);
|
|
24296
|
+
};
|
|
24297
|
+
|
|
24298
|
+
const cleanup = () => {
|
|
24299
|
+
waiterSet.delete(waiterToken);
|
|
24300
|
+
this.events.removeEventListener(
|
|
24301
|
+
"persisted-receipt:readiness",
|
|
24302
|
+
onReadinessChange,
|
|
24303
|
+
);
|
|
24304
|
+
this.events.removeEventListener("replication:change", onRoleChange);
|
|
24305
|
+
this.events.removeEventListener("replicator:mature", onRoleChange);
|
|
24306
|
+
options.signal?.removeEventListener("abort", onCallerAbort);
|
|
24307
|
+
closeSignal.removeEventListener("abort", onClose);
|
|
24308
|
+
if (recoveryTimer) {
|
|
24309
|
+
clearTimeout(recoveryTimer);
|
|
24310
|
+
recoveryTimer = undefined;
|
|
24311
|
+
}
|
|
24312
|
+
confirmationController?.abort(
|
|
24313
|
+
new AbortError("Persisted-receipt readiness generation changed"),
|
|
24314
|
+
);
|
|
24315
|
+
confirmationController = undefined;
|
|
24316
|
+
operationController.abort(
|
|
24317
|
+
new AbortError("Persisted-receipt readiness wait settled"),
|
|
24318
|
+
);
|
|
24319
|
+
};
|
|
24320
|
+
const resolve = (snapshot: PersistedReceiptPeerReady) => {
|
|
24321
|
+
if (settled) return;
|
|
24322
|
+
settled = true;
|
|
24323
|
+
cleanup();
|
|
24324
|
+
deferred.resolve(snapshot);
|
|
24325
|
+
};
|
|
24326
|
+
const reject = (error: unknown) => {
|
|
24327
|
+
if (settled) return;
|
|
24328
|
+
settled = true;
|
|
24329
|
+
cleanup();
|
|
24330
|
+
deferred.reject(
|
|
24331
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
24332
|
+
);
|
|
24333
|
+
};
|
|
24334
|
+
const onCallerAbort = () =>
|
|
24335
|
+
reject(
|
|
24336
|
+
options.signal?.reason instanceof Error
|
|
24337
|
+
? options.signal.reason
|
|
24338
|
+
: new AbortError("Persisted-receipt readiness wait aborted"),
|
|
24339
|
+
);
|
|
24340
|
+
const onClose = () => reject(new ClosedError());
|
|
24341
|
+
const continueWait = () => {
|
|
24342
|
+
if (settled) return false;
|
|
24343
|
+
if (closeSignal.aborted) {
|
|
24344
|
+
onClose();
|
|
24345
|
+
return false;
|
|
24346
|
+
}
|
|
24347
|
+
if (options.signal?.aborted) {
|
|
24348
|
+
onCallerAbort();
|
|
24349
|
+
return false;
|
|
24350
|
+
}
|
|
24351
|
+
if (Date.now() >= deadline) {
|
|
24352
|
+
reject(createTimeoutError());
|
|
24353
|
+
return false;
|
|
24354
|
+
}
|
|
24355
|
+
return true;
|
|
24356
|
+
};
|
|
24357
|
+
const assertInspectionCurrent = () => {
|
|
24358
|
+
if (!continueWait()) {
|
|
24359
|
+
throw new AbortError("Persisted-receipt readiness wait settled");
|
|
24360
|
+
}
|
|
24361
|
+
};
|
|
24362
|
+
const armRecoveryTick = () => {
|
|
24363
|
+
if (settled || recoveryTimer) return;
|
|
24364
|
+
const delayMs = Math.max(
|
|
24365
|
+
50,
|
|
24366
|
+
Math.min(1_000, this.waitForReplicatorRequestIntervalMs),
|
|
24367
|
+
);
|
|
24368
|
+
recoveryTimer = setTimeout(() => {
|
|
24369
|
+
recoveryTimer = undefined;
|
|
24370
|
+
if (!continueWait()) return;
|
|
24371
|
+
this.nudgePersistedReceiptPeerReadiness(key);
|
|
24372
|
+
scheduleCheck();
|
|
24373
|
+
}, delayMs);
|
|
24374
|
+
recoveryTimer.unref?.();
|
|
24375
|
+
};
|
|
24376
|
+
const runCheck = async () => {
|
|
24377
|
+
checkScheduled = false;
|
|
24378
|
+
if (!continueWait()) return;
|
|
24379
|
+
if (checkInFlight) {
|
|
24380
|
+
rerun = true;
|
|
24381
|
+
return;
|
|
24382
|
+
}
|
|
24383
|
+
checkInFlight = true;
|
|
24384
|
+
try {
|
|
24385
|
+
let snapshot = await this.inspectPersistedReceiptPeerReadiness(
|
|
24386
|
+
key,
|
|
24387
|
+
inspectOptions,
|
|
24388
|
+
assertInspectionCurrent,
|
|
24389
|
+
);
|
|
24390
|
+
lastSnapshot = snapshot;
|
|
24391
|
+
if (!continueWait()) return;
|
|
24392
|
+
if (rerun) return;
|
|
24393
|
+
if (snapshot.status === "ready") {
|
|
24394
|
+
// A wake observed while the asynchronous inspection was running may
|
|
24395
|
+
// already have invalidated this snapshot. Drain that coalesced wake
|
|
24396
|
+
// before publishing readiness.
|
|
24397
|
+
resolve(snapshot);
|
|
24398
|
+
return;
|
|
24399
|
+
}
|
|
24400
|
+
if (
|
|
24401
|
+
snapshot.status === "pending" &&
|
|
24402
|
+
snapshot.reason === "replication-confirmation-pending"
|
|
24403
|
+
) {
|
|
24404
|
+
const target = this.persistedReceiptPeerSession(peerHash);
|
|
24405
|
+
if (target) {
|
|
24406
|
+
const currentConfirmationController = new AbortController();
|
|
24407
|
+
confirmationController = currentConfirmationController;
|
|
24408
|
+
try {
|
|
24409
|
+
await this._v2Send.confirmLatestForPeer(
|
|
24410
|
+
{
|
|
24411
|
+
peerHash,
|
|
24412
|
+
peerSession: target.peerSession,
|
|
24413
|
+
receiverTransportSession: target.capabilitySession,
|
|
24414
|
+
},
|
|
24415
|
+
{
|
|
24416
|
+
timeout: Math.max(1, deadline - Date.now()),
|
|
24417
|
+
signal: AbortSignal.any([
|
|
24418
|
+
operationSignal,
|
|
24419
|
+
currentConfirmationController.signal,
|
|
24420
|
+
]),
|
|
24421
|
+
},
|
|
24422
|
+
);
|
|
24423
|
+
} catch (error) {
|
|
24424
|
+
if (!continueWait()) return;
|
|
24425
|
+
if (!(error instanceof AbortError)) {
|
|
24426
|
+
throw error;
|
|
24427
|
+
}
|
|
24428
|
+
rerun = true;
|
|
24429
|
+
} finally {
|
|
24430
|
+
if (confirmationController === currentConfirmationController) {
|
|
24431
|
+
confirmationController = undefined;
|
|
24432
|
+
}
|
|
24433
|
+
}
|
|
24434
|
+
if (!continueWait()) return;
|
|
24435
|
+
snapshot = await this.inspectPersistedReceiptPeerReadiness(
|
|
24436
|
+
key,
|
|
24437
|
+
inspectOptions,
|
|
24438
|
+
assertInspectionCurrent,
|
|
24439
|
+
);
|
|
24440
|
+
lastSnapshot = snapshot;
|
|
24441
|
+
if (!continueWait()) return;
|
|
24442
|
+
if (rerun) return;
|
|
24443
|
+
if (snapshot.status === "ready") {
|
|
24444
|
+
resolve(snapshot);
|
|
24445
|
+
return;
|
|
24446
|
+
}
|
|
24447
|
+
}
|
|
24448
|
+
}
|
|
24449
|
+
if (!continueWait()) return;
|
|
24450
|
+
this.nudgePersistedReceiptPeerReadiness(key);
|
|
24451
|
+
} catch (error) {
|
|
24452
|
+
if (!settled) reject(error);
|
|
24453
|
+
} finally {
|
|
24454
|
+
checkInFlight = false;
|
|
24455
|
+
if (!settled && rerun) {
|
|
24456
|
+
rerun = false;
|
|
24457
|
+
scheduleCheck();
|
|
24458
|
+
} else {
|
|
24459
|
+
armRecoveryTick();
|
|
24460
|
+
}
|
|
24461
|
+
}
|
|
24462
|
+
};
|
|
24463
|
+
const scheduleCheck = (interruptConfirmation = false) => {
|
|
24464
|
+
if (settled) return;
|
|
24465
|
+
if (recoveryTimer) {
|
|
24466
|
+
clearTimeout(recoveryTimer);
|
|
24467
|
+
recoveryTimer = undefined;
|
|
24468
|
+
}
|
|
24469
|
+
if (checkInFlight) {
|
|
24470
|
+
rerun = true;
|
|
24471
|
+
if (interruptConfirmation) {
|
|
24472
|
+
confirmationController?.abort(
|
|
24473
|
+
new AbortError(
|
|
24474
|
+
"Persisted-receipt readiness changed during confirmation",
|
|
24475
|
+
),
|
|
24476
|
+
);
|
|
24477
|
+
}
|
|
24478
|
+
return;
|
|
24479
|
+
}
|
|
24480
|
+
if (checkScheduled) return;
|
|
24481
|
+
checkScheduled = true;
|
|
24482
|
+
void Promise.resolve().then(runCheck);
|
|
24483
|
+
};
|
|
24484
|
+
const onReadinessChange = (
|
|
24485
|
+
event: CustomEvent<PersistedReceiptPeerReadinessEvent>,
|
|
24486
|
+
) => {
|
|
24487
|
+
if (event.detail.peerHash === peerHash) scheduleCheck(true);
|
|
24488
|
+
};
|
|
24489
|
+
const onRoleChange = (event: CustomEvent<ReplicationChangeEvent>) => {
|
|
24490
|
+
if (
|
|
24491
|
+
(entries?.length ?? 0) > 0 ||
|
|
24492
|
+
event.detail.publicKey.hashcode() === peerHash
|
|
24493
|
+
) {
|
|
24494
|
+
scheduleCheck(true);
|
|
24495
|
+
}
|
|
24496
|
+
};
|
|
24497
|
+
|
|
24498
|
+
// Register wake sources before the first state inspection. EventTarget does
|
|
24499
|
+
// not replay a transition that fired between an async check and registration.
|
|
24500
|
+
this.events.addEventListener(
|
|
24501
|
+
"persisted-receipt:readiness",
|
|
24502
|
+
onReadinessChange,
|
|
24503
|
+
);
|
|
24504
|
+
this.events.addEventListener("replication:change", onRoleChange);
|
|
24505
|
+
this.events.addEventListener("replicator:mature", onRoleChange);
|
|
24506
|
+
options.signal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
24507
|
+
closeSignal.addEventListener("abort", onClose, { once: true });
|
|
24508
|
+
if (options.signal?.aborted) {
|
|
24509
|
+
onCallerAbort();
|
|
24510
|
+
} else if (closeSignal.aborted) {
|
|
24511
|
+
onClose();
|
|
24512
|
+
} else {
|
|
24513
|
+
scheduleCheck();
|
|
24514
|
+
}
|
|
24515
|
+
|
|
24516
|
+
const timeout = setTimeout(() => reject(createTimeoutError()), timeoutMs);
|
|
24517
|
+
timeout.unref?.();
|
|
24518
|
+
return deferred.promise.finally(() => clearTimeout(timeout));
|
|
24519
|
+
}
|
|
24520
|
+
|
|
23491
24521
|
async waitForReplicator(
|
|
23492
24522
|
key: PublicSignKey,
|
|
23493
24523
|
options?: {
|
|
@@ -23497,19 +24527,28 @@ export class SharedLog<
|
|
|
23497
24527
|
timeout?: number;
|
|
23498
24528
|
},
|
|
23499
24529
|
) {
|
|
24530
|
+
if (options?.signal?.aborted) {
|
|
24531
|
+
throw new AbortError();
|
|
24532
|
+
}
|
|
23500
24533
|
const deferred = pDefer<void>();
|
|
23501
24534
|
const timeoutMs = options?.timeout ?? this.waitForReplicatorTimeout;
|
|
23502
24535
|
const resolvedRoleAge = options?.eager
|
|
23503
24536
|
? undefined
|
|
23504
24537
|
: (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
|
|
24538
|
+
if (options?.signal?.aborted) {
|
|
24539
|
+
throw new AbortError();
|
|
24540
|
+
}
|
|
23505
24541
|
|
|
23506
24542
|
let settled = false;
|
|
23507
24543
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
23508
24544
|
let requestTimer: ReturnType<typeof setTimeout> | undefined;
|
|
24545
|
+
let checkInFlight = false;
|
|
24546
|
+
let checkAgain = false;
|
|
23509
24547
|
|
|
23510
24548
|
const clear = () => {
|
|
23511
|
-
|
|
23512
|
-
this.events.removeEventListener("
|
|
24549
|
+
checkAgain = false;
|
|
24550
|
+
this.events.removeEventListener("replicator:mature", runCheck);
|
|
24551
|
+
this.events.removeEventListener("replication:change", runCheck);
|
|
23513
24552
|
options?.signal?.removeEventListener("abort", onAbort);
|
|
23514
24553
|
if (timer != null) {
|
|
23515
24554
|
clearTimeout(timer);
|
|
@@ -23645,11 +24684,34 @@ export class SharedLog<
|
|
|
23645
24684
|
await iterator?.close();
|
|
23646
24685
|
}
|
|
23647
24686
|
};
|
|
24687
|
+
const runCheck = () => {
|
|
24688
|
+
if (settled) return;
|
|
24689
|
+
if (checkInFlight) {
|
|
24690
|
+
checkAgain = true;
|
|
24691
|
+
return;
|
|
24692
|
+
}
|
|
24693
|
+
// Reserve synchronously before `check()` can dispatch/re-enter from an
|
|
24694
|
+
// index implementation's first `next()` call.
|
|
24695
|
+
checkInFlight = true;
|
|
24696
|
+
void check()
|
|
24697
|
+
.catch((error) =>
|
|
24698
|
+
reject(error instanceof Error ? error : new Error(String(error))),
|
|
24699
|
+
)
|
|
24700
|
+
.finally(() => {
|
|
24701
|
+
checkInFlight = false;
|
|
24702
|
+
if (!settled && checkAgain) {
|
|
24703
|
+
checkAgain = false;
|
|
24704
|
+
runCheck();
|
|
24705
|
+
}
|
|
24706
|
+
});
|
|
24707
|
+
};
|
|
23648
24708
|
|
|
24709
|
+
// Register before the first asynchronous index read. EventTarget does not
|
|
24710
|
+
// replay a maturity/change event that fires while that read is in flight.
|
|
24711
|
+
this.events.addEventListener("replicator:mature", runCheck);
|
|
24712
|
+
this.events.addEventListener("replication:change", runCheck);
|
|
23649
24713
|
requestReplicationInfo();
|
|
23650
|
-
|
|
23651
|
-
this.events.addEventListener("replicator:mature", check);
|
|
23652
|
-
this.events.addEventListener("replication:change", check);
|
|
24714
|
+
runCheck();
|
|
23653
24715
|
|
|
23654
24716
|
return deferred.promise.finally(clear);
|
|
23655
24717
|
}
|
|
@@ -24866,6 +25928,7 @@ export class SharedLog<
|
|
|
24866
25928
|
options?: {
|
|
24867
25929
|
roleAge?: number;
|
|
24868
25930
|
candidates?: Iterable<string>;
|
|
25931
|
+
freshLeaderPlan?: boolean;
|
|
24869
25932
|
},
|
|
24870
25933
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
24871
25934
|
): Promise<LeaderSelectionContext> {
|
|
@@ -25027,6 +26090,7 @@ export class SharedLog<
|
|
|
25027
26090
|
options?: {
|
|
25028
26091
|
roleAge?: number;
|
|
25029
26092
|
candidates?: Iterable<string>;
|
|
26093
|
+
freshLeaderPlan?: boolean;
|
|
25030
26094
|
},
|
|
25031
26095
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
25032
26096
|
): Promise<Map<string, { intersecting: boolean }>> {
|
|
@@ -26537,6 +27601,7 @@ export class SharedLog<
|
|
|
26537
27601
|
if (!ownsSubscriptionEpoch()) {
|
|
26538
27602
|
return;
|
|
26539
27603
|
}
|
|
27604
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
26540
27605
|
// A reconnect can arrive before the previous exact-session recovery tick
|
|
26541
27606
|
// observes its stale session. Retire that job synchronously so it cannot
|
|
26542
27607
|
// suppress the replacement session's scheduler in the shared peer slot.
|
|
@@ -26709,6 +27774,7 @@ export class SharedLog<
|
|
|
26709
27774
|
publicKey,
|
|
26710
27775
|
replicationLifecycleController,
|
|
26711
27776
|
);
|
|
27777
|
+
this.dispatchPersistedReceiptReadinessChange(peerHash);
|
|
26712
27778
|
}
|
|
26713
27779
|
|
|
26714
27780
|
private getClampedReplicas(customValue?: MinReplicas) {
|