@peerbit/shared-log 16.0.30 → 16.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +101 -4
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/dispatch-lifecycle.d.ts +5 -0
- package/dist/src/sync/dispatch-lifecycle.d.ts.map +1 -1
- package/dist/src/sync/dispatch-lifecycle.js +15 -0
- package/dist/src/sync/dispatch-lifecycle.js.map +1 -1
- package/dist/src/sync/index.d.ts +10 -1
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +8 -1
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +140 -13
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +6 -2
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +181 -122
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +12 -12
- package/src/index.ts +128 -6
- package/src/sync/dispatch-lifecycle.ts +20 -0
- package/src/sync/index.ts +10 -0
- package/src/sync/rateless-iblt.ts +178 -11
- package/src/sync/simple.ts +252 -154
package/src/index.ts
CHANGED
|
@@ -297,6 +297,7 @@ import {
|
|
|
297
297
|
maxReplicas,
|
|
298
298
|
} from "./replication.js";
|
|
299
299
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
300
|
+
import { SyncReceiveAbortError } from "./sync/dispatch-lifecycle.js";
|
|
300
301
|
import { createSyncronizer } from "./sync/factory.js";
|
|
301
302
|
import type {
|
|
302
303
|
SharedLogNativeWireSync,
|
|
@@ -417,6 +418,7 @@ type PendingIHave<T> = {
|
|
|
417
418
|
|
|
418
419
|
type PeerReceiveLeaseBucket = {
|
|
419
420
|
active: number;
|
|
421
|
+
controller: AbortController;
|
|
420
422
|
drain?: DeferredPromise<void>;
|
|
421
423
|
};
|
|
422
424
|
|
|
@@ -433,13 +435,15 @@ type PeerReceiveLeaseState = {
|
|
|
433
435
|
*/
|
|
434
436
|
type PeerReceiveLease = {
|
|
435
437
|
release: () => void;
|
|
438
|
+
signal: AbortSignal;
|
|
436
439
|
};
|
|
437
440
|
|
|
438
441
|
const createOneShotPeerReceiveLease = (
|
|
439
|
-
releaseFn: () => void,
|
|
442
|
+
releaseFn: (() => void) & { signal: AbortSignal },
|
|
440
443
|
): PeerReceiveLease => {
|
|
441
444
|
let released = false;
|
|
442
445
|
return {
|
|
446
|
+
signal: releaseFn.signal,
|
|
443
447
|
release: () => {
|
|
444
448
|
if (released) {
|
|
445
449
|
return;
|
|
@@ -6352,6 +6356,95 @@ export class SharedLog<
|
|
|
6352
6356
|
records.size,
|
|
6353
6357
|
);
|
|
6354
6358
|
const signal = deadline.signal;
|
|
6359
|
+
const recoveryController = new AbortController();
|
|
6360
|
+
const recoverySignal = AbortSignal.any([signal, recoveryController.signal]);
|
|
6361
|
+
const recoveryByPeer = new Map<
|
|
6362
|
+
string,
|
|
6363
|
+
{
|
|
6364
|
+
controller: AbortController;
|
|
6365
|
+
timer: ReturnType<typeof setTimeout>;
|
|
6366
|
+
}
|
|
6367
|
+
>();
|
|
6368
|
+
let recoveryCursor = 0;
|
|
6369
|
+
const recoverSelectedPeers = (selected: Set<string>) => {
|
|
6370
|
+
// Recovery is advisory work, not a receipt or a replacement leader plan.
|
|
6371
|
+
// Never occupy transfer/request slots while waiting for it. The separate
|
|
6372
|
+
// bounded pool rotates through fresh candidates so quiet/incomplete peers
|
|
6373
|
+
// cannot indefinitely hide a later recoverable peer.
|
|
6374
|
+
for (const [peer, state] of recoveryByPeer) {
|
|
6375
|
+
if (!selected.has(peer)) {
|
|
6376
|
+
clearTimeout(state.timer);
|
|
6377
|
+
state.controller.abort();
|
|
6378
|
+
}
|
|
6379
|
+
}
|
|
6380
|
+
const peers = [...selected];
|
|
6381
|
+
for (
|
|
6382
|
+
let visited = 0;
|
|
6383
|
+
visited < peers.length &&
|
|
6384
|
+
recoveryByPeer.size < MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL &&
|
|
6385
|
+
!recoverySignal.aborted;
|
|
6386
|
+
visited++
|
|
6387
|
+
) {
|
|
6388
|
+
const peer = peers[recoveryCursor++ % peers.length]!;
|
|
6389
|
+
if (recoveryByPeer.has(peer)) continue;
|
|
6390
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
6391
|
+
if (
|
|
6392
|
+
current &&
|
|
6393
|
+
this._v2Send.isLatestConfirmedForPeer({
|
|
6394
|
+
peerHash: peer,
|
|
6395
|
+
peerSession: current.peerSession,
|
|
6396
|
+
receiverTransportSession: current.capabilitySession,
|
|
6397
|
+
})
|
|
6398
|
+
) {
|
|
6399
|
+
continue;
|
|
6400
|
+
}
|
|
6401
|
+
const expiresAt = Math.min(
|
|
6402
|
+
deadline.deadline,
|
|
6403
|
+
Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS,
|
|
6404
|
+
);
|
|
6405
|
+
const controller = new AbortController();
|
|
6406
|
+
const attemptSignal = AbortSignal.any([
|
|
6407
|
+
recoverySignal,
|
|
6408
|
+
controller.signal,
|
|
6409
|
+
]);
|
|
6410
|
+
const timer = setTimeout(
|
|
6411
|
+
() => controller.abort(),
|
|
6412
|
+
Math.max(1, expiresAt - Date.now()),
|
|
6413
|
+
);
|
|
6414
|
+
timer.unref?.();
|
|
6415
|
+
const state = { controller, timer };
|
|
6416
|
+
recoveryByPeer.set(peer, state);
|
|
6417
|
+
void (async () => {
|
|
6418
|
+
// Resolve only an authenticated transport-cache key. Keep this slot
|
|
6419
|
+
// reserved until the lookup actually settles, even if a custom
|
|
6420
|
+
// resolver ignores cancellation, rather than launching duplicates.
|
|
6421
|
+
const key = await this._resolvePublicKeyFromHash(peer);
|
|
6422
|
+
if (
|
|
6423
|
+
attemptSignal.aborted ||
|
|
6424
|
+
!key ||
|
|
6425
|
+
key.hashcode() !== peer ||
|
|
6426
|
+
Date.now() >= expiresAt
|
|
6427
|
+
) {
|
|
6428
|
+
return;
|
|
6429
|
+
}
|
|
6430
|
+
// Reuse the exact-session watchdog, capability/subscriber recovery,
|
|
6431
|
+
// and replacement-session handling from the public preflight. Its
|
|
6432
|
+
// result is ignored: settlement still checks the exact entry leaders
|
|
6433
|
+
// and accepts only the subsequent durable, session-bound receipts.
|
|
6434
|
+
await this.waitForPersistedReceiptPeerReadiness(key, {
|
|
6435
|
+
timeout: Math.max(1, expiresAt - Date.now()),
|
|
6436
|
+
signal: attemptSignal,
|
|
6437
|
+
});
|
|
6438
|
+
})()
|
|
6439
|
+
.catch(() => undefined)
|
|
6440
|
+
.finally(() => {
|
|
6441
|
+
clearTimeout(timer);
|
|
6442
|
+
if (recoveryByPeer.get(peer) === state) {
|
|
6443
|
+
recoveryByPeer.delete(peer);
|
|
6444
|
+
}
|
|
6445
|
+
});
|
|
6446
|
+
}
|
|
6447
|
+
};
|
|
6355
6448
|
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
6356
6449
|
let initialTransferPending = transferOnFirstRound;
|
|
6357
6450
|
let needsInitialLeaderCheck = true;
|
|
@@ -6418,6 +6511,7 @@ export class SharedLog<
|
|
|
6418
6511
|
// transport epoch. A revision/session change purges them before they can
|
|
6419
6512
|
// survive an away-and-back leader transition or combine with a later peer.
|
|
6420
6513
|
const hashesByPeer = new Map<string, string[]>();
|
|
6514
|
+
const recoveryCandidates = new Set<string>();
|
|
6421
6515
|
const entryArray = [...records.values()];
|
|
6422
6516
|
const leadersByEntry = await this.planPersistedDeliveryLeaders(
|
|
6423
6517
|
entryArray,
|
|
@@ -6462,6 +6556,7 @@ export class SharedLog<
|
|
|
6462
6556
|
if (acknowledgements.size >= minAcks) continue;
|
|
6463
6557
|
for (const peer of leaders.keys()) {
|
|
6464
6558
|
if (peer === selfHash) continue;
|
|
6559
|
+
recoveryCandidates.add(peer);
|
|
6465
6560
|
const current = this.persistedReceiptPeerSession(peer);
|
|
6466
6561
|
if (!current) continue;
|
|
6467
6562
|
if (acknowledgements.has(peer)) continue;
|
|
@@ -6471,6 +6566,7 @@ export class SharedLog<
|
|
|
6471
6566
|
}
|
|
6472
6567
|
}
|
|
6473
6568
|
if (!isRoundOwnershipCurrent()) continue;
|
|
6569
|
+
recoverSelectedPeers(recoveryCandidates);
|
|
6474
6570
|
|
|
6475
6571
|
const operationQueue = new PQueue({
|
|
6476
6572
|
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
@@ -6831,6 +6927,11 @@ export class SharedLog<
|
|
|
6831
6927
|
}
|
|
6832
6928
|
throw new PersistedDeliveryError(error, committedHashes);
|
|
6833
6929
|
} finally {
|
|
6930
|
+
recoveryController.abort();
|
|
6931
|
+
for (const state of recoveryByPeer.values()) {
|
|
6932
|
+
clearTimeout(state.timer);
|
|
6933
|
+
state.controller.abort();
|
|
6934
|
+
}
|
|
6834
6935
|
if (ownedDeadline) deadline.dispose();
|
|
6835
6936
|
}
|
|
6836
6937
|
}
|
|
@@ -7740,7 +7841,7 @@ export class SharedLog<
|
|
|
7740
7841
|
allowReplicationInfoBlocked?: boolean;
|
|
7741
7842
|
allowCleanupGate?: boolean;
|
|
7742
7843
|
},
|
|
7743
|
-
): (() => void) | undefined {
|
|
7844
|
+
): ((() => void) & { signal: AbortSignal }) | undefined {
|
|
7744
7845
|
if (
|
|
7745
7846
|
!this._peerSessions.isReceiveAdmissionOpen(
|
|
7746
7847
|
peerHash,
|
|
@@ -7754,7 +7855,10 @@ export class SharedLog<
|
|
|
7754
7855
|
|
|
7755
7856
|
let state = this._activeReceiveHandlersByPeer.get(peerHash);
|
|
7756
7857
|
if (!state) {
|
|
7757
|
-
const current: PeerReceiveLeaseBucket = {
|
|
7858
|
+
const current: PeerReceiveLeaseBucket = {
|
|
7859
|
+
active: 0,
|
|
7860
|
+
controller: new AbortController(),
|
|
7861
|
+
};
|
|
7758
7862
|
state = { current, activeBuckets: new Set() };
|
|
7759
7863
|
this._activeReceiveHandlersByPeer.set(peerHash, state);
|
|
7760
7864
|
}
|
|
@@ -7762,7 +7866,7 @@ export class SharedLog<
|
|
|
7762
7866
|
bucket.active += 1;
|
|
7763
7867
|
state.activeBuckets.add(bucket);
|
|
7764
7868
|
let released = false;
|
|
7765
|
-
|
|
7869
|
+
const release = () => {
|
|
7766
7870
|
if (released) {
|
|
7767
7871
|
return;
|
|
7768
7872
|
}
|
|
@@ -7781,6 +7885,7 @@ export class SharedLog<
|
|
|
7781
7885
|
this._activeReceiveHandlersByPeer.delete(peerHash);
|
|
7782
7886
|
}
|
|
7783
7887
|
};
|
|
7888
|
+
return Object.assign(release, { signal: bucket.controller.signal });
|
|
7784
7889
|
}
|
|
7785
7890
|
|
|
7786
7891
|
private async drainPeerReceiveHandlers(peerHash: string): Promise<void> {
|
|
@@ -7793,7 +7898,7 @@ export class SharedLog<
|
|
|
7793
7898
|
// sync traffic without joining the drain for the previous subscription. Cleanup
|
|
7794
7899
|
// callers gate admission first; terminal callers also repeat until empty.
|
|
7795
7900
|
const buckets = [...state.activeBuckets];
|
|
7796
|
-
state.current = { active: 0 };
|
|
7901
|
+
state.current = { active: 0, controller: new AbortController() };
|
|
7797
7902
|
const drain = Promise.all(
|
|
7798
7903
|
buckets.map((bucket) => {
|
|
7799
7904
|
bucket.drain ??= pDefer<void>();
|
|
@@ -7806,6 +7911,14 @@ export class SharedLog<
|
|
|
7806
7911
|
this._receiveHandlerDrainByPeer.set(peerHash, drains);
|
|
7807
7912
|
}
|
|
7808
7913
|
drains.add(drain);
|
|
7914
|
+
// Cancel only work admitted into this snapshot, never a replacement's
|
|
7915
|
+
// current bucket. Aborting requests cooperation; the physical receive
|
|
7916
|
+
// leases below still own all cleanup until their actual finally settles.
|
|
7917
|
+
for (const bucket of buckets) {
|
|
7918
|
+
bucket.controller.abort(
|
|
7919
|
+
new SyncReceiveAbortError("peer receive generation draining"),
|
|
7920
|
+
);
|
|
7921
|
+
}
|
|
7809
7922
|
try {
|
|
7810
7923
|
await drain;
|
|
7811
7924
|
} finally {
|
|
@@ -20953,6 +21066,10 @@ export class SharedLog<
|
|
|
20953
21066
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
20954
21067
|
try {
|
|
20955
21068
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
21069
|
+
// A receive may be awaiting a synchronizer response shipment. Cancel
|
|
21070
|
+
// its dispatch generation before draining that receive, rather than
|
|
21071
|
+
// waiting for _close() to reach the synchronizer's final teardown.
|
|
21072
|
+
this.syncronizer?.beginClose?.();
|
|
20956
21073
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
20957
21074
|
await this.drainSubscriptionChangeCallbacks();
|
|
20958
21075
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -21091,6 +21208,7 @@ export class SharedLog<
|
|
|
21091
21208
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
21092
21209
|
try {
|
|
21093
21210
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
21211
|
+
this.syncronizer?.beginClose?.();
|
|
21094
21212
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
21095
21213
|
await this.drainSubscriptionChangeCallbacks();
|
|
21096
21214
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -23550,7 +23668,11 @@ export class SharedLog<
|
|
|
23550
23668
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
23551
23669
|
}
|
|
23552
23670
|
return;
|
|
23553
|
-
} else if (
|
|
23671
|
+
} else if (
|
|
23672
|
+
await this.syncronizer.onMessage(msg, context, {
|
|
23673
|
+
signal: peerReceiveLease.signal,
|
|
23674
|
+
})
|
|
23675
|
+
) {
|
|
23554
23676
|
return; // the syncronizer has handled the message
|
|
23555
23677
|
} else if (msg instanceof BlocksMessage) {
|
|
23556
23678
|
await this.remoteBlocks.onMessage(msg.message, {
|
|
@@ -16,6 +16,26 @@
|
|
|
16
16
|
// caller-specific fields such as epochs, batches and retained-work
|
|
17
17
|
// counters); the registry owns the per-target active sets, the listener
|
|
18
18
|
// add/remove pairing and the dispose gating.
|
|
19
|
+
import { AbortError } from "@peerbit/time";
|
|
20
|
+
|
|
21
|
+
/** Local receive drain cancellation; never a transport or persistence failure. */
|
|
22
|
+
export class SyncReceiveAbortError extends AbortError {}
|
|
23
|
+
|
|
24
|
+
export const isSyncDispatchCancellation = (
|
|
25
|
+
error: unknown,
|
|
26
|
+
signal: AbortSignal | undefined,
|
|
27
|
+
inactive = signal?.aborted === true,
|
|
28
|
+
): boolean => {
|
|
29
|
+
if (!inactive) return false;
|
|
30
|
+
// Keep the existing owner/disconnect error policy. Only the new receive
|
|
31
|
+
// cancellation must distinguish unrelated failures that race its abort.
|
|
32
|
+
if (!(signal?.reason instanceof SyncReceiveAbortError)) return true;
|
|
33
|
+
return (
|
|
34
|
+
error === signal.reason ||
|
|
35
|
+
error instanceof AbortError ||
|
|
36
|
+
(error instanceof Error && error.name === "AbortError")
|
|
37
|
+
);
|
|
38
|
+
};
|
|
19
39
|
|
|
20
40
|
export interface DispatchTargetLifecycleBase<LC> {
|
|
21
41
|
lifecycle: LC;
|
package/src/sync/index.ts
CHANGED
|
@@ -231,6 +231,9 @@ export interface Syncronizer<R extends "u32" | "u64"> {
|
|
|
231
231
|
onMessage(
|
|
232
232
|
message: TransportMessage,
|
|
233
233
|
context: RequestContext,
|
|
234
|
+
// Exact admitted receive lifetime, not a peer-hash-wide disconnect.
|
|
235
|
+
// Implementations must still settle physical work before returning.
|
|
236
|
+
options?: { signal?: AbortSignal },
|
|
234
237
|
): Promise<boolean> | boolean;
|
|
235
238
|
|
|
236
239
|
onReceivedEntries(properties: {
|
|
@@ -251,6 +254,13 @@ export interface Syncronizer<R extends "u32" | "u64"> {
|
|
|
251
254
|
onPeerDisconnected(key: PublicSignKey | string): void;
|
|
252
255
|
|
|
253
256
|
open(): Promise<void> | void;
|
|
257
|
+
/**
|
|
258
|
+
* Synchronously fence dispatch admission and cancel the current generation's
|
|
259
|
+
* network waits before SharedLog drains admitted receives. Do not stop shared
|
|
260
|
+
* indexes or discard physical-work accounting here: close() still runs after
|
|
261
|
+
* those receives settle. Optional for existing custom synchronizers.
|
|
262
|
+
*/
|
|
263
|
+
beginClose?(): void;
|
|
254
264
|
close(): Promise<void> | void;
|
|
255
265
|
|
|
256
266
|
get pending(): number;
|
|
@@ -27,7 +27,9 @@ import { TransportMessage } from "../message.js";
|
|
|
27
27
|
import { type EntryReplicated } from "../ranges.js";
|
|
28
28
|
import {
|
|
29
29
|
DispatchLifecycleRegistry,
|
|
30
|
+
SyncReceiveAbortError,
|
|
30
31
|
isOwnershipGenerationActive,
|
|
32
|
+
isSyncDispatchCancellation,
|
|
31
33
|
isTrackedSessionActive,
|
|
32
34
|
} from "./dispatch-lifecycle.js";
|
|
33
35
|
import type {
|
|
@@ -827,6 +829,7 @@ type OutgoingRatelessSyncProcess = {
|
|
|
827
829
|
lastSeqNo: bigint;
|
|
828
830
|
}) => { symbols: CodedSymbolBatch; exhaustedAfterSend: boolean } | undefined;
|
|
829
831
|
startSimpleFallback: () => Promise<void>;
|
|
832
|
+
cancelSimpleFallback: (reason: unknown) => void;
|
|
830
833
|
simpleFallbackStarted: boolean;
|
|
831
834
|
free: (reason?: unknown) => void;
|
|
832
835
|
processController: AbortController;
|
|
@@ -863,6 +866,7 @@ type IncomingRatelessSyncProcess = {
|
|
|
863
866
|
symbols: CodedSymbolInput;
|
|
864
867
|
}) => Promise<IncomingRatelessProcessResult>;
|
|
865
868
|
requestAll: () => Promise<void>;
|
|
869
|
+
drain: () => Promise<void>;
|
|
866
870
|
fallbackToSimple: (reason?: unknown) => Promise<void>;
|
|
867
871
|
free: (reason?: unknown) => void;
|
|
868
872
|
complete: () => void;
|
|
@@ -1127,6 +1131,9 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1127
1131
|
timeoutMs?: number;
|
|
1128
1132
|
retryIntervalsMs?: number[];
|
|
1129
1133
|
}): RepairSession {
|
|
1134
|
+
if (this.ratelessClosed) {
|
|
1135
|
+
return this.simple.startRepairSession(properties);
|
|
1136
|
+
}
|
|
1130
1137
|
const mode = properties.mode ?? "best-effort";
|
|
1131
1138
|
const targets = [...new Set(properties.targets)];
|
|
1132
1139
|
const timeoutMs = Math.max(
|
|
@@ -1568,6 +1575,35 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1568
1575
|
);
|
|
1569
1576
|
}
|
|
1570
1577
|
|
|
1578
|
+
private async settleIncomingReceive(
|
|
1579
|
+
process: IncomingRatelessSyncProcess,
|
|
1580
|
+
signal: AbortSignal,
|
|
1581
|
+
): Promise<void> {
|
|
1582
|
+
const errors: unknown[] = [];
|
|
1583
|
+
try {
|
|
1584
|
+
await process.drain();
|
|
1585
|
+
} catch (error) {
|
|
1586
|
+
errors.push(error);
|
|
1587
|
+
}
|
|
1588
|
+
if (signal.aborted) {
|
|
1589
|
+
try {
|
|
1590
|
+
process.free(signal.reason);
|
|
1591
|
+
} catch (error) {
|
|
1592
|
+
errors.push(error);
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
if (errors.length === 1) throw errors[0];
|
|
1596
|
+
if (errors.length > 1) {
|
|
1597
|
+
throw new AggregateError(
|
|
1598
|
+
errors,
|
|
1599
|
+
"incoming receive drain and cleanup failed",
|
|
1600
|
+
{
|
|
1601
|
+
cause: errors[0],
|
|
1602
|
+
},
|
|
1603
|
+
);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1571
1607
|
async onMaybeMissingEntries(properties: {
|
|
1572
1608
|
entries: Map<string, SyncEntryCoordinates<D>>;
|
|
1573
1609
|
targets: string[];
|
|
@@ -2005,6 +2041,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2005
2041
|
let symbolsProduced = startSyncSymbols.length;
|
|
2006
2042
|
let symbolBudgetExhausted = false;
|
|
2007
2043
|
let simpleFallbackPromise: Promise<void> | undefined;
|
|
2044
|
+
const simpleFallbackController = new AbortController();
|
|
2008
2045
|
const symbolBudget = getOutgoingRatelessSymbolBudget(
|
|
2009
2046
|
properties.coordinates.length,
|
|
2010
2047
|
startSyncSymbols.length,
|
|
@@ -2052,7 +2089,14 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2052
2089
|
this.simple.onMaybeMissingHashes({
|
|
2053
2090
|
hashes: properties.outgoingHashes,
|
|
2054
2091
|
targets: [target],
|
|
2055
|
-
|
|
2092
|
+
// Encoder retirement must not cancel an admitted fallback.
|
|
2093
|
+
// A receive awaiting that exact memoized operation can do so.
|
|
2094
|
+
signal: lifecycle.callerSignal
|
|
2095
|
+
? AbortSignal.any([
|
|
2096
|
+
lifecycle.callerSignal,
|
|
2097
|
+
simpleFallbackController.signal,
|
|
2098
|
+
])
|
|
2099
|
+
: simpleFallbackController.signal,
|
|
2056
2100
|
}),
|
|
2057
2101
|
);
|
|
2058
2102
|
}
|
|
@@ -2147,6 +2191,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2147
2191
|
return { symbols, exhaustedAfterSend };
|
|
2148
2192
|
},
|
|
2149
2193
|
startSimpleFallback,
|
|
2194
|
+
cancelSimpleFallback: (reason) =>
|
|
2195
|
+
simpleFallbackController.abort(reason),
|
|
2150
2196
|
simpleFallbackStarted: false,
|
|
2151
2197
|
free: clear,
|
|
2152
2198
|
outgoingHashes: properties.outgoingHashes,
|
|
@@ -2383,7 +2429,74 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2383
2429
|
async onMessage(
|
|
2384
2430
|
message: TransportMessage,
|
|
2385
2431
|
context: RequestContext,
|
|
2432
|
+
options?: { signal?: AbortSignal },
|
|
2433
|
+
): Promise<boolean> {
|
|
2434
|
+
const signal = options?.signal;
|
|
2435
|
+
if (!signal) {
|
|
2436
|
+
return this.onMessageForReceive(message, context);
|
|
2437
|
+
}
|
|
2438
|
+
let cancel: (() => void) | undefined;
|
|
2439
|
+
let settle: (() => Promise<void>) | undefined;
|
|
2440
|
+
const onAbort = () => cancel?.();
|
|
2441
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2442
|
+
const errors: unknown[] = [];
|
|
2443
|
+
let handled = false;
|
|
2444
|
+
try {
|
|
2445
|
+
handled = await this.onMessageForReceive(message, context, {
|
|
2446
|
+
signal,
|
|
2447
|
+
bind: (cancelWork, settleWork) => {
|
|
2448
|
+
cancel = cancelWork;
|
|
2449
|
+
settle = settleWork;
|
|
2450
|
+
if (signal.aborted) {
|
|
2451
|
+
onAbort();
|
|
2452
|
+
}
|
|
2453
|
+
},
|
|
2454
|
+
});
|
|
2455
|
+
} catch (error) {
|
|
2456
|
+
errors.push(error);
|
|
2457
|
+
} finally {
|
|
2458
|
+
// Keep cancellation connected while a bounded fallback's logical
|
|
2459
|
+
// completion still leaves its lower send physically draining.
|
|
2460
|
+
try {
|
|
2461
|
+
await settle?.();
|
|
2462
|
+
} catch (error) {
|
|
2463
|
+
if (!errors.includes(error)) {
|
|
2464
|
+
errors.push(error);
|
|
2465
|
+
}
|
|
2466
|
+
} finally {
|
|
2467
|
+
signal.removeEventListener("abort", onAbort);
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
if (errors.length === 1) throw errors[0];
|
|
2471
|
+
if (errors.length > 1) {
|
|
2472
|
+
throw new AggregateError(
|
|
2473
|
+
errors,
|
|
2474
|
+
"sync receive and physical cleanup failed",
|
|
2475
|
+
{
|
|
2476
|
+
cause: errors[0],
|
|
2477
|
+
},
|
|
2478
|
+
);
|
|
2479
|
+
}
|
|
2480
|
+
return handled;
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
private async onMessageForReceive(
|
|
2484
|
+
message: TransportMessage,
|
|
2485
|
+
context: RequestContext,
|
|
2486
|
+
options?: {
|
|
2487
|
+
signal: AbortSignal;
|
|
2488
|
+
bind: (cancel: () => void, settle?: () => Promise<void>) => void;
|
|
2489
|
+
},
|
|
2386
2490
|
): Promise<boolean> {
|
|
2491
|
+
if (options?.signal.aborted) {
|
|
2492
|
+
return (
|
|
2493
|
+
message instanceof StartSync ||
|
|
2494
|
+
message instanceof MoreSymbols ||
|
|
2495
|
+
message instanceof RequestMoreSymbols ||
|
|
2496
|
+
message instanceof RequestAll ||
|
|
2497
|
+
(await this.simple.onMessage(message, context, options))
|
|
2498
|
+
);
|
|
2499
|
+
}
|
|
2387
2500
|
const profile = this.properties.sync?.profile;
|
|
2388
2501
|
if (message instanceof StartSync) {
|
|
2389
2502
|
const from = context.from;
|
|
@@ -2508,6 +2621,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2508
2621
|
symbolBudget: getIncomingRatelessSymbolBudget(message.symbols.length),
|
|
2509
2622
|
process: async () => undefined,
|
|
2510
2623
|
requestAll: async () => {},
|
|
2624
|
+
drain: async () => {},
|
|
2511
2625
|
fallbackToSimple: async () => {},
|
|
2512
2626
|
free,
|
|
2513
2627
|
complete,
|
|
@@ -2519,6 +2633,10 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2519
2633
|
onOwnershipAbort,
|
|
2520
2634
|
{ once: true },
|
|
2521
2635
|
);
|
|
2636
|
+
options?.bind(
|
|
2637
|
+
() => controller.abort(options.signal.reason),
|
|
2638
|
+
() => this.settleIncomingReceive(obj, options.signal),
|
|
2639
|
+
);
|
|
2522
2640
|
obj.deadlineTimeout = setTimeout(() => {
|
|
2523
2641
|
void obj.fallbackToSimple(
|
|
2524
2642
|
new Error("incoming rateless process deadline exceeded"),
|
|
@@ -2531,6 +2649,9 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2531
2649
|
}
|
|
2532
2650
|
|
|
2533
2651
|
let requestAllPromise: Promise<void> | undefined;
|
|
2652
|
+
obj.drain = async () => {
|
|
2653
|
+
await requestAllPromise;
|
|
2654
|
+
};
|
|
2534
2655
|
obj.requestAll = () => {
|
|
2535
2656
|
if (requestAllPromise) {
|
|
2536
2657
|
return requestAllPromise;
|
|
@@ -2563,6 +2684,12 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2563
2684
|
syncId,
|
|
2564
2685
|
});
|
|
2565
2686
|
}
|
|
2687
|
+
} catch (error) {
|
|
2688
|
+
// Classify before complete() aborts the process as normal cleanup;
|
|
2689
|
+
// that cleanup must not disguise a genuine lower-send failure.
|
|
2690
|
+
if (!isSyncDispatchCancellation(error, controller.signal)) {
|
|
2691
|
+
throw error;
|
|
2692
|
+
}
|
|
2566
2693
|
} finally {
|
|
2567
2694
|
fallbackSendPending = false;
|
|
2568
2695
|
if (this.isIncomingSyncProcessActive(obj)) {
|
|
@@ -2640,7 +2767,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2640
2767
|
const processAborted = controller.signal.aborted;
|
|
2641
2768
|
free(error);
|
|
2642
2769
|
if (
|
|
2643
|
-
processAborted
|
|
2770
|
+
(processAborted &&
|
|
2771
|
+
isSyncDispatchCancellation(error, controller.signal)) ||
|
|
2644
2772
|
!this.isIncomingSyncGenerationActive(ownershipLifecycleController)
|
|
2645
2773
|
) {
|
|
2646
2774
|
return true;
|
|
@@ -2921,7 +3049,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2921
3049
|
},
|
|
2922
3050
|
);
|
|
2923
3051
|
} catch (error) {
|
|
2924
|
-
if (
|
|
3052
|
+
if (
|
|
3053
|
+
isSyncDispatchCancellation(
|
|
3054
|
+
error,
|
|
3055
|
+
controller.signal,
|
|
3056
|
+
!this.isIncomingSyncProcessActive(obj),
|
|
3057
|
+
)
|
|
3058
|
+
) {
|
|
2925
3059
|
return true;
|
|
2926
3060
|
}
|
|
2927
3061
|
free(error);
|
|
@@ -2956,6 +3090,10 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
2956
3090
|
) {
|
|
2957
3091
|
return true;
|
|
2958
3092
|
}
|
|
3093
|
+
options?.bind(
|
|
3094
|
+
() => obj.controller.abort(options.signal.reason),
|
|
3095
|
+
() => this.settleIncomingReceive(obj, options.signal),
|
|
3096
|
+
);
|
|
2959
3097
|
let outProcess: IncomingRatelessProcessResult;
|
|
2960
3098
|
try {
|
|
2961
3099
|
outProcess = await obj.process(message);
|
|
@@ -3002,7 +3140,13 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3002
3140
|
signal: obj.controller.signal,
|
|
3003
3141
|
},
|
|
3004
3142
|
);
|
|
3005
|
-
} catch {
|
|
3143
|
+
} catch (error) {
|
|
3144
|
+
if (
|
|
3145
|
+
obj.controller.signal.reason instanceof SyncReceiveAbortError &&
|
|
3146
|
+
!isSyncDispatchCancellation(error, obj.controller.signal)
|
|
3147
|
+
) {
|
|
3148
|
+
throw error;
|
|
3149
|
+
}
|
|
3006
3150
|
if (profile) {
|
|
3007
3151
|
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
3008
3152
|
name: "rateless.sendRequestMoreSymbols",
|
|
@@ -3036,6 +3180,14 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3036
3180
|
if (context.from?.hashcode() !== obj.target) {
|
|
3037
3181
|
return true;
|
|
3038
3182
|
}
|
|
3183
|
+
options?.bind(
|
|
3184
|
+
() => obj.processController.abort(options.signal.reason),
|
|
3185
|
+
async () => {
|
|
3186
|
+
if (options.signal.aborted) {
|
|
3187
|
+
obj.free(options.signal.reason);
|
|
3188
|
+
}
|
|
3189
|
+
},
|
|
3190
|
+
);
|
|
3039
3191
|
const signal = obj.signal;
|
|
3040
3192
|
if (signal.aborted) {
|
|
3041
3193
|
return true;
|
|
@@ -3063,7 +3215,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3063
3215
|
},
|
|
3064
3216
|
);
|
|
3065
3217
|
} catch (error) {
|
|
3066
|
-
if (signal
|
|
3218
|
+
if (isSyncDispatchCancellation(error, signal)) {
|
|
3067
3219
|
return true;
|
|
3068
3220
|
}
|
|
3069
3221
|
throw error;
|
|
@@ -3098,7 +3250,14 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3098
3250
|
}
|
|
3099
3251
|
// RequestAll ends only this target's rateless attempt. Other target
|
|
3100
3252
|
// encoders and response authorizations remain independently owned.
|
|
3101
|
-
|
|
3253
|
+
let fallback: Promise<void> | undefined;
|
|
3254
|
+
options?.bind(
|
|
3255
|
+
() => p.cancelSimpleFallback(options.signal.reason),
|
|
3256
|
+
async () => {
|
|
3257
|
+
await fallback;
|
|
3258
|
+
},
|
|
3259
|
+
);
|
|
3260
|
+
fallback = p.startSimpleFallback();
|
|
3102
3261
|
p.free();
|
|
3103
3262
|
await fallback;
|
|
3104
3263
|
return true;
|
|
@@ -3171,13 +3330,17 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3171
3330
|
entries: 0,
|
|
3172
3331
|
};
|
|
3173
3332
|
try {
|
|
3333
|
+
const signal = options?.signal
|
|
3334
|
+
? AbortSignal.any([response.signal, options.signal])
|
|
3335
|
+
: response.signal;
|
|
3174
3336
|
responseShipment =
|
|
3175
3337
|
await this.simple.shipAuthorizedMaybeSyncResponse({
|
|
3176
3338
|
hashes: response.authorized,
|
|
3177
3339
|
from,
|
|
3178
3340
|
response: message,
|
|
3179
|
-
signal
|
|
3341
|
+
signal,
|
|
3180
3342
|
});
|
|
3343
|
+
rollbackRatelessAuthorization = signal.aborted;
|
|
3181
3344
|
} catch (error) {
|
|
3182
3345
|
firstError = error;
|
|
3183
3346
|
rollbackRatelessAuthorization = true;
|
|
@@ -3206,6 +3369,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3206
3369
|
leases: simpleLeases,
|
|
3207
3370
|
from,
|
|
3208
3371
|
response: simpleMessage,
|
|
3372
|
+
signal: options?.signal,
|
|
3209
3373
|
});
|
|
3210
3374
|
} catch (error) {
|
|
3211
3375
|
firstError ??= error;
|
|
@@ -3226,7 +3390,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3226
3390
|
}
|
|
3227
3391
|
}
|
|
3228
3392
|
}
|
|
3229
|
-
return this.simple.onMessage(message, context);
|
|
3393
|
+
return this.simple.onMessage(message, context, options);
|
|
3230
3394
|
}
|
|
3231
3395
|
|
|
3232
3396
|
onReceivedEntries(properties: {
|
|
@@ -3303,13 +3467,16 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3303
3467
|
return this.simple.open();
|
|
3304
3468
|
}
|
|
3305
3469
|
|
|
3306
|
-
|
|
3470
|
+
beginClose(): void {
|
|
3307
3471
|
this.ratelessClosed = true;
|
|
3308
|
-
// Abort ownership first. Process abort listeners then cancel any in-flight
|
|
3309
|
-
// StartSync/MoreSymbols send before they detach or free native state.
|
|
3310
3472
|
const reason = new Error("rateless synchronizer closed");
|
|
3311
3473
|
this.cancelRatelessRepairSessions(reason);
|
|
3312
3474
|
this.ratelessDispatchLifecycleController.abort(reason);
|
|
3475
|
+
this.simple.beginClose();
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3478
|
+
close(): Promise<void> | void {
|
|
3479
|
+
this.beginClose();
|
|
3313
3480
|
for (const obj of [...this.ingoingSyncProcesses.values()]) {
|
|
3314
3481
|
obj.free();
|
|
3315
3482
|
}
|