@peerbit/shared-log 12.3.4 → 12.3.5-3f16953
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-envelope.d.ts +18 -0
- package/dist/src/fanout-envelope.d.ts.map +1 -0
- package/dist/src/fanout-envelope.js +85 -0
- package/dist/src/fanout-envelope.js.map +1 -0
- package/dist/src/index.d.ts +41 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1217 -326
- package/dist/src/index.js.map +1 -1
- package/dist/src/pid.d.ts.map +1 -1
- package/dist/src/pid.js +21 -5
- package/dist/src/pid.js.map +1 -1
- package/dist/src/ranges.d.ts.map +1 -1
- package/dist/src/ranges.js +7 -3
- package/dist/src/ranges.js.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +42 -3
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/package.json +20 -20
- package/src/fanout-envelope.ts +27 -0
- package/src/index.ts +1734 -698
- package/src/pid.ts +22 -4
- package/src/ranges.ts +7 -3
- package/src/sync/rateless-iblt.ts +58 -3
package/src/index.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { BorshError, field, variant } from "@dao-xyz/borsh";
|
|
1
|
+
import { BorshError, deserialize, field, serialize, variant } from "@dao-xyz/borsh";
|
|
2
2
|
import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
|
|
3
3
|
import { cidifyString } from "@peerbit/blocks-interface";
|
|
4
4
|
import { Cache } from "@peerbit/cache";
|
|
5
5
|
import {
|
|
6
6
|
AccessError,
|
|
7
7
|
PublicSignKey,
|
|
8
|
+
getPublicKeyFromPeerId,
|
|
8
9
|
sha256Base64Sync,
|
|
9
10
|
sha256Sync,
|
|
10
11
|
} from "@peerbit/crypto";
|
|
@@ -31,7 +32,16 @@ import {
|
|
|
31
32
|
} from "@peerbit/log";
|
|
32
33
|
import { logger as loggerFn } from "@peerbit/logger";
|
|
33
34
|
import { ClosedError, Program, type ProgramEvents } from "@peerbit/program";
|
|
34
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
FanoutChannel,
|
|
37
|
+
type FanoutProviderHandle,
|
|
38
|
+
type FanoutTree,
|
|
39
|
+
type FanoutTreeChannelOptions,
|
|
40
|
+
type FanoutTreeDataEvent,
|
|
41
|
+
type FanoutTreeUnicastEvent,
|
|
42
|
+
type FanoutTreeJoinOptions,
|
|
43
|
+
waitForSubscribers,
|
|
44
|
+
} from "@peerbit/pubsub";
|
|
35
45
|
import {
|
|
36
46
|
SubscriptionEvent,
|
|
37
47
|
UnsubcriptionEvent,
|
|
@@ -40,10 +50,10 @@ import { RPC, type RequestContext } from "@peerbit/rpc";
|
|
|
40
50
|
import {
|
|
41
51
|
AcknowledgeDelivery,
|
|
42
52
|
AnyWhere,
|
|
53
|
+
DataMessage,
|
|
54
|
+
MessageHeader,
|
|
43
55
|
NotStartedError,
|
|
44
|
-
SeekDelivery,
|
|
45
56
|
SilentDelivery,
|
|
46
|
-
type WithMode,
|
|
47
57
|
} from "@peerbit/stream-interface";
|
|
48
58
|
import {
|
|
49
59
|
AbortError,
|
|
@@ -69,6 +79,7 @@ import {
|
|
|
69
79
|
ResponseIPrune,
|
|
70
80
|
createExchangeHeadsMessages,
|
|
71
81
|
} from "./exchange-heads.js";
|
|
82
|
+
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
72
83
|
import {
|
|
73
84
|
MAX_U32,
|
|
74
85
|
MAX_U64,
|
|
@@ -189,6 +200,36 @@ const getLatestEntry = (
|
|
|
189
200
|
return latest;
|
|
190
201
|
};
|
|
191
202
|
|
|
203
|
+
const hashToSeed32 = (str: string) => {
|
|
204
|
+
// FNV-1a 32-bit, fast and deterministic.
|
|
205
|
+
let hash = 0x811c9dc5;
|
|
206
|
+
for (let i = 0; i < str.length; i++) {
|
|
207
|
+
hash ^= str.charCodeAt(i);
|
|
208
|
+
hash = Math.imul(hash, 0x01000193);
|
|
209
|
+
}
|
|
210
|
+
return hash >>> 0;
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const pickDeterministicSubset = (peers: string[], seed: number, max: number) => {
|
|
214
|
+
if (peers.length <= max) return peers;
|
|
215
|
+
|
|
216
|
+
const subset: string[] = [];
|
|
217
|
+
const used = new Set<string>();
|
|
218
|
+
let x = seed || 1;
|
|
219
|
+
while (subset.length < max) {
|
|
220
|
+
// xorshift32
|
|
221
|
+
x ^= x << 13;
|
|
222
|
+
x ^= x >>> 17;
|
|
223
|
+
x ^= x << 5;
|
|
224
|
+
const peer = peers[(x >>> 0) % peers.length];
|
|
225
|
+
if (!used.has(peer)) {
|
|
226
|
+
used.add(peer);
|
|
227
|
+
subset.push(peer);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return subset;
|
|
231
|
+
};
|
|
232
|
+
|
|
192
233
|
export type ReplicationLimitsOptions =
|
|
193
234
|
| Partial<ReplicationLimits>
|
|
194
235
|
| { min?: number; max?: number };
|
|
@@ -373,6 +414,7 @@ export type SharedLogOptions<
|
|
|
373
414
|
compatibility?: number;
|
|
374
415
|
domain?: ReplicationDomainConstructor<D>;
|
|
375
416
|
eagerBlocks?: boolean | { cacheSize?: number };
|
|
417
|
+
fanout?: SharedLogFanoutOptions;
|
|
376
418
|
};
|
|
377
419
|
|
|
378
420
|
export const DEFAULT_MIN_REPLICAS = 2;
|
|
@@ -385,6 +427,10 @@ export const WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS = 3;
|
|
|
385
427
|
// Prefer making pruning robust without timing-based heuristics.
|
|
386
428
|
export const WAIT_FOR_PRUNE_DELAY = 0;
|
|
387
429
|
const PRUNE_DEBOUNCE_INTERVAL = 500;
|
|
430
|
+
const CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS = 250;
|
|
431
|
+
const CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS = 5_000;
|
|
432
|
+
const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
|
|
433
|
+
const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
|
|
388
434
|
|
|
389
435
|
// DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
|
|
390
436
|
const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
|
|
@@ -395,6 +441,17 @@ const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
|
|
|
395
441
|
|
|
396
442
|
const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
|
|
397
443
|
|
|
444
|
+
const DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS: Omit<
|
|
445
|
+
FanoutTreeChannelOptions,
|
|
446
|
+
"role"
|
|
447
|
+
> = {
|
|
448
|
+
msgRate: 30,
|
|
449
|
+
msgSize: 1024,
|
|
450
|
+
uploadLimitBps: 5_000_000,
|
|
451
|
+
maxChildren: 24,
|
|
452
|
+
repair: true,
|
|
453
|
+
};
|
|
454
|
+
|
|
398
455
|
const getIdForDynamicRange = (publicKey: PublicSignKey) => {
|
|
399
456
|
return sha256Sync(
|
|
400
457
|
concat([publicKey.bytes, new TextEncoder().encode("dynamic")]),
|
|
@@ -424,13 +481,29 @@ export type DeliveryOptions = {
|
|
|
424
481
|
signal?: AbortSignal;
|
|
425
482
|
};
|
|
426
483
|
|
|
427
|
-
export type
|
|
484
|
+
export type SharedLogFanoutOptions = {
|
|
485
|
+
root?: string;
|
|
486
|
+
channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
|
|
487
|
+
join?: FanoutTreeJoinOptions;
|
|
488
|
+
};
|
|
489
|
+
|
|
490
|
+
type SharedAppendBaseOptions<T> = AppendOptions<T> & {
|
|
428
491
|
replicas?: AbsoluteReplicas | number;
|
|
429
492
|
replicate?: boolean;
|
|
430
|
-
target?: "all" | "replicators" | "none";
|
|
431
|
-
delivery?: false | true | DeliveryOptions;
|
|
432
493
|
};
|
|
433
494
|
|
|
495
|
+
export type SharedAppendOptions<T> =
|
|
496
|
+
| (SharedAppendBaseOptions<T> & {
|
|
497
|
+
target?: "replicators" | "none";
|
|
498
|
+
delivery?: false | true | DeliveryOptions;
|
|
499
|
+
})
|
|
500
|
+
| (SharedAppendBaseOptions<T> & {
|
|
501
|
+
// target=all uses the fanout data plane and intentionally does not expose
|
|
502
|
+
// per-recipient settle semantics from RPC delivery options.
|
|
503
|
+
target: "all";
|
|
504
|
+
delivery?: false | undefined;
|
|
505
|
+
});
|
|
506
|
+
|
|
434
507
|
export type ReplicatorJoinEvent = { publicKey: PublicSignKey };
|
|
435
508
|
export type ReplicatorLeaveEvent = { publicKey: PublicSignKey };
|
|
436
509
|
export type ReplicationChangeEvent = { publicKey: PublicSignKey };
|
|
@@ -463,11 +536,12 @@ export class SharedLog<
|
|
|
463
536
|
|
|
464
537
|
private _replicationRangeIndex!: Index<ReplicationRangeIndexable<R>>;
|
|
465
538
|
private _entryCoordinatesIndex!: Index<EntryReplicated<R>>;
|
|
466
|
-
|
|
467
|
-
|
|
539
|
+
private coordinateToHash!: Cache<string>;
|
|
540
|
+
private recentlyRebalanced!: Cache<string>;
|
|
468
541
|
|
|
469
|
-
|
|
470
|
-
|
|
542
|
+
uniqueReplicators!: Set<string>;
|
|
543
|
+
private _replicatorJoinEmitted!: Set<string>;
|
|
544
|
+
private _replicatorsReconciled!: boolean;
|
|
471
545
|
|
|
472
546
|
/* private _totalParticipation!: number; */
|
|
473
547
|
|
|
@@ -476,6 +550,10 @@ export class SharedLog<
|
|
|
476
550
|
|
|
477
551
|
private _onSubscriptionFn!: (arg: any) => any;
|
|
478
552
|
private _onUnsubscriptionFn!: (arg: any) => any;
|
|
553
|
+
private _onFanoutDataFn?: (arg: any) => void;
|
|
554
|
+
private _onFanoutUnicastFn?: (arg: any) => void;
|
|
555
|
+
private _fanoutChannel?: FanoutChannel;
|
|
556
|
+
private _providerHandle?: FanoutProviderHandle;
|
|
479
557
|
|
|
480
558
|
private _isTrustedReplicator?: (
|
|
481
559
|
publicKey: PublicSignKey,
|
|
@@ -519,6 +597,15 @@ export class SharedLog<
|
|
|
519
597
|
>; // map of peerId to timeout
|
|
520
598
|
|
|
521
599
|
private latestReplicationInfoMessage!: Map<string, bigint>;
|
|
600
|
+
// Peers that have unsubscribed from this log's topic. We ignore replication-info
|
|
601
|
+
// messages from them until we see a new subscription, to avoid re-introducing
|
|
602
|
+
// stale membership state during close/unsubscribe races.
|
|
603
|
+
private _replicationInfoBlockedPeers!: Set<string>;
|
|
604
|
+
private _replicationInfoRequestByPeer!: Map<
|
|
605
|
+
string,
|
|
606
|
+
{ attempts: number; timer?: ReturnType<typeof setTimeout> }
|
|
607
|
+
>;
|
|
608
|
+
private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
|
|
522
609
|
|
|
523
610
|
private remoteBlocks!: RemoteBlocks;
|
|
524
611
|
|
|
@@ -552,6 +639,10 @@ export class SharedLog<
|
|
|
552
639
|
|
|
553
640
|
private _requestIPruneSent!: Map<string, Set<string>>; // tracks entry hash to peer hash for requesting I prune messages
|
|
554
641
|
private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
|
|
642
|
+
private _checkedPruneRetries!: Map<
|
|
643
|
+
string,
|
|
644
|
+
{ attempts: number; timer?: ReturnType<typeof setTimeout> }
|
|
645
|
+
>;
|
|
555
646
|
|
|
556
647
|
private replicationChangeDebounceFn!: ReturnType<
|
|
557
648
|
typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
|
|
@@ -597,6 +688,590 @@ export class SharedLog<
|
|
|
597
688
|
return (this.compatibility ?? Number.MAX_VALUE) < 9;
|
|
598
689
|
}
|
|
599
690
|
|
|
691
|
+
private getFanoutChannelOptions(
|
|
692
|
+
options?: SharedLogFanoutOptions,
|
|
693
|
+
): Omit<FanoutTreeChannelOptions, "role"> {
|
|
694
|
+
return {
|
|
695
|
+
...DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS,
|
|
696
|
+
...(options?.channel ?? {}),
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
private async _openFanoutChannel(options?: SharedLogFanoutOptions) {
|
|
701
|
+
this._closeFanoutChannel();
|
|
702
|
+
if (!options) {
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
const fanoutService = (this.node.services as any).fanout;
|
|
707
|
+
if (!fanoutService) {
|
|
708
|
+
throw new Error(
|
|
709
|
+
`Fanout is configured for shared-log topic ${this.topic}, but no fanout service is available on this client`,
|
|
710
|
+
);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
const resolvedRoot =
|
|
714
|
+
options.root ??
|
|
715
|
+
(await (fanoutService as any)?.topicRootControlPlane?.resolveTopicRoot?.(
|
|
716
|
+
this.topic,
|
|
717
|
+
));
|
|
718
|
+
if (!resolvedRoot) {
|
|
719
|
+
throw new Error(
|
|
720
|
+
`Fanout is configured for shared-log topic ${this.topic}, but no fanout root was provided and none could be resolved`,
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
const channel = new FanoutChannel(fanoutService, {
|
|
725
|
+
topic: this.topic,
|
|
726
|
+
root: resolvedRoot,
|
|
727
|
+
});
|
|
728
|
+
this._fanoutChannel = channel;
|
|
729
|
+
|
|
730
|
+
this._onFanoutDataFn =
|
|
731
|
+
this._onFanoutDataFn ||
|
|
732
|
+
((evt: any) => {
|
|
733
|
+
const detail = (evt as CustomEvent<FanoutTreeDataEvent>)?.detail;
|
|
734
|
+
if (!detail) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
void this._onFanoutData(detail).catch((error) => logger.error(error));
|
|
738
|
+
});
|
|
739
|
+
channel.addEventListener("data", this._onFanoutDataFn);
|
|
740
|
+
|
|
741
|
+
this._onFanoutUnicastFn =
|
|
742
|
+
this._onFanoutUnicastFn ||
|
|
743
|
+
((evt: any) => {
|
|
744
|
+
const detail = (evt as CustomEvent<FanoutTreeUnicastEvent>)?.detail;
|
|
745
|
+
if (!detail) {
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
void this._onFanoutUnicast(detail).catch((error) => logger.error(error));
|
|
749
|
+
});
|
|
750
|
+
channel.addEventListener("unicast", this._onFanoutUnicastFn);
|
|
751
|
+
|
|
752
|
+
try {
|
|
753
|
+
const channelOptions = this.getFanoutChannelOptions(options);
|
|
754
|
+
if (resolvedRoot === fanoutService.publicKeyHash) {
|
|
755
|
+
await channel.openAsRoot(channelOptions);
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
await channel.join(channelOptions, options.join);
|
|
759
|
+
} catch (error) {
|
|
760
|
+
this._closeFanoutChannel();
|
|
761
|
+
throw error;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
private _closeFanoutChannel() {
|
|
766
|
+
if (this._fanoutChannel) {
|
|
767
|
+
if (this._onFanoutDataFn) {
|
|
768
|
+
this._fanoutChannel.removeEventListener("data", this._onFanoutDataFn);
|
|
769
|
+
}
|
|
770
|
+
if (this._onFanoutUnicastFn) {
|
|
771
|
+
this._fanoutChannel.removeEventListener(
|
|
772
|
+
"unicast",
|
|
773
|
+
this._onFanoutUnicastFn,
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
this._fanoutChannel.close();
|
|
777
|
+
}
|
|
778
|
+
this._fanoutChannel = undefined;
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
private async _onFanoutData(detail: FanoutTreeDataEvent) {
|
|
782
|
+
let envelope: FanoutEnvelope;
|
|
783
|
+
try {
|
|
784
|
+
envelope = deserialize(detail.payload, FanoutEnvelope);
|
|
785
|
+
} catch (error) {
|
|
786
|
+
if (error instanceof BorshError) {
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
throw error;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
let message: TransportMessage;
|
|
793
|
+
try {
|
|
794
|
+
message = deserialize(envelope.payload, TransportMessage);
|
|
795
|
+
} catch (error) {
|
|
796
|
+
if (error instanceof BorshError) {
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
throw error;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
if (!(message instanceof ExchangeHeadsMessage)) {
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
const from =
|
|
807
|
+
(await this._resolvePublicKeyFromHash(envelope.from)) ??
|
|
808
|
+
({ hashcode: () => envelope.from } as PublicSignKey);
|
|
809
|
+
|
|
810
|
+
const contextMessage = new DataMessage({
|
|
811
|
+
header: new MessageHeader({
|
|
812
|
+
session: 0,
|
|
813
|
+
mode: new AnyWhere(),
|
|
814
|
+
priority: 0,
|
|
815
|
+
}),
|
|
816
|
+
});
|
|
817
|
+
contextMessage.header.timestamp = envelope.timestamp;
|
|
818
|
+
|
|
819
|
+
await this.onMessage(message, {
|
|
820
|
+
from,
|
|
821
|
+
message: contextMessage,
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
private async _onFanoutUnicast(detail: FanoutTreeUnicastEvent) {
|
|
826
|
+
let message: TransportMessage;
|
|
827
|
+
try {
|
|
828
|
+
message = deserialize(detail.payload, TransportMessage);
|
|
829
|
+
} catch (error) {
|
|
830
|
+
if (error instanceof BorshError) {
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
throw error;
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
const fromHash = detail.origin || detail.from;
|
|
837
|
+
const from =
|
|
838
|
+
(await this._resolvePublicKeyFromHash(fromHash)) ??
|
|
839
|
+
({ hashcode: () => fromHash } as PublicSignKey);
|
|
840
|
+
|
|
841
|
+
const contextMessage = new DataMessage({
|
|
842
|
+
header: new MessageHeader({
|
|
843
|
+
session: 0,
|
|
844
|
+
mode: new AnyWhere(),
|
|
845
|
+
priority: 0,
|
|
846
|
+
}),
|
|
847
|
+
});
|
|
848
|
+
contextMessage.header.timestamp = detail.timestamp;
|
|
849
|
+
|
|
850
|
+
await this.onMessage(message, {
|
|
851
|
+
from,
|
|
852
|
+
message: contextMessage,
|
|
853
|
+
});
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
private async _publishExchangeHeadsViaFanout(
|
|
857
|
+
message: ExchangeHeadsMessage<any>,
|
|
858
|
+
): Promise<void> {
|
|
859
|
+
if (!this._fanoutChannel) {
|
|
860
|
+
throw new Error(
|
|
861
|
+
`No fanout channel configured for shared-log topic ${this.topic}`,
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
const envelope = new FanoutEnvelope({
|
|
865
|
+
from: this.node.identity.publicKey.hashcode(),
|
|
866
|
+
timestamp: BigInt(Date.now()),
|
|
867
|
+
payload: serialize(message),
|
|
868
|
+
});
|
|
869
|
+
await this._fanoutChannel.publish(serialize(envelope));
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
private _parseDeliveryOptions(
|
|
873
|
+
deliveryArg: false | true | DeliveryOptions | undefined,
|
|
874
|
+
): {
|
|
875
|
+
delivery?: DeliveryOptions;
|
|
876
|
+
requireRecipients: boolean;
|
|
877
|
+
settleMin?: number;
|
|
878
|
+
wrap?: (promise: Promise<void>) => Promise<void>;
|
|
879
|
+
} {
|
|
880
|
+
const delivery: DeliveryOptions | undefined =
|
|
881
|
+
deliveryArg === undefined || deliveryArg === false
|
|
882
|
+
? undefined
|
|
883
|
+
: deliveryArg === true
|
|
884
|
+
? {}
|
|
885
|
+
: deliveryArg;
|
|
886
|
+
if (!delivery) {
|
|
887
|
+
return {
|
|
888
|
+
delivery: undefined,
|
|
889
|
+
requireRecipients: false,
|
|
890
|
+
settleMin: undefined,
|
|
891
|
+
wrap: undefined,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const deliverySettle = delivery.settle ?? true;
|
|
896
|
+
const deliveryTimeout = delivery.timeout;
|
|
897
|
+
const deliverySignal = delivery.signal;
|
|
898
|
+
const requireRecipients = delivery.requireRecipients === true;
|
|
899
|
+
const settleMin =
|
|
900
|
+
typeof deliverySettle === "object" && Number.isFinite(deliverySettle.min)
|
|
901
|
+
? Math.max(0, Math.floor(deliverySettle.min))
|
|
902
|
+
: undefined;
|
|
903
|
+
|
|
904
|
+
const wrap =
|
|
905
|
+
deliveryTimeout == null && deliverySignal == null
|
|
906
|
+
? undefined
|
|
907
|
+
: (promise: Promise<void>) =>
|
|
908
|
+
new Promise<void>((resolve, reject) => {
|
|
909
|
+
let settled = false;
|
|
910
|
+
let timer: ReturnType<typeof setTimeout> | undefined = undefined;
|
|
911
|
+
const onAbort = () => {
|
|
912
|
+
if (settled) {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
settled = true;
|
|
916
|
+
promise.catch(() => {});
|
|
917
|
+
cleanup();
|
|
918
|
+
reject(new AbortError());
|
|
919
|
+
};
|
|
920
|
+
|
|
921
|
+
const cleanup = () => {
|
|
922
|
+
if (timer != null) {
|
|
923
|
+
clearTimeout(timer);
|
|
924
|
+
timer = undefined;
|
|
925
|
+
}
|
|
926
|
+
deliverySignal?.removeEventListener("abort", onAbort);
|
|
927
|
+
};
|
|
928
|
+
|
|
929
|
+
if (deliverySignal) {
|
|
930
|
+
if (deliverySignal.aborted) {
|
|
931
|
+
onAbort();
|
|
932
|
+
return;
|
|
933
|
+
}
|
|
934
|
+
deliverySignal.addEventListener("abort", onAbort);
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
if (deliveryTimeout != null) {
|
|
938
|
+
timer = setTimeout(() => {
|
|
939
|
+
if (settled) {
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
settled = true;
|
|
943
|
+
promise.catch(() => {});
|
|
944
|
+
cleanup();
|
|
945
|
+
reject(new TimeoutError(`Timeout waiting for delivery`));
|
|
946
|
+
}, deliveryTimeout);
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
promise
|
|
950
|
+
.then(() => {
|
|
951
|
+
if (settled) {
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
settled = true;
|
|
955
|
+
cleanup();
|
|
956
|
+
resolve();
|
|
957
|
+
})
|
|
958
|
+
.catch((error) => {
|
|
959
|
+
if (settled) {
|
|
960
|
+
return;
|
|
961
|
+
}
|
|
962
|
+
settled = true;
|
|
963
|
+
cleanup();
|
|
964
|
+
reject(error);
|
|
965
|
+
});
|
|
966
|
+
});
|
|
967
|
+
|
|
968
|
+
return {
|
|
969
|
+
delivery,
|
|
970
|
+
requireRecipients,
|
|
971
|
+
settleMin,
|
|
972
|
+
wrap,
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
private async _appendDeliverToReplicators(
|
|
977
|
+
entry: Entry<T>,
|
|
978
|
+
minReplicasValue: number,
|
|
979
|
+
leaders: Map<string, any>,
|
|
980
|
+
selfHash: string,
|
|
981
|
+
isLeader: boolean,
|
|
982
|
+
deliveryArg: false | true | DeliveryOptions | undefined,
|
|
983
|
+
) {
|
|
984
|
+
const { delivery, requireRecipients, settleMin, wrap } =
|
|
985
|
+
this._parseDeliveryOptions(deliveryArg);
|
|
986
|
+
const pending: Promise<void>[] = [];
|
|
987
|
+
const track = (promise: Promise<void>) => {
|
|
988
|
+
pending.push(wrap ? wrap(promise) : promise);
|
|
989
|
+
};
|
|
990
|
+
const fanoutUnicastOptions =
|
|
991
|
+
delivery?.timeout != null || delivery?.signal != null
|
|
992
|
+
? { timeoutMs: delivery.timeout, signal: delivery.signal }
|
|
993
|
+
: undefined;
|
|
994
|
+
|
|
995
|
+
for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
|
|
996
|
+
await this._mergeLeadersFromGidReferences(message, minReplicasValue, leaders);
|
|
997
|
+
const leadersForDelivery = delivery ? new Set(leaders.keys()) : undefined;
|
|
998
|
+
|
|
999
|
+
const set = this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
|
|
1000
|
+
const hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
|
|
1001
|
+
if (!hasRemotePeers) {
|
|
1002
|
+
if (requireRecipients) {
|
|
1003
|
+
throw new NoPeersError(this.rpc.topic);
|
|
1004
|
+
}
|
|
1005
|
+
continue;
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
if (!delivery) {
|
|
1009
|
+
this.rpc
|
|
1010
|
+
.send(message, {
|
|
1011
|
+
mode: isLeader
|
|
1012
|
+
? new SilentDelivery({ redundancy: 1, to: set })
|
|
1013
|
+
: new AcknowledgeDelivery({ redundancy: 1, to: set }),
|
|
1014
|
+
})
|
|
1015
|
+
.catch((error) => logger.error(error));
|
|
1016
|
+
continue;
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
const orderedRemoteRecipients: string[] = [];
|
|
1020
|
+
for (const peer of leadersForDelivery!) {
|
|
1021
|
+
if (peer === selfHash) {
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
orderedRemoteRecipients.push(peer);
|
|
1025
|
+
}
|
|
1026
|
+
for (const peer of set) {
|
|
1027
|
+
if (peer === selfHash) {
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
if (leadersForDelivery!.has(peer)) {
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
orderedRemoteRecipients.push(peer);
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
const ackTo: string[] = [];
|
|
1037
|
+
let silentTo: string[] | undefined;
|
|
1038
|
+
// Default delivery semantics: require enough remote ACKs to reach the requested
|
|
1039
|
+
// replication degree (local append counts as 1).
|
|
1040
|
+
const ackLimit =
|
|
1041
|
+
settleMin == null ? Math.max(0, minReplicasValue - 1) : settleMin;
|
|
1042
|
+
|
|
1043
|
+
for (const peer of orderedRemoteRecipients) {
|
|
1044
|
+
if (ackTo.length < ackLimit) {
|
|
1045
|
+
ackTo.push(peer);
|
|
1046
|
+
} else {
|
|
1047
|
+
silentTo ||= [];
|
|
1048
|
+
silentTo.push(peer);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
if (requireRecipients && orderedRemoteRecipients.length === 0) {
|
|
1053
|
+
throw new NoPeersError(this.rpc.topic);
|
|
1054
|
+
}
|
|
1055
|
+
if (requireRecipients && ackTo.length + (silentTo?.length || 0) === 0) {
|
|
1056
|
+
throw new NoPeersError(this.rpc.topic);
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
if (ackTo.length > 0) {
|
|
1060
|
+
const payload = serialize(message);
|
|
1061
|
+
for (const peer of ackTo) {
|
|
1062
|
+
track(
|
|
1063
|
+
(async () => {
|
|
1064
|
+
// Unified decision point:
|
|
1065
|
+
// - If we can prove a cheap direct path (connected or routed), use it.
|
|
1066
|
+
// - Otherwise, fall back to the fanout unicast ACK path (bounded overlay routing).
|
|
1067
|
+
// - If that fails, fall back to pubsub/RPC routing which may flood to discover routes.
|
|
1068
|
+
const pubsub: any = this.node.services.pubsub as any;
|
|
1069
|
+
const canDirectFast =
|
|
1070
|
+
Boolean(pubsub?.peers?.get?.(peer)?.isWritable) ||
|
|
1071
|
+
Boolean(
|
|
1072
|
+
pubsub?.routes?.isReachable?.(
|
|
1073
|
+
pubsub?.publicKeyHash,
|
|
1074
|
+
peer,
|
|
1075
|
+
0,
|
|
1076
|
+
),
|
|
1077
|
+
);
|
|
1078
|
+
|
|
1079
|
+
if (canDirectFast) {
|
|
1080
|
+
await this.rpc.send(message, {
|
|
1081
|
+
mode: new AcknowledgeDelivery({
|
|
1082
|
+
redundancy: 1,
|
|
1083
|
+
to: [peer],
|
|
1084
|
+
}),
|
|
1085
|
+
});
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
if (this._fanoutChannel) {
|
|
1090
|
+
try {
|
|
1091
|
+
await this._fanoutChannel.unicastToAck(
|
|
1092
|
+
peer,
|
|
1093
|
+
payload,
|
|
1094
|
+
fanoutUnicastOptions,
|
|
1095
|
+
);
|
|
1096
|
+
return;
|
|
1097
|
+
} catch {
|
|
1098
|
+
// fall back below
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
await this.rpc.send(message, {
|
|
1102
|
+
mode: new AcknowledgeDelivery({
|
|
1103
|
+
redundancy: 1,
|
|
1104
|
+
to: [peer],
|
|
1105
|
+
}),
|
|
1106
|
+
});
|
|
1107
|
+
})(),
|
|
1108
|
+
);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
if (silentTo?.length) {
|
|
1113
|
+
this.rpc
|
|
1114
|
+
.send(message, {
|
|
1115
|
+
mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
|
|
1116
|
+
})
|
|
1117
|
+
.catch((error) => logger.error(error));
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
if (pending.length > 0) {
|
|
1122
|
+
await Promise.all(pending);
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
private async _mergeLeadersFromGidReferences(
|
|
1127
|
+
message: ExchangeHeadsMessage<any>,
|
|
1128
|
+
minReplicasValue: number,
|
|
1129
|
+
leaders: Map<string, any>,
|
|
1130
|
+
) {
|
|
1131
|
+
const gidReferences = message.heads[0]?.gidRefrences;
|
|
1132
|
+
if (!gidReferences || gidReferences.length === 0) {
|
|
1133
|
+
return;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
for (const gidReference of gidReferences) {
|
|
1137
|
+
const entryFromGid = this.log.entryIndex.getHeads(gidReference, false);
|
|
1138
|
+
for (const gidEntry of await entryFromGid.all()) {
|
|
1139
|
+
let coordinates = await this.getCoordinates(gidEntry);
|
|
1140
|
+
if (coordinates == null) {
|
|
1141
|
+
coordinates = await this.createCoordinates(gidEntry, minReplicasValue);
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
const found = await this._findLeaders(coordinates);
|
|
1145
|
+
for (const [key, value] of found) {
|
|
1146
|
+
leaders.set(key, value);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
private async _appendDeliverToAllFanout(entry: Entry<T>) {
|
|
1153
|
+
for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
|
|
1154
|
+
await this._publishExchangeHeadsViaFanout(message);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
private async _resolvePublicKeyFromHash(
|
|
1159
|
+
hash: string,
|
|
1160
|
+
): Promise<PublicSignKey | undefined> {
|
|
1161
|
+
const fanoutService = (this.node.services as any).fanout;
|
|
1162
|
+
return (
|
|
1163
|
+
fanoutService?.getPublicKey?.(hash) ??
|
|
1164
|
+
this.node.services.pubsub.getPublicKey(hash)
|
|
1165
|
+
);
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
private async _getTopicSubscribers(
|
|
1169
|
+
topic: string,
|
|
1170
|
+
): Promise<PublicSignKey[] | undefined> {
|
|
1171
|
+
const maxPeers = 64;
|
|
1172
|
+
|
|
1173
|
+
// Prefer the bounded peer set we already know from the fanout overlay.
|
|
1174
|
+
if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
|
|
1175
|
+
const hashes = this._fanoutChannel
|
|
1176
|
+
.getPeerHashes({ includeSelf: false })
|
|
1177
|
+
.slice(0, maxPeers);
|
|
1178
|
+
if (hashes.length === 0) return [];
|
|
1179
|
+
|
|
1180
|
+
const keys = await Promise.all(
|
|
1181
|
+
hashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
|
|
1182
|
+
);
|
|
1183
|
+
const uniqueKeys: PublicSignKey[] = [];
|
|
1184
|
+
const seen = new Set<string>();
|
|
1185
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
1186
|
+
for (const key of keys) {
|
|
1187
|
+
if (!key) continue;
|
|
1188
|
+
const hash = key.hashcode();
|
|
1189
|
+
if (hash === selfHash) continue;
|
|
1190
|
+
if (seen.has(hash)) continue;
|
|
1191
|
+
seen.add(hash);
|
|
1192
|
+
uniqueKeys.push(key);
|
|
1193
|
+
}
|
|
1194
|
+
return uniqueKeys;
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
1198
|
+
const hashes: string[] = [];
|
|
1199
|
+
|
|
1200
|
+
// Best-effort provider discovery (bounded). This requires bootstrap trackers.
|
|
1201
|
+
try {
|
|
1202
|
+
const fanoutService = (this.node.services as any).fanout;
|
|
1203
|
+
if (fanoutService?.queryProviders) {
|
|
1204
|
+
const ns = `shared-log|${this.topic}`;
|
|
1205
|
+
const seed = hashToSeed32(topic);
|
|
1206
|
+
const providers: string[] = await fanoutService.queryProviders(ns, {
|
|
1207
|
+
want: maxPeers,
|
|
1208
|
+
seed,
|
|
1209
|
+
});
|
|
1210
|
+
for (const h of providers ?? []) {
|
|
1211
|
+
if (!h || h === selfHash) continue;
|
|
1212
|
+
hashes.push(h);
|
|
1213
|
+
if (hashes.length >= maxPeers) break;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
} catch {
|
|
1217
|
+
// Best-effort only.
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
// Next, use already-connected peer streams (bounded and cheap).
|
|
1221
|
+
const peerMap: Map<string, unknown> | undefined = (this.node.services.pubsub as any)
|
|
1222
|
+
?.peers;
|
|
1223
|
+
if (peerMap?.keys) {
|
|
1224
|
+
for (const h of peerMap.keys()) {
|
|
1225
|
+
if (!h || h === selfHash) continue;
|
|
1226
|
+
hashes.push(h);
|
|
1227
|
+
if (hashes.length >= maxPeers) break;
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
// Finally, fall back to libp2p connections (e.g. bootstrap peers) without requiring
|
|
1232
|
+
// any global topic membership view.
|
|
1233
|
+
if (hashes.length < maxPeers) {
|
|
1234
|
+
const connectionManager = (this.node.services.pubsub as any)?.components
|
|
1235
|
+
?.connectionManager;
|
|
1236
|
+
const connections = connectionManager?.getConnections?.() ?? [];
|
|
1237
|
+
for (const conn of connections) {
|
|
1238
|
+
const peerId = conn?.remotePeer;
|
|
1239
|
+
if (!peerId) continue;
|
|
1240
|
+
try {
|
|
1241
|
+
const h = getPublicKeyFromPeerId(peerId).hashcode();
|
|
1242
|
+
if (!h || h === selfHash) continue;
|
|
1243
|
+
hashes.push(h);
|
|
1244
|
+
if (hashes.length >= maxPeers) break;
|
|
1245
|
+
} catch {
|
|
1246
|
+
// Best-effort only.
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
if (hashes.length === 0) return [];
|
|
1252
|
+
|
|
1253
|
+
const uniqueHashes: string[] = [];
|
|
1254
|
+
const seen = new Set<string>();
|
|
1255
|
+
for (const h of hashes) {
|
|
1256
|
+
if (seen.has(h)) continue;
|
|
1257
|
+
seen.add(h);
|
|
1258
|
+
uniqueHashes.push(h);
|
|
1259
|
+
if (uniqueHashes.length >= maxPeers) break;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
const keys = await Promise.all(
|
|
1263
|
+
uniqueHashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
|
|
1264
|
+
);
|
|
1265
|
+
const uniqueKeys: PublicSignKey[] = [];
|
|
1266
|
+
for (const key of keys) {
|
|
1267
|
+
if (!key) continue;
|
|
1268
|
+
const hash = key.hashcode();
|
|
1269
|
+
if (hash === selfHash) continue;
|
|
1270
|
+
uniqueKeys.push(key);
|
|
1271
|
+
}
|
|
1272
|
+
return uniqueKeys;
|
|
1273
|
+
}
|
|
1274
|
+
|
|
600
1275
|
// @deprecated
|
|
601
1276
|
private async getRole() {
|
|
602
1277
|
const segments = await this.getMyReplicationSegments();
|
|
@@ -1004,8 +1679,9 @@ export class SharedLog<
|
|
|
1004
1679
|
})
|
|
1005
1680
|
.all();
|
|
1006
1681
|
|
|
1007
|
-
|
|
1008
|
-
|
|
1682
|
+
this.uniqueReplicators.delete(keyHash);
|
|
1683
|
+
this._replicatorJoinEmitted.delete(keyHash);
|
|
1684
|
+
await this.replicationIndex.del({ query: { hash: keyHash } });
|
|
1009
1685
|
|
|
1010
1686
|
await this.updateOldestTimestampFromIndex();
|
|
1011
1687
|
|
|
@@ -1030,14 +1706,14 @@ export class SharedLog<
|
|
|
1030
1706
|
}
|
|
1031
1707
|
}
|
|
1032
1708
|
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1709
|
+
const timestamp = BigInt(+new Date());
|
|
1710
|
+
for (const x of deleted) {
|
|
1711
|
+
this.replicationChangeDebounceFn.add({
|
|
1712
|
+
range: x.value,
|
|
1713
|
+
type: "removed",
|
|
1714
|
+
timestamp,
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1041
1717
|
|
|
1042
1718
|
const pendingMaturity = this.pendingMaturity.get(keyHash);
|
|
1043
1719
|
if (pendingMaturity) {
|
|
@@ -1118,9 +1794,10 @@ export class SharedLog<
|
|
|
1118
1794
|
{ query: { hash: from.hashcode() } },
|
|
1119
1795
|
{ shape: { id: true } },
|
|
1120
1796
|
);
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1797
|
+
if ((await otherSegmentsIterator.next(1)).length === 0) {
|
|
1798
|
+
this.uniqueReplicators.delete(from.hashcode());
|
|
1799
|
+
this._replicatorJoinEmitted.delete(from.hashcode());
|
|
1800
|
+
}
|
|
1124
1801
|
await otherSegmentsIterator.close();
|
|
1125
1802
|
|
|
1126
1803
|
await this.updateOldestTimestampFromIndex();
|
|
@@ -1160,6 +1837,7 @@ export class SharedLog<
|
|
|
1160
1837
|
|
|
1161
1838
|
let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
|
|
1162
1839
|
let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
|
|
1840
|
+
let isStoppedReplicating = false;
|
|
1163
1841
|
if (reset) {
|
|
1164
1842
|
deleted = (
|
|
1165
1843
|
await this.replicationIndex
|
|
@@ -1198,6 +1876,7 @@ export class SharedLog<
|
|
|
1198
1876
|
}
|
|
1199
1877
|
|
|
1200
1878
|
isNewReplicator = prevCount === 0 && ranges.length > 0;
|
|
1879
|
+
isStoppedReplicating = prevCount > 0 && ranges.length === 0;
|
|
1201
1880
|
} else {
|
|
1202
1881
|
let batchSize = 100;
|
|
1203
1882
|
let existing: ReplicationRangeIndexable<R>[] = [];
|
|
@@ -1281,7 +1960,16 @@ export class SharedLog<
|
|
|
1281
1960
|
diffs = changes;
|
|
1282
1961
|
}
|
|
1283
1962
|
|
|
1284
|
-
|
|
1963
|
+
const fromHash = from.hashcode();
|
|
1964
|
+
// Track replicator membership transitions synchronously so join/leave events are
|
|
1965
|
+
// idempotent even if we process concurrent reset messages/unsubscribes.
|
|
1966
|
+
const stoppedTransition =
|
|
1967
|
+
ranges.length === 0 ? this.uniqueReplicators.delete(fromHash) : false;
|
|
1968
|
+
if (ranges.length === 0) {
|
|
1969
|
+
this._replicatorJoinEmitted.delete(fromHash);
|
|
1970
|
+
} else {
|
|
1971
|
+
this.uniqueReplicators.add(fromHash);
|
|
1972
|
+
}
|
|
1285
1973
|
|
|
1286
1974
|
let now = +new Date();
|
|
1287
1975
|
let minRoleAge = await this.getDefaultMinRoleAge();
|
|
@@ -1327,13 +2015,13 @@ export class SharedLog<
|
|
|
1327
2015
|
}),
|
|
1328
2016
|
);
|
|
1329
2017
|
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
2018
|
+
if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
|
|
2019
|
+
// TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
|
|
2020
|
+
this.replicationChangeDebounceFn.add({
|
|
2021
|
+
...diff,
|
|
2022
|
+
matured: true,
|
|
2023
|
+
}); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
|
|
2024
|
+
}
|
|
1337
2025
|
pendingRanges.delete(diff.range.idString);
|
|
1338
2026
|
if (pendingRanges.size === 0) {
|
|
1339
2027
|
this.pendingMaturity.delete(diff.range.hash);
|
|
@@ -1379,27 +2067,38 @@ export class SharedLog<
|
|
|
1379
2067
|
}),
|
|
1380
2068
|
);
|
|
1381
2069
|
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
2070
|
+
if (isNewReplicator) {
|
|
2071
|
+
if (!this._replicatorJoinEmitted.has(fromHash)) {
|
|
2072
|
+
this._replicatorJoinEmitted.add(fromHash);
|
|
2073
|
+
this.events.dispatchEvent(
|
|
2074
|
+
new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
|
|
2075
|
+
detail: { publicKey: from },
|
|
2076
|
+
}),
|
|
2077
|
+
);
|
|
2078
|
+
}
|
|
1388
2079
|
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
2080
|
+
if (isAllMature) {
|
|
2081
|
+
this.events.dispatchEvent(
|
|
2082
|
+
new CustomEvent<ReplicatorMatureEvent>("replicator:mature", {
|
|
2083
|
+
detail: { publicKey: from },
|
|
1393
2084
|
}),
|
|
1394
2085
|
);
|
|
1395
2086
|
}
|
|
1396
2087
|
}
|
|
1397
2088
|
|
|
1398
|
-
if (
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
2089
|
+
if (isStoppedReplicating && stoppedTransition) {
|
|
2090
|
+
this.events.dispatchEvent(
|
|
2091
|
+
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
2092
|
+
detail: { publicKey: from },
|
|
2093
|
+
}),
|
|
2094
|
+
);
|
|
2095
|
+
}
|
|
2096
|
+
|
|
2097
|
+
if (rebalance) {
|
|
2098
|
+
for (const diff of diffs) {
|
|
2099
|
+
this.replicationChangeDebounceFn.add(diff);
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
1403
2102
|
|
|
1404
2103
|
if (!from.equals(this.node.identity.publicKey)) {
|
|
1405
2104
|
this.rebalanceParticipationDebounced?.call();
|
|
@@ -1432,6 +2131,20 @@ export class SharedLog<
|
|
|
1432
2131
|
if (change) {
|
|
1433
2132
|
let addedOrReplaced = change.filter((x) => x.type !== "removed");
|
|
1434
2133
|
if (addedOrReplaced.length > 0) {
|
|
2134
|
+
// Provider discovery keep-alive (best-effort). This enables bounded targeted fetches
|
|
2135
|
+
// without relying on any global subscriber list.
|
|
2136
|
+
try {
|
|
2137
|
+
const fanoutService = (this.node.services as any).fanout;
|
|
2138
|
+
if (fanoutService?.provide && !this._providerHandle) {
|
|
2139
|
+
this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
|
|
2140
|
+
ttlMs: 120_000,
|
|
2141
|
+
announceIntervalMs: 60_000,
|
|
2142
|
+
});
|
|
2143
|
+
}
|
|
2144
|
+
} catch {
|
|
2145
|
+
// Best-effort only.
|
|
2146
|
+
}
|
|
2147
|
+
|
|
1435
2148
|
let message:
|
|
1436
2149
|
| AllReplicatingSegmentsMessage
|
|
1437
2150
|
| AddedReplicationSegmentMessage
|
|
@@ -1506,6 +2219,82 @@ export class SharedLog<
|
|
|
1506
2219
|
}
|
|
1507
2220
|
}
|
|
1508
2221
|
|
|
2222
|
+
private clearCheckedPruneRetry(hash: string) {
|
|
2223
|
+
const state = this._checkedPruneRetries.get(hash);
|
|
2224
|
+
if (state?.timer) {
|
|
2225
|
+
clearTimeout(state.timer);
|
|
2226
|
+
}
|
|
2227
|
+
this._checkedPruneRetries.delete(hash);
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2230
|
+
private scheduleCheckedPruneRetry(args: {
|
|
2231
|
+
entry: EntryReplicated<R> | ShallowOrFullEntry<any>;
|
|
2232
|
+
leaders: Map<string, unknown> | Set<string>;
|
|
2233
|
+
}) {
|
|
2234
|
+
if (this.closed) return;
|
|
2235
|
+
if (this._pendingDeletes.has(args.entry.hash)) return;
|
|
2236
|
+
|
|
2237
|
+
const hash = args.entry.hash;
|
|
2238
|
+
const state =
|
|
2239
|
+
this._checkedPruneRetries.get(hash) ?? { attempts: 0 };
|
|
2240
|
+
|
|
2241
|
+
if (state.timer) return;
|
|
2242
|
+
if (state.attempts >= CHECKED_PRUNE_RETRY_MAX_ATTEMPTS) {
|
|
2243
|
+
// Avoid unbounded background retries; a new replication-change event can
|
|
2244
|
+
// always re-enqueue pruning with fresh leader info.
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
const attempt = state.attempts + 1;
|
|
2249
|
+
const jitterMs = Math.floor(Math.random() * 250);
|
|
2250
|
+
const delayMs = Math.min(
|
|
2251
|
+
CHECKED_PRUNE_RETRY_MAX_DELAY_MS,
|
|
2252
|
+
1_000 * 2 ** (attempt - 1) + jitterMs,
|
|
2253
|
+
);
|
|
2254
|
+
|
|
2255
|
+
state.attempts = attempt;
|
|
2256
|
+
state.timer = setTimeout(async () => {
|
|
2257
|
+
const st = this._checkedPruneRetries.get(hash);
|
|
2258
|
+
if (st) st.timer = undefined;
|
|
2259
|
+
if (this.closed) return;
|
|
2260
|
+
if (this._pendingDeletes.has(hash)) return;
|
|
2261
|
+
|
|
2262
|
+
let leadersMap: Map<string, any> | undefined;
|
|
2263
|
+
try {
|
|
2264
|
+
const replicas = decodeReplicas(args.entry).getValue(this);
|
|
2265
|
+
leadersMap = await this.findLeadersFromEntry(args.entry, replicas, {
|
|
2266
|
+
roleAge: 0,
|
|
2267
|
+
});
|
|
2268
|
+
} catch {
|
|
2269
|
+
// Best-effort only.
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
if (!leadersMap || leadersMap.size === 0) {
|
|
2273
|
+
if (args.leaders instanceof Map) {
|
|
2274
|
+
leadersMap = args.leaders as any;
|
|
2275
|
+
} else {
|
|
2276
|
+
leadersMap = new Map<string, any>();
|
|
2277
|
+
for (const k of args.leaders) {
|
|
2278
|
+
leadersMap.set(k, { intersecting: true });
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
try {
|
|
2284
|
+
const leadersForRetry = leadersMap ?? new Map<string, any>();
|
|
2285
|
+
await this.pruneDebouncedFnAddIfNotKeeping({
|
|
2286
|
+
key: hash,
|
|
2287
|
+
// TODO types
|
|
2288
|
+
value: { entry: args.entry as any, leaders: leadersForRetry },
|
|
2289
|
+
});
|
|
2290
|
+
} catch {
|
|
2291
|
+
// Best-effort only; pruning will be re-attempted on future changes.
|
|
2292
|
+
}
|
|
2293
|
+
}, delayMs);
|
|
2294
|
+
state.timer.unref?.();
|
|
2295
|
+
this._checkedPruneRetries.set(hash, state);
|
|
2296
|
+
}
|
|
2297
|
+
|
|
1509
2298
|
async append(
|
|
1510
2299
|
data: T,
|
|
1511
2300
|
options?: SharedAppendOptions<T> | undefined,
|
|
@@ -1571,286 +2360,30 @@ export class SharedLog<
|
|
|
1571
2360
|
if (options?.target !== "none") {
|
|
1572
2361
|
const target = options?.target;
|
|
1573
2362
|
const deliveryArg = options?.delivery;
|
|
1574
|
-
const
|
|
1575
|
-
deliveryArg === undefined || deliveryArg === false
|
|
1576
|
-
? undefined
|
|
1577
|
-
: deliveryArg === true
|
|
1578
|
-
? {}
|
|
1579
|
-
: deliveryArg;
|
|
1580
|
-
|
|
1581
|
-
let requireRecipients = false;
|
|
1582
|
-
let settleMin: number | undefined;
|
|
1583
|
-
let guardDelivery:
|
|
1584
|
-
| ((promise: Promise<void>) => Promise<void>)
|
|
1585
|
-
| undefined = undefined;
|
|
1586
|
-
|
|
1587
|
-
let firstDeliveryPromise: Promise<void> | undefined;
|
|
1588
|
-
let deliveryPromises: Promise<void>[] | undefined;
|
|
1589
|
-
let addDeliveryPromise: ((promise: Promise<void>) => void) | undefined;
|
|
1590
|
-
|
|
1591
|
-
const leadersForDelivery =
|
|
1592
|
-
delivery && (target === "replicators" || !target)
|
|
1593
|
-
? new Set(leaders.keys())
|
|
1594
|
-
: undefined;
|
|
1595
|
-
|
|
1596
|
-
if (delivery) {
|
|
1597
|
-
const deliverySettle = delivery.settle ?? true;
|
|
1598
|
-
const deliveryTimeout = delivery.timeout;
|
|
1599
|
-
const deliverySignal = delivery.signal;
|
|
1600
|
-
requireRecipients = delivery.requireRecipients === true;
|
|
1601
|
-
settleMin =
|
|
1602
|
-
typeof deliverySettle === "object" &&
|
|
1603
|
-
Number.isFinite(deliverySettle.min)
|
|
1604
|
-
? Math.max(0, Math.floor(deliverySettle.min))
|
|
1605
|
-
: undefined;
|
|
1606
|
-
|
|
1607
|
-
guardDelivery =
|
|
1608
|
-
deliveryTimeout == null && deliverySignal == null
|
|
1609
|
-
? undefined
|
|
1610
|
-
: (promise: Promise<void>) =>
|
|
1611
|
-
new Promise<void>((resolve, reject) => {
|
|
1612
|
-
let settled = false;
|
|
1613
|
-
let timer: ReturnType<typeof setTimeout> | undefined =
|
|
1614
|
-
undefined;
|
|
1615
|
-
const onAbort = () => {
|
|
1616
|
-
if (settled) {
|
|
1617
|
-
return;
|
|
1618
|
-
}
|
|
1619
|
-
settled = true;
|
|
1620
|
-
promise.catch(() => {});
|
|
1621
|
-
cleanup();
|
|
1622
|
-
reject(new AbortError());
|
|
1623
|
-
};
|
|
1624
|
-
|
|
1625
|
-
const cleanup = () => {
|
|
1626
|
-
if (timer != null) {
|
|
1627
|
-
clearTimeout(timer);
|
|
1628
|
-
timer = undefined;
|
|
1629
|
-
}
|
|
1630
|
-
deliverySignal?.removeEventListener("abort", onAbort);
|
|
1631
|
-
};
|
|
2363
|
+
const hasDelivery = !(deliveryArg === undefined || deliveryArg === false);
|
|
1632
2364
|
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
}
|
|
1638
|
-
deliverySignal.addEventListener("abort", onAbort);
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
if (deliveryTimeout != null) {
|
|
1642
|
-
timer = setTimeout(() => {
|
|
1643
|
-
if (settled) {
|
|
1644
|
-
return;
|
|
1645
|
-
}
|
|
1646
|
-
settled = true;
|
|
1647
|
-
promise.catch(() => {});
|
|
1648
|
-
cleanup();
|
|
1649
|
-
reject(new TimeoutError(`Timeout waiting for delivery`));
|
|
1650
|
-
}, deliveryTimeout);
|
|
1651
|
-
}
|
|
1652
|
-
|
|
1653
|
-
promise
|
|
1654
|
-
.then(() => {
|
|
1655
|
-
if (settled) {
|
|
1656
|
-
return;
|
|
1657
|
-
}
|
|
1658
|
-
settled = true;
|
|
1659
|
-
cleanup();
|
|
1660
|
-
resolve();
|
|
1661
|
-
})
|
|
1662
|
-
.catch((e) => {
|
|
1663
|
-
if (settled) {
|
|
1664
|
-
return;
|
|
1665
|
-
}
|
|
1666
|
-
settled = true;
|
|
1667
|
-
cleanup();
|
|
1668
|
-
reject(e);
|
|
1669
|
-
});
|
|
1670
|
-
});
|
|
1671
|
-
|
|
1672
|
-
addDeliveryPromise = (promise: Promise<void>) => {
|
|
1673
|
-
if (!firstDeliveryPromise) {
|
|
1674
|
-
firstDeliveryPromise = promise;
|
|
1675
|
-
return;
|
|
1676
|
-
}
|
|
1677
|
-
if (!deliveryPromises) {
|
|
1678
|
-
deliveryPromises = [firstDeliveryPromise, promise];
|
|
1679
|
-
firstDeliveryPromise = undefined;
|
|
1680
|
-
return;
|
|
1681
|
-
}
|
|
1682
|
-
deliveryPromises.push(promise);
|
|
1683
|
-
};
|
|
2365
|
+
if (target === "all" && hasDelivery) {
|
|
2366
|
+
throw new Error(
|
|
2367
|
+
`delivery options are not supported with target="all"; fanout broadcast is fire-and-forward`,
|
|
2368
|
+
);
|
|
1684
2369
|
}
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
if (target === "replicators" || !target) {
|
|
1690
|
-
if (message.heads[0].gidRefrences.length > 0) {
|
|
1691
|
-
for (const ref of message.heads[0].gidRefrences) {
|
|
1692
|
-
const entryFromGid = this.log.entryIndex.getHeads(ref, false);
|
|
1693
|
-
for (const entry of await entryFromGid.all()) {
|
|
1694
|
-
let coordinates = await this.getCoordinates(entry);
|
|
1695
|
-
if (coordinates == null) {
|
|
1696
|
-
coordinates = await this.createCoordinates(
|
|
1697
|
-
entry,
|
|
1698
|
-
minReplicasValue,
|
|
1699
|
-
);
|
|
1700
|
-
// TODO are we every to come here?
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
const result = await this._findLeaders(coordinates);
|
|
1704
|
-
for (const [k, v] of result) {
|
|
1705
|
-
leaders.set(k, v);
|
|
1706
|
-
}
|
|
1707
|
-
}
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
const set = this.addPeersToGidPeerHistory(
|
|
1712
|
-
result.entry.meta.gid,
|
|
1713
|
-
leaders.keys(),
|
|
1714
|
-
);
|
|
1715
|
-
let hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
|
|
1716
|
-
if (!hasRemotePeers) {
|
|
1717
|
-
if (requireRecipients) {
|
|
1718
|
-
throw new NoPeersError(this.rpc.topic);
|
|
1719
|
-
}
|
|
1720
|
-
continue;
|
|
1721
|
-
}
|
|
1722
|
-
|
|
1723
|
-
if (!delivery) {
|
|
1724
|
-
this.rpc
|
|
1725
|
-
.send(message, {
|
|
1726
|
-
mode: isLeader
|
|
1727
|
-
? new SilentDelivery({ redundancy: 1, to: set })
|
|
1728
|
-
: new AcknowledgeDelivery({ redundancy: 1, to: set }),
|
|
1729
|
-
})
|
|
1730
|
-
.catch((e) => logger.error(e));
|
|
1731
|
-
continue;
|
|
1732
|
-
}
|
|
1733
|
-
|
|
1734
|
-
let expectedRemoteRecipientsCount = 0;
|
|
1735
|
-
const ackTo: string[] = [];
|
|
1736
|
-
let silentTo: string[] | undefined;
|
|
1737
|
-
const ackLimit =
|
|
1738
|
-
settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
|
|
1739
|
-
|
|
1740
|
-
// Always settle towards the current expected replicators for this entry,
|
|
1741
|
-
// not the entire gid peer history.
|
|
1742
|
-
for (const peer of leadersForDelivery!) {
|
|
1743
|
-
if (peer === selfHash) {
|
|
1744
|
-
continue;
|
|
1745
|
-
}
|
|
1746
|
-
expectedRemoteRecipientsCount++;
|
|
1747
|
-
if (ackTo.length < ackLimit) {
|
|
1748
|
-
ackTo.push(peer);
|
|
1749
|
-
} else {
|
|
1750
|
-
silentTo ||= [];
|
|
1751
|
-
silentTo.push(peer);
|
|
1752
|
-
}
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
|
-
// Still deliver to known peers for the gid (best-effort), but don't let them
|
|
1756
|
-
// satisfy the settle requirement.
|
|
1757
|
-
for (const peer of set) {
|
|
1758
|
-
if (peer === selfHash) {
|
|
1759
|
-
continue;
|
|
1760
|
-
}
|
|
1761
|
-
if (leadersForDelivery!.has(peer)) {
|
|
1762
|
-
continue;
|
|
1763
|
-
}
|
|
1764
|
-
silentTo ||= [];
|
|
1765
|
-
silentTo.push(peer);
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
if (requireRecipients && expectedRemoteRecipientsCount === 0) {
|
|
1769
|
-
throw new NoPeersError(this.rpc.topic);
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
|
-
if (
|
|
1773
|
-
requireRecipients &&
|
|
1774
|
-
ackTo.length + (silentTo?.length || 0) === 0
|
|
1775
|
-
) {
|
|
1776
|
-
throw new NoPeersError(this.rpc.topic);
|
|
1777
|
-
}
|
|
1778
|
-
|
|
1779
|
-
if (ackTo.length > 0) {
|
|
1780
|
-
const promise = this.rpc.send(message, {
|
|
1781
|
-
mode: new AcknowledgeDelivery({
|
|
1782
|
-
redundancy: 1,
|
|
1783
|
-
to: ackTo,
|
|
1784
|
-
}),
|
|
1785
|
-
});
|
|
1786
|
-
addDeliveryPromise!(
|
|
1787
|
-
guardDelivery ? guardDelivery(promise) : promise,
|
|
1788
|
-
);
|
|
1789
|
-
}
|
|
1790
|
-
|
|
1791
|
-
if (silentTo?.length) {
|
|
1792
|
-
this.rpc
|
|
1793
|
-
.send(message, {
|
|
1794
|
-
mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
|
|
1795
|
-
})
|
|
1796
|
-
.catch((e) => logger.error(e));
|
|
1797
|
-
}
|
|
1798
|
-
} else {
|
|
1799
|
-
if (!delivery) {
|
|
1800
|
-
this.rpc.send(message).catch((e) => logger.error(e));
|
|
1801
|
-
continue;
|
|
1802
|
-
}
|
|
1803
|
-
|
|
1804
|
-
const subscribers = await this.node.services.pubsub.getSubscribers(
|
|
1805
|
-
this.rpc.topic,
|
|
1806
|
-
);
|
|
1807
|
-
|
|
1808
|
-
const ackTo: PublicSignKey[] = [];
|
|
1809
|
-
let silentTo: PublicSignKey[] | undefined;
|
|
1810
|
-
const ackLimit =
|
|
1811
|
-
settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
|
|
1812
|
-
for (const subscriber of subscribers || []) {
|
|
1813
|
-
if (subscriber.hashcode() === selfHash) {
|
|
1814
|
-
continue;
|
|
1815
|
-
}
|
|
1816
|
-
if (ackTo.length < ackLimit) {
|
|
1817
|
-
ackTo.push(subscriber);
|
|
1818
|
-
} else {
|
|
1819
|
-
silentTo ||= [];
|
|
1820
|
-
silentTo.push(subscriber);
|
|
1821
|
-
}
|
|
1822
|
-
}
|
|
1823
|
-
|
|
1824
|
-
if (
|
|
1825
|
-
requireRecipients &&
|
|
1826
|
-
ackTo.length + (silentTo?.length || 0) === 0
|
|
1827
|
-
) {
|
|
1828
|
-
throw new NoPeersError(this.rpc.topic);
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
if (ackTo.length > 0) {
|
|
1832
|
-
const promise = this.rpc.send(message, {
|
|
1833
|
-
mode: new AcknowledgeDelivery({ redundancy: 1, to: ackTo }),
|
|
1834
|
-
});
|
|
1835
|
-
addDeliveryPromise!(
|
|
1836
|
-
guardDelivery ? guardDelivery(promise) : promise,
|
|
1837
|
-
);
|
|
1838
|
-
}
|
|
1839
|
-
|
|
1840
|
-
if (silentTo?.length) {
|
|
1841
|
-
this.rpc
|
|
1842
|
-
.send(message, {
|
|
1843
|
-
mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
|
|
1844
|
-
})
|
|
1845
|
-
.catch((e) => logger.error(e));
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
2370
|
+
if (target === "all" && !this._fanoutChannel) {
|
|
2371
|
+
throw new Error(
|
|
2372
|
+
`No fanout channel configured for shared-log topic ${this.topic}`,
|
|
2373
|
+
);
|
|
1848
2374
|
}
|
|
1849
2375
|
|
|
1850
|
-
if (
|
|
1851
|
-
await
|
|
1852
|
-
} else
|
|
1853
|
-
await
|
|
2376
|
+
if (target === "all") {
|
|
2377
|
+
await this._appendDeliverToAllFanout(result.entry);
|
|
2378
|
+
} else {
|
|
2379
|
+
await this._appendDeliverToReplicators(
|
|
2380
|
+
result.entry,
|
|
2381
|
+
minReplicasValue,
|
|
2382
|
+
leaders,
|
|
2383
|
+
selfHash,
|
|
2384
|
+
isLeader,
|
|
2385
|
+
deliveryArg,
|
|
2386
|
+
);
|
|
1854
2387
|
}
|
|
1855
2388
|
}
|
|
1856
2389
|
|
|
@@ -1891,14 +2424,18 @@ export class SharedLog<
|
|
|
1891
2424
|
this.domain.resolution,
|
|
1892
2425
|
);
|
|
1893
2426
|
this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
2427
|
+
this._pendingDeletes = new Map();
|
|
2428
|
+
this._pendingIHave = new Map();
|
|
2429
|
+
this.latestReplicationInfoMessage = new Map();
|
|
2430
|
+
this._replicationInfoBlockedPeers = new Set();
|
|
2431
|
+
this._replicationInfoRequestByPeer = new Map();
|
|
2432
|
+
this._replicationInfoApplyQueueByPeer = new Map();
|
|
2433
|
+
this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
|
|
2434
|
+
this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
|
|
2435
|
+
|
|
2436
|
+
this.uniqueReplicators = new Set();
|
|
2437
|
+
this._replicatorJoinEmitted = new Set();
|
|
2438
|
+
this._replicatorsReconciled = false;
|
|
1902
2439
|
|
|
1903
2440
|
this.openTime = +new Date();
|
|
1904
2441
|
this.oldestOpenTime = this.openTime;
|
|
@@ -1935,6 +2472,13 @@ export class SharedLog<
|
|
|
1935
2472
|
}
|
|
1936
2473
|
|
|
1937
2474
|
this._closeController = new AbortController();
|
|
2475
|
+
this._closeController.signal.addEventListener("abort", () => {
|
|
2476
|
+
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
2477
|
+
if (state.timer) clearTimeout(state.timer);
|
|
2478
|
+
}
|
|
2479
|
+
this._replicationInfoRequestByPeer.clear();
|
|
2480
|
+
});
|
|
2481
|
+
|
|
1938
2482
|
this._isTrustedReplicator = options?.canReplicate;
|
|
1939
2483
|
this.keep = options?.keep;
|
|
1940
2484
|
this.pendingMaturity = new Map();
|
|
@@ -1942,19 +2486,56 @@ export class SharedLog<
|
|
|
1942
2486
|
const id = sha256Base64Sync(this.log.id);
|
|
1943
2487
|
const storage = await this.node.storage.sublevel(id);
|
|
1944
2488
|
|
|
1945
|
-
const localBlocks = await new AnyBlockStore(
|
|
1946
|
-
|
|
1947
|
-
)
|
|
2489
|
+
const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
|
|
2490
|
+
const fanoutService = (this.node.services as any).fanout as FanoutTree | undefined;
|
|
2491
|
+
const blockProviderNamespace = (cid: string) => `cid:${cid}`;
|
|
1948
2492
|
this.remoteBlocks = new RemoteBlocks({
|
|
1949
2493
|
local: localBlocks,
|
|
1950
|
-
publish: (message, options) =>
|
|
1951
|
-
this.rpc.send(
|
|
1952
|
-
new BlocksMessage(message),
|
|
1953
|
-
(options as WithMode).mode instanceof AnyWhere ? undefined : options,
|
|
1954
|
-
),
|
|
2494
|
+
publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
|
|
1955
2495
|
waitFor: this.rpc.waitFor.bind(this.rpc),
|
|
1956
2496
|
publicKey: this.node.identity.publicKey,
|
|
1957
2497
|
eagerBlocks: options?.eagerBlocks ?? true,
|
|
2498
|
+
resolveProviders: async (cid, opts) => {
|
|
2499
|
+
// 1) tracker-backed provider directory (best-effort, bounded)
|
|
2500
|
+
try {
|
|
2501
|
+
const providers = await fanoutService?.queryProviders(
|
|
2502
|
+
blockProviderNamespace(cid),
|
|
2503
|
+
{
|
|
2504
|
+
want: 8,
|
|
2505
|
+
timeoutMs: 2_000,
|
|
2506
|
+
queryTimeoutMs: 500,
|
|
2507
|
+
bootstrapMaxPeers: 2,
|
|
2508
|
+
signal: opts?.signal,
|
|
2509
|
+
},
|
|
2510
|
+
);
|
|
2511
|
+
if (providers && providers.length > 0) return providers;
|
|
2512
|
+
} catch {
|
|
2513
|
+
// ignore discovery failures
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
// 2) fallback to currently connected RPC peers
|
|
2517
|
+
const self = this.node.identity.publicKey.hashcode();
|
|
2518
|
+
const out: string[] = [];
|
|
2519
|
+
const peers = (this.rpc as any)?.peers;
|
|
2520
|
+
for (const h of peers?.keys?.() ?? []) {
|
|
2521
|
+
if (h === self) continue;
|
|
2522
|
+
if (out.includes(h)) continue;
|
|
2523
|
+
out.push(h);
|
|
2524
|
+
if (out.length >= 32) break;
|
|
2525
|
+
}
|
|
2526
|
+
return out;
|
|
2527
|
+
},
|
|
2528
|
+
onPut: async (cid) => {
|
|
2529
|
+
// Best-effort directory announce for "get without remote.from" workflows.
|
|
2530
|
+
try {
|
|
2531
|
+
await fanoutService?.announceProvider(blockProviderNamespace(cid), {
|
|
2532
|
+
ttlMs: 120_000,
|
|
2533
|
+
bootstrapMaxPeers: 2,
|
|
2534
|
+
});
|
|
2535
|
+
} catch {
|
|
2536
|
+
// ignore announce failures
|
|
2537
|
+
}
|
|
2538
|
+
},
|
|
1958
2539
|
});
|
|
1959
2540
|
|
|
1960
2541
|
await this.remoteBlocks.start();
|
|
@@ -1981,9 +2562,10 @@ export class SharedLog<
|
|
|
1981
2562
|
],
|
|
1982
2563
|
})) > 0;
|
|
1983
2564
|
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
2565
|
+
this._gidPeersHistory = new Map();
|
|
2566
|
+
this._requestIPruneSent = new Map();
|
|
2567
|
+
this._requestIPruneResponseReplicatorSet = new Map();
|
|
2568
|
+
this._checkedPruneRetries = new Map();
|
|
1987
2569
|
|
|
1988
2570
|
this.replicationChangeDebounceFn = debounceAggregationChanges<
|
|
1989
2571
|
ReplicationRangeIndexable<R>
|
|
@@ -2068,6 +2650,87 @@ export class SharedLog<
|
|
|
2068
2650
|
|
|
2069
2651
|
await this.log.open(this.remoteBlocks, this.node.identity, {
|
|
2070
2652
|
keychain: this.node.services.keychain,
|
|
2653
|
+
resolveRemotePeers: async (hash, options) => {
|
|
2654
|
+
if (options?.signal?.aborted) return undefined;
|
|
2655
|
+
|
|
2656
|
+
const maxPeers = 8;
|
|
2657
|
+
const self = this.node.identity.publicKey.hashcode();
|
|
2658
|
+
const seed = hashToSeed32(hash);
|
|
2659
|
+
|
|
2660
|
+
// Best hint: peers that have recently confirmed having this entry hash.
|
|
2661
|
+
const hinted = this._requestIPruneResponseReplicatorSet.get(hash);
|
|
2662
|
+
if (hinted && hinted.size > 0) {
|
|
2663
|
+
const peers = [...hinted].filter((p) => p !== self);
|
|
2664
|
+
return peers.length > 0
|
|
2665
|
+
? pickDeterministicSubset(peers, seed, maxPeers)
|
|
2666
|
+
: undefined;
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
// Next: peers we already contacted about this hash (may still have it).
|
|
2670
|
+
const contacted = this._requestIPruneSent.get(hash);
|
|
2671
|
+
if (contacted && contacted.size > 0) {
|
|
2672
|
+
const peers = [...contacted].filter((p) => p !== self);
|
|
2673
|
+
return peers.length > 0
|
|
2674
|
+
? pickDeterministicSubset(peers, seed, maxPeers)
|
|
2675
|
+
: undefined;
|
|
2676
|
+
}
|
|
2677
|
+
|
|
2678
|
+
let candidates: string[] | undefined;
|
|
2679
|
+
|
|
2680
|
+
// Prefer the replicator cache; fall back to subscribers if we have no other signal.
|
|
2681
|
+
const replicatorCandidates = [...this.uniqueReplicators].filter(
|
|
2682
|
+
(p) => p !== self,
|
|
2683
|
+
);
|
|
2684
|
+
if (replicatorCandidates.length > 0) {
|
|
2685
|
+
candidates = replicatorCandidates;
|
|
2686
|
+
} else {
|
|
2687
|
+
try {
|
|
2688
|
+
const subscribers = await this._getTopicSubscribers(this.topic);
|
|
2689
|
+
const subscriberCandidates =
|
|
2690
|
+
subscribers?.map((k) => k.hashcode()).filter((p) => p !== self) ??
|
|
2691
|
+
[];
|
|
2692
|
+
candidates =
|
|
2693
|
+
subscriberCandidates.length > 0 ? subscriberCandidates : undefined;
|
|
2694
|
+
} catch {
|
|
2695
|
+
// Best-effort only.
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
if (!candidates || candidates.length === 0) {
|
|
2699
|
+
// Last resort: peers we are already directly connected to. This avoids
|
|
2700
|
+
// depending on global membership knowledge in early-join scenarios.
|
|
2701
|
+
const peerMap = (this.node.services.pubsub as any)?.peers;
|
|
2702
|
+
if (peerMap?.keys) {
|
|
2703
|
+
candidates = [...peerMap.keys()];
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
if (!candidates || candidates.length === 0) {
|
|
2708
|
+
// Even if the pubsub stream has no established peer streams yet, we may
|
|
2709
|
+
// still have a libp2p connection to one or more peers (e.g. bootstrap).
|
|
2710
|
+
const connectionManager = (this.node.services.pubsub as any)?.components
|
|
2711
|
+
?.connectionManager;
|
|
2712
|
+
const connections = connectionManager?.getConnections?.() ?? [];
|
|
2713
|
+
const connectionHashes: string[] = [];
|
|
2714
|
+
for (const conn of connections) {
|
|
2715
|
+
const peerId = conn?.remotePeer;
|
|
2716
|
+
if (!peerId) continue;
|
|
2717
|
+
try {
|
|
2718
|
+
connectionHashes.push(getPublicKeyFromPeerId(peerId).hashcode());
|
|
2719
|
+
} catch {
|
|
2720
|
+
// Best-effort only.
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
if (connectionHashes.length > 0) {
|
|
2724
|
+
candidates = connectionHashes;
|
|
2725
|
+
}
|
|
2726
|
+
}
|
|
2727
|
+
}
|
|
2728
|
+
|
|
2729
|
+
if (!candidates || candidates.length === 0) return undefined;
|
|
2730
|
+
const peers = candidates.filter((p) => p !== self);
|
|
2731
|
+
if (peers.length === 0) return undefined;
|
|
2732
|
+
return pickDeterministicSubset(peers, seed, maxPeers);
|
|
2733
|
+
},
|
|
2071
2734
|
...this._logProperties,
|
|
2072
2735
|
onChange: async (change) => {
|
|
2073
2736
|
await this.onChange(change);
|
|
@@ -2148,6 +2811,7 @@ export class SharedLog<
|
|
|
2148
2811
|
);
|
|
2149
2812
|
|
|
2150
2813
|
await this.rpc.subscribe();
|
|
2814
|
+
await this._openFanoutChannel(options?.fanout);
|
|
2151
2815
|
|
|
2152
2816
|
// mark all our replicaiton ranges as "new", this would allow other peers to understand that we recently reopend our database and might need some sync and warmup
|
|
2153
2817
|
await this.updateTimestampOfOwnedReplicationRanges(); // TODO do we need to do this before subscribing?
|
|
@@ -2234,17 +2898,15 @@ export class SharedLog<
|
|
|
2234
2898
|
await this.rebalanceParticipation();
|
|
2235
2899
|
|
|
2236
2900
|
// Take into account existing subscription
|
|
2237
|
-
(await this.
|
|
2238
|
-
(v
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
},
|
|
2247
|
-
);
|
|
2901
|
+
(await this._getTopicSubscribers(this.topic))?.forEach((v) => {
|
|
2902
|
+
if (v.equals(this.node.identity.publicKey)) {
|
|
2903
|
+
return;
|
|
2904
|
+
}
|
|
2905
|
+
if (this.closed) {
|
|
2906
|
+
return;
|
|
2907
|
+
}
|
|
2908
|
+
this.handleSubscriptionChange(v, [this.topic], true);
|
|
2909
|
+
});
|
|
2248
2910
|
}
|
|
2249
2911
|
|
|
2250
2912
|
async reset() {
|
|
@@ -2278,7 +2940,7 @@ export class SharedLog<
|
|
|
2278
2940
|
})
|
|
2279
2941
|
.then(async () => {
|
|
2280
2942
|
// is reachable, announce change events
|
|
2281
|
-
const key = await this.
|
|
2943
|
+
const key = await this._resolvePublicKeyFromHash(
|
|
2282
2944
|
segment.value.hash,
|
|
2283
2945
|
);
|
|
2284
2946
|
if (!key) {
|
|
@@ -2288,22 +2950,26 @@ export class SharedLog<
|
|
|
2288
2950
|
);
|
|
2289
2951
|
}
|
|
2290
2952
|
|
|
2291
|
-
|
|
2953
|
+
const keyHash = key.hashcode();
|
|
2954
|
+
this.uniqueReplicators.add(keyHash);
|
|
2292
2955
|
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2956
|
+
if (!this._replicatorJoinEmitted.has(keyHash)) {
|
|
2957
|
+
this._replicatorJoinEmitted.add(keyHash);
|
|
2958
|
+
this.events.dispatchEvent(
|
|
2959
|
+
new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
|
|
2960
|
+
detail: { publicKey: key },
|
|
2961
|
+
}),
|
|
2962
|
+
);
|
|
2963
|
+
this.events.dispatchEvent(
|
|
2964
|
+
new CustomEvent<ReplicationChangeEvent>(
|
|
2965
|
+
"replication:change",
|
|
2966
|
+
{
|
|
2967
|
+
detail: { publicKey: key },
|
|
2968
|
+
},
|
|
2969
|
+
),
|
|
2970
|
+
);
|
|
2971
|
+
}
|
|
2972
|
+
})
|
|
2307
2973
|
.catch(async (e) => {
|
|
2308
2974
|
if (isNotStartedError(e)) {
|
|
2309
2975
|
return; // TODO test this path
|
|
@@ -2435,48 +3101,59 @@ export class SharedLog<
|
|
|
2435
3101
|
numbers: this.indexableDomain.numbers,
|
|
2436
3102
|
});
|
|
2437
3103
|
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
3104
|
+
// Check abort signal before building result
|
|
3105
|
+
if (options?.signal?.aborted) {
|
|
3106
|
+
return [];
|
|
3107
|
+
}
|
|
2442
3108
|
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
3109
|
+
// add all in flight
|
|
3110
|
+
for (const [key, _] of this.syncronizer.syncInFlight) {
|
|
3111
|
+
set.add(key);
|
|
3112
|
+
}
|
|
2447
3113
|
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
3114
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3115
|
+
|
|
3116
|
+
if (options?.reachableOnly) {
|
|
3117
|
+
const directPeers: Map<string, unknown> | undefined = (this.node.services
|
|
3118
|
+
.pubsub as any)?.peers;
|
|
3119
|
+
|
|
3120
|
+
// Prefer the live pubsub subscriber set when filtering reachability. In some
|
|
3121
|
+
// flows peers can be reachable/active even before (or without) subscriber
|
|
3122
|
+
// state converging, so also consider direct pubsub peers.
|
|
3123
|
+
const subscribers =
|
|
3124
|
+
(await this._getTopicSubscribers(this.topic)) ?? undefined;
|
|
3125
|
+
const subscriberHashcodes = subscribers
|
|
3126
|
+
? new Set(subscribers.map((key) => key.hashcode()))
|
|
2458
3127
|
: undefined;
|
|
2459
3128
|
|
|
3129
|
+
// If reachability is requested but we have no basis for filtering yet
|
|
3130
|
+
// (subscriber snapshot hasn't converged), return the full cover set.
|
|
3131
|
+
// Otherwise, only keep peers we can currently reach.
|
|
3132
|
+
const canFilter =
|
|
3133
|
+
directPeers != null ||
|
|
3134
|
+
(subscriberHashcodes && subscriberHashcodes.size > 0);
|
|
3135
|
+
if (!canFilter) {
|
|
3136
|
+
return [...set];
|
|
3137
|
+
}
|
|
3138
|
+
|
|
2460
3139
|
const reachable: string[] = [];
|
|
2461
|
-
const selfHash = this.node.identity.publicKey.hashcode();
|
|
2462
3140
|
for (const peer of set) {
|
|
2463
3141
|
if (peer === selfHash) {
|
|
2464
3142
|
reachable.push(peer);
|
|
2465
3143
|
continue;
|
|
2466
3144
|
}
|
|
2467
3145
|
if (
|
|
2468
|
-
subscriberHashcodes
|
|
2469
|
-
|
|
2470
|
-
: this.uniqueReplicators.has(peer)
|
|
3146
|
+
(subscriberHashcodes && subscriberHashcodes.has(peer)) ||
|
|
3147
|
+
(directPeers && directPeers.has(peer))
|
|
2471
3148
|
) {
|
|
2472
3149
|
reachable.push(peer);
|
|
2473
3150
|
}
|
|
2474
3151
|
}
|
|
2475
3152
|
return reachable;
|
|
2476
|
-
|
|
3153
|
+
}
|
|
2477
3154
|
|
|
2478
|
-
|
|
2479
|
-
|
|
3155
|
+
return [...set];
|
|
3156
|
+
} catch (error) {
|
|
2480
3157
|
// Handle race conditions where the index gets closed during the operation
|
|
2481
3158
|
if (isNotStartedError(error as Error)) {
|
|
2482
3159
|
return [];
|
|
@@ -2497,6 +3174,13 @@ export class SharedLog<
|
|
|
2497
3174
|
this.pendingMaturity.clear();
|
|
2498
3175
|
|
|
2499
3176
|
this.distributeQueue?.clear();
|
|
3177
|
+
this._closeFanoutChannel();
|
|
3178
|
+
try {
|
|
3179
|
+
this._providerHandle?.close();
|
|
3180
|
+
} catch {
|
|
3181
|
+
// ignore
|
|
3182
|
+
}
|
|
3183
|
+
this._providerHandle = undefined;
|
|
2500
3184
|
this.coordinateToHash.clear();
|
|
2501
3185
|
this.recentlyRebalanced.clear();
|
|
2502
3186
|
this.uniqueReplicators.clear();
|
|
@@ -2518,28 +3202,87 @@ export class SharedLog<
|
|
|
2518
3202
|
v.clear();
|
|
2519
3203
|
v.promise.resolve(); // TODO or reject?
|
|
2520
3204
|
}
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
3205
|
+
for (const [_k, v] of this._pendingIHave) {
|
|
3206
|
+
v.clear();
|
|
3207
|
+
}
|
|
3208
|
+
for (const [_k, v] of this._checkedPruneRetries) {
|
|
3209
|
+
if (v.timer) clearTimeout(v.timer);
|
|
3210
|
+
}
|
|
2524
3211
|
|
|
2525
3212
|
await this.remoteBlocks.stop();
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
3213
|
+
this._pendingDeletes.clear();
|
|
3214
|
+
this._pendingIHave.clear();
|
|
3215
|
+
this._checkedPruneRetries.clear();
|
|
3216
|
+
this.latestReplicationInfoMessage.clear();
|
|
3217
|
+
this._gidPeersHistory.clear();
|
|
3218
|
+
this._requestIPruneSent.clear();
|
|
3219
|
+
this._requestIPruneResponseReplicatorSet.clear();
|
|
3220
|
+
// Cancel any pending debounced timers so they can't fire after we've torn down
|
|
3221
|
+
// indexes/RPC state.
|
|
3222
|
+
this.rebalanceParticipationDebounced?.close();
|
|
3223
|
+
this.replicationChangeDebounceFn?.close?.();
|
|
3224
|
+
this.pruneDebouncedFn?.close?.();
|
|
3225
|
+
this.responseToPruneDebouncedFn?.close?.();
|
|
3226
|
+
this.pruneDebouncedFn = undefined as any;
|
|
3227
|
+
this.rebalanceParticipationDebounced = undefined;
|
|
3228
|
+
this._replicationRangeIndex.stop();
|
|
3229
|
+
this._entryCoordinatesIndex.stop();
|
|
2536
3230
|
this._replicationRangeIndex = undefined as any;
|
|
2537
3231
|
this._entryCoordinatesIndex = undefined as any;
|
|
2538
3232
|
|
|
2539
3233
|
this.cpuUsage?.stop?.();
|
|
2540
3234
|
/* this._totalParticipation = 0; */
|
|
2541
3235
|
}
|
|
2542
|
-
|
|
3236
|
+
async close(from?: Program): Promise<boolean> {
|
|
3237
|
+
// Best-effort: announce that we are going offline before tearing down
|
|
3238
|
+
// RPC/subscription state.
|
|
3239
|
+
//
|
|
3240
|
+
// Important: do not delete our local replication ranges here. Keeping them
|
|
3241
|
+
// allows `replicate: { type: "resume" }` to restore the previous role on
|
|
3242
|
+
// restart. Explicit `unreplicate()` still clears local state.
|
|
3243
|
+
try {
|
|
3244
|
+
if (!this.closed) {
|
|
3245
|
+
// Prevent any late debounced timers (rebalance/prune) from publishing
|
|
3246
|
+
// replication info after we announce "segments: []". These races can leave
|
|
3247
|
+
// stale segments on remotes after rapid open/close cycles.
|
|
3248
|
+
this._isReplicating = false;
|
|
3249
|
+
this._isAdaptiveReplicating = false;
|
|
3250
|
+
this.rebalanceParticipationDebounced?.close();
|
|
3251
|
+
this.replicationChangeDebounceFn?.close?.();
|
|
3252
|
+
this.pruneDebouncedFn?.close?.();
|
|
3253
|
+
this.responseToPruneDebouncedFn?.close?.();
|
|
3254
|
+
|
|
3255
|
+
// Ensure the "I'm leaving" replication reset is actually published before
|
|
3256
|
+
// the RPC child program closes and unsubscribes from its topic. If we fire
|
|
3257
|
+
// and forget here, the publish can race with `super.close()` and get dropped,
|
|
3258
|
+
// leaving stale replication segments on remotes (flaky join/leave tests).
|
|
3259
|
+
// Also ensure close is bounded even when shard overlays are mid-reconcile.
|
|
3260
|
+
const abort = new AbortController();
|
|
3261
|
+
const abortTimer = setTimeout(() => {
|
|
3262
|
+
try {
|
|
3263
|
+
abort.abort(
|
|
3264
|
+
new TimeoutError(
|
|
3265
|
+
"shared-log close replication reset timed out",
|
|
3266
|
+
),
|
|
3267
|
+
);
|
|
3268
|
+
} catch {
|
|
3269
|
+
abort.abort();
|
|
3270
|
+
}
|
|
3271
|
+
}, 2_000);
|
|
3272
|
+
try {
|
|
3273
|
+
await this.rpc
|
|
3274
|
+
.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
|
|
3275
|
+
priority: 1,
|
|
3276
|
+
signal: abort.signal,
|
|
3277
|
+
})
|
|
3278
|
+
.catch(() => {});
|
|
3279
|
+
} finally {
|
|
3280
|
+
clearTimeout(abortTimer);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
} catch {
|
|
3284
|
+
// ignore: close should be resilient even if we were never fully started
|
|
3285
|
+
}
|
|
2543
3286
|
const superClosed = await super.close(from);
|
|
2544
3287
|
if (!superClosed) {
|
|
2545
3288
|
return superClosed;
|
|
@@ -2549,12 +3292,50 @@ export class SharedLog<
|
|
|
2549
3292
|
return true;
|
|
2550
3293
|
}
|
|
2551
3294
|
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
3295
|
+
async drop(from?: Program): Promise<boolean> {
|
|
3296
|
+
// Best-effort: announce that we are going offline before tearing down
|
|
3297
|
+
// RPC/subscription state (same reasoning as in `close()`).
|
|
3298
|
+
try {
|
|
3299
|
+
if (!this.closed) {
|
|
3300
|
+
this._isReplicating = false;
|
|
3301
|
+
this._isAdaptiveReplicating = false;
|
|
3302
|
+
this.rebalanceParticipationDebounced?.close();
|
|
3303
|
+
this.replicationChangeDebounceFn?.close?.();
|
|
3304
|
+
this.pruneDebouncedFn?.close?.();
|
|
3305
|
+
this.responseToPruneDebouncedFn?.close?.();
|
|
3306
|
+
|
|
3307
|
+
const abort = new AbortController();
|
|
3308
|
+
const abortTimer = setTimeout(() => {
|
|
3309
|
+
try {
|
|
3310
|
+
abort.abort(
|
|
3311
|
+
new TimeoutError(
|
|
3312
|
+
"shared-log drop replication reset timed out",
|
|
3313
|
+
),
|
|
3314
|
+
);
|
|
3315
|
+
} catch {
|
|
3316
|
+
abort.abort();
|
|
3317
|
+
}
|
|
3318
|
+
}, 2_000);
|
|
3319
|
+
try {
|
|
3320
|
+
await this.rpc
|
|
3321
|
+
.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
|
|
3322
|
+
priority: 1,
|
|
3323
|
+
signal: abort.signal,
|
|
3324
|
+
})
|
|
3325
|
+
.catch(() => {});
|
|
3326
|
+
} finally {
|
|
3327
|
+
clearTimeout(abortTimer);
|
|
3328
|
+
}
|
|
3329
|
+
}
|
|
3330
|
+
} catch {
|
|
3331
|
+
// ignore: drop should be resilient even if we were never fully started
|
|
3332
|
+
}
|
|
3333
|
+
|
|
3334
|
+
const superDropped = await super.drop(from);
|
|
3335
|
+
if (!superDropped) {
|
|
3336
|
+
return superDropped;
|
|
3337
|
+
}
|
|
3338
|
+
await this._entryCoordinatesIndex.drop();
|
|
2558
3339
|
await this._replicationRangeIndex.drop();
|
|
2559
3340
|
await this.log.drop();
|
|
2560
3341
|
await this._close();
|
|
@@ -2921,20 +3702,20 @@ export class SharedLog<
|
|
|
2921
3702
|
return;
|
|
2922
3703
|
}
|
|
2923
3704
|
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
3705
|
+
const segments = (await this.getMyReplicationSegments()).map((x) =>
|
|
3706
|
+
x.toReplicationRange(),
|
|
3707
|
+
);
|
|
2927
3708
|
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
3709
|
+
this.rpc
|
|
3710
|
+
.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
3711
|
+
mode: new AcknowledgeDelivery({ to: [context.from], redundancy: 1 }),
|
|
3712
|
+
})
|
|
3713
|
+
.catch((e) => logger.error(e.toString()));
|
|
2933
3714
|
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
3715
|
+
// for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
|
|
3716
|
+
if (this.v8Behaviour) {
|
|
3717
|
+
const role = this.getRole();
|
|
3718
|
+
if (role instanceof Replicator) {
|
|
2938
3719
|
const fixedSettings = !this._isAdaptiveReplicating;
|
|
2939
3720
|
if (fixedSettings) {
|
|
2940
3721
|
await this.rpc.send(
|
|
@@ -2959,71 +3740,91 @@ export class SharedLog<
|
|
|
2959
3740
|
return;
|
|
2960
3741
|
}
|
|
2961
3742
|
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
const prev = this.latestReplicationInfoMessage.get(from.hashcode());
|
|
2973
|
-
if (prev && prev > messageTimestamp) {
|
|
3743
|
+
const replicationInfoMessage = msg as
|
|
3744
|
+
| AllReplicatingSegmentsMessage
|
|
3745
|
+
| AddedReplicationSegmentMessage;
|
|
3746
|
+
|
|
3747
|
+
// Process replication updates even if the sender isn't yet considered "ready" by
|
|
3748
|
+
// `Program.waitFor()`. Dropping these messages can lead to missing replicator info
|
|
3749
|
+
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
3750
|
+
const from = context.from!;
|
|
3751
|
+
const fromHash = from.hashcode();
|
|
3752
|
+
if (this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
2974
3753
|
return;
|
|
2975
3754
|
}
|
|
3755
|
+
const messageTimestamp = context.message.header.timestamp;
|
|
3756
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
3757
|
+
try {
|
|
3758
|
+
// The peer may have unsubscribed after this message was queued.
|
|
3759
|
+
if (this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
3760
|
+
return;
|
|
3761
|
+
}
|
|
2976
3762
|
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
3763
|
+
// Process in-order to avoid races where repeated reset messages arrive
|
|
3764
|
+
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
3765
|
+
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
3766
|
+
if (prev && prev > messageTimestamp) {
|
|
3767
|
+
return;
|
|
3768
|
+
}
|
|
2981
3769
|
|
|
2982
|
-
|
|
2983
|
-
return;
|
|
2984
|
-
}
|
|
3770
|
+
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
2985
3771
|
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
x.toReplicationRangeIndexable(from),
|
|
2990
|
-
),
|
|
2991
|
-
from,
|
|
2992
|
-
{
|
|
2993
|
-
reset,
|
|
2994
|
-
checkDuplicates: true,
|
|
2995
|
-
timestamp: Number(messageTimestamp),
|
|
2996
|
-
},
|
|
2997
|
-
);
|
|
2998
|
-
})().catch((e) => {
|
|
2999
|
-
if (isNotStartedError(e)) {
|
|
3000
|
-
return;
|
|
3001
|
-
}
|
|
3002
|
-
logger.error(
|
|
3003
|
-
`Failed to apply replication settings from '${from.hashcode()}': ${
|
|
3004
|
-
e?.message ?? e
|
|
3005
|
-
}`,
|
|
3006
|
-
);
|
|
3007
|
-
});
|
|
3008
|
-
} else if (msg instanceof StoppedReplicating) {
|
|
3009
|
-
if (context.from.equals(this.node.identity.publicKey)) {
|
|
3010
|
-
return;
|
|
3011
|
-
}
|
|
3772
|
+
if (this.closed) {
|
|
3773
|
+
return;
|
|
3774
|
+
}
|
|
3012
3775
|
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3776
|
+
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
3777
|
+
await this.addReplicationRange(
|
|
3778
|
+
replicationInfoMessage.segments.map((x) =>
|
|
3779
|
+
x.toReplicationRangeIndexable(from),
|
|
3780
|
+
),
|
|
3781
|
+
from,
|
|
3782
|
+
{
|
|
3783
|
+
reset,
|
|
3784
|
+
checkDuplicates: true,
|
|
3785
|
+
timestamp: Number(messageTimestamp),
|
|
3786
|
+
},
|
|
3787
|
+
);
|
|
3017
3788
|
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3789
|
+
// If the peer reports any replication segments, stop re-requesting.
|
|
3790
|
+
// (Empty reports can be transient during startup.)
|
|
3791
|
+
if (replicationInfoMessage.segments.length > 0) {
|
|
3792
|
+
this.cancelReplicationInfoRequests(fromHash);
|
|
3793
|
+
}
|
|
3794
|
+
} catch (e) {
|
|
3795
|
+
if (isNotStartedError(e as Error)) {
|
|
3796
|
+
return;
|
|
3797
|
+
}
|
|
3798
|
+
logger.error(
|
|
3799
|
+
`Failed to apply replication settings from '${fromHash}': ${
|
|
3800
|
+
(e as any)?.message ?? e
|
|
3801
|
+
}`,
|
|
3802
|
+
);
|
|
3803
|
+
}
|
|
3025
3804
|
});
|
|
3026
|
-
|
|
3805
|
+
} else if (msg instanceof StoppedReplicating) {
|
|
3806
|
+
if (context.from.equals(this.node.identity.publicKey)) {
|
|
3807
|
+
return;
|
|
3808
|
+
}
|
|
3809
|
+
const fromHash = context.from.hashcode();
|
|
3810
|
+
if (this._replicationInfoBlockedPeers.has(fromHash)) {
|
|
3811
|
+
return;
|
|
3812
|
+
}
|
|
3813
|
+
|
|
3814
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
3815
|
+
msg.segmentIds,
|
|
3816
|
+
context.from,
|
|
3817
|
+
);
|
|
3818
|
+
|
|
3819
|
+
await this.removeReplicationRanges(rangesToRemove, context.from);
|
|
3820
|
+
const timestamp = BigInt(+new Date());
|
|
3821
|
+
for (const range of rangesToRemove) {
|
|
3822
|
+
this.replicationChangeDebounceFn.add({
|
|
3823
|
+
range,
|
|
3824
|
+
type: "removed",
|
|
3825
|
+
timestamp,
|
|
3826
|
+
});
|
|
3827
|
+
}
|
|
3027
3828
|
} else {
|
|
3028
3829
|
throw new Error("Unexpected message");
|
|
3029
3830
|
}
|
|
@@ -3325,10 +4126,10 @@ export class SharedLog<
|
|
|
3325
4126
|
}
|
|
3326
4127
|
}
|
|
3327
4128
|
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
4129
|
+
async waitForReplicator(
|
|
4130
|
+
key: PublicSignKey,
|
|
4131
|
+
options?: {
|
|
4132
|
+
signal?: AbortSignal;
|
|
3332
4133
|
eager?: boolean;
|
|
3333
4134
|
roleAge?: number;
|
|
3334
4135
|
timeout?: number;
|
|
@@ -3340,9 +4141,9 @@ export class SharedLog<
|
|
|
3340
4141
|
? undefined
|
|
3341
4142
|
: (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
|
|
3342
4143
|
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
4144
|
+
let settled = false;
|
|
4145
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
4146
|
+
let requestTimer: ReturnType<typeof setTimeout> | undefined;
|
|
3346
4147
|
|
|
3347
4148
|
const clear = () => {
|
|
3348
4149
|
this.events.removeEventListener("replicator:mature", check);
|
|
@@ -3358,14 +4159,19 @@ export class SharedLog<
|
|
|
3358
4159
|
}
|
|
3359
4160
|
};
|
|
3360
4161
|
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
4162
|
+
const resolve = async () => {
|
|
4163
|
+
if (settled) {
|
|
4164
|
+
return;
|
|
4165
|
+
}
|
|
4166
|
+
settled = true;
|
|
4167
|
+
clear();
|
|
4168
|
+
// `waitForReplicator()` is typically used as a precondition before join/replicate
|
|
4169
|
+
// flows. A replicator can become mature and enqueue a debounced rebalance
|
|
4170
|
+
// (`replicationChangeDebounceFn`) slightly later. Flush here so callers don't
|
|
4171
|
+
// observe a "late" rebalance after the wait resolves.
|
|
4172
|
+
await this.replicationChangeDebounceFn?.flush?.();
|
|
4173
|
+
deferred.resolve();
|
|
4174
|
+
};
|
|
3369
4175
|
|
|
3370
4176
|
const reject = (error: Error) => {
|
|
3371
4177
|
if (settled) {
|
|
@@ -3409,13 +4215,14 @@ export class SharedLog<
|
|
|
3409
4215
|
|
|
3410
4216
|
this.rpc
|
|
3411
4217
|
.send(new RequestReplicationInfoMessage(), {
|
|
3412
|
-
mode: new
|
|
4218
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [key] }),
|
|
3413
4219
|
})
|
|
3414
4220
|
.catch((e) => {
|
|
3415
4221
|
// Best-effort: missing peers / unopened RPC should not fail the wait logic.
|
|
3416
4222
|
if (isNotStartedError(e as Error)) {
|
|
3417
4223
|
return;
|
|
3418
4224
|
}
|
|
4225
|
+
logger.error(e?.toString?.() ?? String(e));
|
|
3419
4226
|
});
|
|
3420
4227
|
|
|
3421
4228
|
if (requestAttempts < maxRequestAttempts) {
|
|
@@ -3423,29 +4230,29 @@ export class SharedLog<
|
|
|
3423
4230
|
}
|
|
3424
4231
|
};
|
|
3425
4232
|
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
return;
|
|
3436
|
-
}
|
|
3437
|
-
if (!options?.eager && resolvedRoleAge != null) {
|
|
3438
|
-
if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
|
|
4233
|
+
const check = async () => {
|
|
4234
|
+
const iterator = this.replicationIndex?.iterate(
|
|
4235
|
+
{ query: new StringMatch({ key: "hash", value: key.hashcode() }) },
|
|
4236
|
+
{ reference: true },
|
|
4237
|
+
);
|
|
4238
|
+
try {
|
|
4239
|
+
const rects = await iterator?.next(1);
|
|
4240
|
+
const rect = rects?.[0]?.value;
|
|
4241
|
+
if (!rect) {
|
|
3439
4242
|
return;
|
|
3440
4243
|
}
|
|
4244
|
+
if (!options?.eager && resolvedRoleAge != null) {
|
|
4245
|
+
if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
|
|
4246
|
+
return;
|
|
4247
|
+
}
|
|
4248
|
+
}
|
|
4249
|
+
await resolve();
|
|
4250
|
+
} catch (error) {
|
|
4251
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
4252
|
+
} finally {
|
|
4253
|
+
await iterator?.close();
|
|
3441
4254
|
}
|
|
3442
|
-
|
|
3443
|
-
} catch (error) {
|
|
3444
|
-
reject(error instanceof Error ? error : new Error(String(error)));
|
|
3445
|
-
} finally {
|
|
3446
|
-
await iterator?.close();
|
|
3447
|
-
}
|
|
3448
|
-
};
|
|
4255
|
+
};
|
|
3449
4256
|
|
|
3450
4257
|
requestReplicationInfo();
|
|
3451
4258
|
check();
|
|
@@ -3462,59 +4269,77 @@ export class SharedLog<
|
|
|
3462
4269
|
coverageThreshold?: number;
|
|
3463
4270
|
waitForNewPeers?: boolean;
|
|
3464
4271
|
}) {
|
|
3465
|
-
// if no remotes, just return
|
|
3466
|
-
const subscribers = await this.node.services.pubsub.getSubscribers(
|
|
3467
|
-
this.rpc.topic,
|
|
3468
|
-
);
|
|
3469
|
-
let waitForNewPeers = options?.waitForNewPeers;
|
|
3470
|
-
if (!waitForNewPeers && (subscribers?.length ?? 0) === 0) {
|
|
3471
|
-
throw new NoPeersError(this.rpc.topic);
|
|
3472
|
-
}
|
|
3473
|
-
|
|
3474
4272
|
let coverageThreshold = options?.coverageThreshold ?? 1;
|
|
3475
4273
|
let deferred = pDefer<void>();
|
|
4274
|
+
let settled = false;
|
|
3476
4275
|
|
|
3477
4276
|
const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
|
|
3478
4277
|
const providedCustomRoleAge = options?.roleAge != null;
|
|
3479
4278
|
|
|
3480
|
-
|
|
4279
|
+
const resolve = () => {
|
|
4280
|
+
if (settled) return;
|
|
4281
|
+
settled = true;
|
|
4282
|
+
deferred.resolve();
|
|
4283
|
+
};
|
|
4284
|
+
|
|
4285
|
+
const reject = (error: unknown) => {
|
|
4286
|
+
if (settled) return;
|
|
4287
|
+
settled = true;
|
|
4288
|
+
deferred.reject(error);
|
|
4289
|
+
};
|
|
4290
|
+
|
|
4291
|
+
let checkInFlight: Promise<void> | undefined;
|
|
4292
|
+
const checkCoverage = async () => {
|
|
3481
4293
|
const coverage = await this.calculateCoverage({
|
|
3482
4294
|
roleAge,
|
|
3483
4295
|
});
|
|
3484
4296
|
|
|
3485
4297
|
if (coverage >= coverageThreshold) {
|
|
3486
|
-
|
|
4298
|
+
resolve();
|
|
3487
4299
|
return true;
|
|
3488
4300
|
}
|
|
3489
4301
|
return false;
|
|
3490
4302
|
};
|
|
4303
|
+
|
|
4304
|
+
const scheduleCheckCoverage = () => {
|
|
4305
|
+
if (settled || checkInFlight) {
|
|
4306
|
+
return;
|
|
4307
|
+
}
|
|
4308
|
+
|
|
4309
|
+
checkInFlight = checkCoverage()
|
|
4310
|
+
.then(() => {})
|
|
4311
|
+
.catch(reject)
|
|
4312
|
+
.finally(() => {
|
|
4313
|
+
checkInFlight = undefined;
|
|
4314
|
+
});
|
|
4315
|
+
};
|
|
3491
4316
|
const onReplicatorMature = () => {
|
|
3492
|
-
|
|
4317
|
+
scheduleCheckCoverage();
|
|
3493
4318
|
};
|
|
3494
4319
|
const onReplicationChange = () => {
|
|
3495
|
-
|
|
4320
|
+
scheduleCheckCoverage();
|
|
3496
4321
|
};
|
|
3497
4322
|
this.events.addEventListener("replicator:mature", onReplicatorMature);
|
|
3498
4323
|
this.events.addEventListener("replication:change", onReplicationChange);
|
|
3499
|
-
await checkCoverage();
|
|
4324
|
+
await checkCoverage().catch(reject);
|
|
3500
4325
|
|
|
3501
|
-
let
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
4326
|
+
let intervalMs = providedCustomRoleAge ? 100 : 250;
|
|
4327
|
+
let interval =
|
|
4328
|
+
roleAge > 0
|
|
4329
|
+
? setInterval(() => {
|
|
4330
|
+
scheduleCheckCoverage();
|
|
4331
|
+
}, intervalMs)
|
|
4332
|
+
: undefined;
|
|
3506
4333
|
|
|
3507
4334
|
let timeout = options?.timeout ?? this.waitForReplicatorTimeout;
|
|
3508
4335
|
const timer = setTimeout(() => {
|
|
3509
4336
|
clear();
|
|
3510
|
-
|
|
3511
|
-
new TimeoutError(`Timeout waiting for mature replicators`),
|
|
3512
|
-
);
|
|
4337
|
+
reject(new TimeoutError(`Timeout waiting for mature replicators`));
|
|
3513
4338
|
}, timeout);
|
|
3514
4339
|
|
|
3515
4340
|
const abortListener = () => {
|
|
3516
4341
|
clear();
|
|
3517
|
-
|
|
4342
|
+
reject(new AbortError());
|
|
3518
4343
|
};
|
|
3519
4344
|
|
|
3520
4345
|
if (options?.signal) {
|
|
@@ -3708,9 +4533,7 @@ export class SharedLog<
|
|
|
3708
4533
|
let subscribers = 1;
|
|
3709
4534
|
if (!this.rpc.closed) {
|
|
3710
4535
|
try {
|
|
3711
|
-
subscribers =
|
|
3712
|
-
(await this.node.services.pubsub.getSubscribers(this.rpc.topic))
|
|
3713
|
-
?.length ?? 1;
|
|
4536
|
+
subscribers = (await this._getTopicSubscribers(this.rpc.topic))?.length ?? 1;
|
|
3714
4537
|
} catch {
|
|
3715
4538
|
// Best-effort only; fall back to 1.
|
|
3716
4539
|
}
|
|
@@ -3825,22 +4648,29 @@ export class SharedLog<
|
|
|
3825
4648
|
const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
|
|
3826
4649
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3827
4650
|
|
|
3828
|
-
//
|
|
3829
|
-
//
|
|
3830
|
-
//
|
|
4651
|
+
// Prefer `uniqueReplicators` (replicator cache) as soon as it has any data.
|
|
4652
|
+
// Falling back to live pubsub subscribers can include non-replicators and can
|
|
4653
|
+
// break delivery/availability when writers are not directly connected.
|
|
3831
4654
|
let peerFilter: Set<string> | undefined = undefined;
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
4655
|
+
const selfReplicating = await this.isReplicating();
|
|
4656
|
+
if (this.uniqueReplicators.size > 0) {
|
|
4657
|
+
peerFilter = new Set(this.uniqueReplicators);
|
|
4658
|
+
if (selfReplicating) {
|
|
4659
|
+
peerFilter.add(selfHash);
|
|
4660
|
+
} else {
|
|
4661
|
+
peerFilter.delete(selfHash);
|
|
4662
|
+
}
|
|
3836
4663
|
} else {
|
|
3837
4664
|
try {
|
|
3838
4665
|
const subscribers =
|
|
3839
|
-
(await this.
|
|
3840
|
-
undefined;
|
|
4666
|
+
(await this._getTopicSubscribers(this.topic)) ?? undefined;
|
|
3841
4667
|
if (subscribers && subscribers.length > 0) {
|
|
3842
4668
|
peerFilter = new Set(subscribers.map((key) => key.hashcode()));
|
|
3843
|
-
|
|
4669
|
+
if (selfReplicating) {
|
|
4670
|
+
peerFilter.add(selfHash);
|
|
4671
|
+
} else {
|
|
4672
|
+
peerFilter.delete(selfHash);
|
|
4673
|
+
}
|
|
3844
4674
|
}
|
|
3845
4675
|
} catch {
|
|
3846
4676
|
// Best-effort only; if pubsub isn't ready, do a full scan.
|
|
@@ -3886,76 +4716,160 @@ export class SharedLog<
|
|
|
3886
4716
|
);
|
|
3887
4717
|
}
|
|
3888
4718
|
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
4719
|
+
private withReplicationInfoApplyQueue(
|
|
4720
|
+
peerHash: string,
|
|
4721
|
+
fn: () => Promise<void>,
|
|
4722
|
+
): Promise<void> {
|
|
4723
|
+
const prev = this._replicationInfoApplyQueueByPeer.get(peerHash);
|
|
4724
|
+
const next = (prev ?? Promise.resolve())
|
|
4725
|
+
.catch(() => {
|
|
4726
|
+
// Avoid stuck queues if a previous apply failed.
|
|
4727
|
+
})
|
|
4728
|
+
.then(fn);
|
|
4729
|
+
this._replicationInfoApplyQueueByPeer.set(peerHash, next);
|
|
4730
|
+
return next.finally(() => {
|
|
4731
|
+
if (this._replicationInfoApplyQueueByPeer.get(peerHash) === next) {
|
|
4732
|
+
this._replicationInfoApplyQueueByPeer.delete(peerHash);
|
|
4733
|
+
}
|
|
4734
|
+
});
|
|
4735
|
+
}
|
|
4736
|
+
|
|
4737
|
+
private cancelReplicationInfoRequests(peerHash: string) {
|
|
4738
|
+
const state = this._replicationInfoRequestByPeer.get(peerHash);
|
|
4739
|
+
if (!state) return;
|
|
4740
|
+
if (state.timer) {
|
|
4741
|
+
clearTimeout(state.timer);
|
|
4742
|
+
}
|
|
4743
|
+
this._replicationInfoRequestByPeer.delete(peerHash);
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4746
|
+
private scheduleReplicationInfoRequests(peer: PublicSignKey) {
|
|
4747
|
+
const peerHash = peer.hashcode();
|
|
4748
|
+
if (this._replicationInfoRequestByPeer.has(peerHash)) {
|
|
3895
4749
|
return;
|
|
3896
4750
|
}
|
|
3897
4751
|
|
|
3898
|
-
|
|
3899
|
-
|
|
4752
|
+
const state: { attempts: number; timer?: ReturnType<typeof setTimeout> } = {
|
|
4753
|
+
attempts: 0,
|
|
4754
|
+
};
|
|
4755
|
+
this._replicationInfoRequestByPeer.set(peerHash, state);
|
|
3900
4756
|
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
4757
|
+
const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
|
|
4758
|
+
const maxAttempts = Math.min(
|
|
4759
|
+
5,
|
|
4760
|
+
this.waitForReplicatorRequestMaxAttempts ??
|
|
4761
|
+
WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
|
|
4762
|
+
);
|
|
4763
|
+
|
|
4764
|
+
const tick = () => {
|
|
4765
|
+
if (this.closed || this._closeController.signal.aborted) {
|
|
4766
|
+
this.cancelReplicationInfoRequests(peerHash);
|
|
4767
|
+
return;
|
|
3906
4768
|
}
|
|
3907
4769
|
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
4770
|
+
state.attempts++;
|
|
4771
|
+
|
|
4772
|
+
this.rpc
|
|
4773
|
+
.send(new RequestReplicationInfoMessage(), {
|
|
4774
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [peer] }),
|
|
4775
|
+
})
|
|
4776
|
+
.catch((e) => {
|
|
4777
|
+
// Best-effort: missing peers / unopened RPC should not fail join flows.
|
|
4778
|
+
if (isNotStartedError(e as Error)) {
|
|
4779
|
+
return;
|
|
4780
|
+
}
|
|
4781
|
+
logger.error(e?.toString?.() ?? String(e));
|
|
4782
|
+
});
|
|
4783
|
+
|
|
4784
|
+
if (state.attempts >= maxAttempts) {
|
|
4785
|
+
this.cancelReplicationInfoRequests(peerHash);
|
|
4786
|
+
return;
|
|
3913
4787
|
}
|
|
3914
4788
|
|
|
3915
|
-
|
|
4789
|
+
state.timer = setTimeout(tick, intervalMs);
|
|
4790
|
+
state.timer.unref?.();
|
|
4791
|
+
};
|
|
3916
4792
|
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
})) > 0 &&
|
|
3920
|
-
this.events.dispatchEvent(
|
|
3921
|
-
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
3922
|
-
detail: { publicKey },
|
|
3923
|
-
}),
|
|
3924
|
-
);
|
|
3925
|
-
}
|
|
4793
|
+
tick();
|
|
4794
|
+
}
|
|
3926
4795
|
|
|
3927
|
-
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
|
|
4796
|
+
async handleSubscriptionChange(
|
|
4797
|
+
publicKey: PublicSignKey,
|
|
4798
|
+
topics: string[],
|
|
4799
|
+
subscribed: boolean,
|
|
4800
|
+
) {
|
|
4801
|
+
if (!topics.includes(this.topic)) {
|
|
4802
|
+
return;
|
|
4803
|
+
}
|
|
4804
|
+
|
|
4805
|
+
const peerHash = publicKey.hashcode();
|
|
4806
|
+
if (subscribed) {
|
|
4807
|
+
this._replicationInfoBlockedPeers.delete(peerHash);
|
|
4808
|
+
} else {
|
|
4809
|
+
this._replicationInfoBlockedPeers.add(peerHash);
|
|
4810
|
+
}
|
|
4811
|
+
|
|
4812
|
+
if (!subscribed) {
|
|
4813
|
+
// Emit replicator:leave at most once per (join -> leave) transition, even if we
|
|
4814
|
+
// concurrently process unsubscribe + replication reset messages for the same peer.
|
|
4815
|
+
const stoppedTransition = this.uniqueReplicators.delete(peerHash);
|
|
4816
|
+
this._replicatorJoinEmitted.delete(peerHash);
|
|
4817
|
+
|
|
4818
|
+
this.cancelReplicationInfoRequests(peerHash);
|
|
4819
|
+
this.removePeerFromGidPeerHistory(peerHash);
|
|
4820
|
+
|
|
4821
|
+
for (const [k, v] of this._requestIPruneSent) {
|
|
4822
|
+
v.delete(peerHash);
|
|
4823
|
+
if (v.size === 0) {
|
|
4824
|
+
this._requestIPruneSent.delete(k);
|
|
4825
|
+
}
|
|
4826
|
+
}
|
|
4827
|
+
|
|
4828
|
+
for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
|
|
4829
|
+
v.delete(peerHash);
|
|
4830
|
+
if (v.size === 0) {
|
|
4831
|
+
this._requestIPruneResponseReplicatorSet.delete(k);
|
|
4832
|
+
}
|
|
4833
|
+
}
|
|
4834
|
+
|
|
4835
|
+
this.syncronizer.onPeerDisconnected(publicKey);
|
|
4836
|
+
|
|
4837
|
+
stoppedTransition &&
|
|
4838
|
+
this.events.dispatchEvent(
|
|
4839
|
+
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
4840
|
+
detail: { publicKey },
|
|
3934
4841
|
}),
|
|
3935
|
-
|
|
3936
|
-
|
|
3937
|
-
},
|
|
3938
|
-
)
|
|
3939
|
-
.catch((e) => logger.error(e.toString()));
|
|
4842
|
+
);
|
|
4843
|
+
}
|
|
3940
4844
|
|
|
3941
|
-
|
|
3942
|
-
|
|
4845
|
+
if (subscribed) {
|
|
4846
|
+
const replicationSegments = await this.getMyReplicationSegments();
|
|
4847
|
+
if (replicationSegments.length > 0) {
|
|
3943
4848
|
this.rpc
|
|
3944
|
-
.send(
|
|
3945
|
-
|
|
3946
|
-
|
|
4849
|
+
.send(
|
|
4850
|
+
new AllReplicatingSegmentsMessage({
|
|
4851
|
+
segments: replicationSegments.map((x) => x.toReplicationRange()),
|
|
4852
|
+
}),
|
|
4853
|
+
{
|
|
4854
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
|
|
4855
|
+
},
|
|
4856
|
+
)
|
|
3947
4857
|
.catch((e) => logger.error(e.toString()));
|
|
4858
|
+
|
|
4859
|
+
if (this.v8Behaviour) {
|
|
4860
|
+
// for backwards compatibility
|
|
4861
|
+
this.rpc
|
|
4862
|
+
.send(new ResponseRoleMessage({ role: await this.getRole() }), {
|
|
4863
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
|
|
4864
|
+
})
|
|
4865
|
+
.catch((e) => logger.error(e.toString()));
|
|
4866
|
+
}
|
|
3948
4867
|
}
|
|
3949
|
-
}
|
|
3950
4868
|
|
|
3951
|
-
|
|
3952
|
-
|
|
4869
|
+
// Request the remote peer's replication info. This makes joins resilient to
|
|
4870
|
+
// timing-sensitive delivery/order issues where we may miss their initial
|
|
3953
4871
|
// replication announcement.
|
|
3954
|
-
this.
|
|
3955
|
-
.send(new RequestReplicationInfoMessage(), {
|
|
3956
|
-
mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
|
|
3957
|
-
})
|
|
3958
|
-
.catch((e) => logger.error(e.toString()));
|
|
4872
|
+
this.scheduleReplicationInfoRequests(publicKey);
|
|
3959
4873
|
} else {
|
|
3960
4874
|
await this.removeReplicator(publicKey);
|
|
3961
4875
|
}
|
|
@@ -3998,8 +4912,8 @@ export class SharedLog<
|
|
|
3998
4912
|
leaders: Map<string, unknown> | Set<string>;
|
|
3999
4913
|
}
|
|
4000
4914
|
>,
|
|
4001
|
-
|
|
4002
|
-
|
|
4915
|
+
options?: { timeout?: number; unchecked?: boolean },
|
|
4916
|
+
): Promise<any>[] {
|
|
4003
4917
|
if (options?.unchecked) {
|
|
4004
4918
|
return [...entries.values()].map((x) => {
|
|
4005
4919
|
this._gidPeersHistory.delete(x.entry.meta.gid);
|
|
@@ -4024,30 +4938,57 @@ export class SharedLog<
|
|
|
4024
4938
|
// - An entry is joined, where min replicas is lower than before (for all heads for this particular gid) and therefore we are not replicating anymore for this particular gid
|
|
4025
4939
|
// - Peers join and leave, which means we might not be a replicator anymore
|
|
4026
4940
|
|
|
4027
|
-
|
|
4941
|
+
const promises: Promise<any>[] = [];
|
|
4028
4942
|
|
|
4029
|
-
|
|
4030
|
-
|
|
4943
|
+
let peerToEntries: Map<string, string[]> = new Map();
|
|
4944
|
+
let cleanupTimer: ReturnType<typeof setTimeout>[] = [];
|
|
4945
|
+
const explicitTimeout = options?.timeout != null;
|
|
4031
4946
|
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4037
|
-
|
|
4038
|
-
|
|
4947
|
+
for (const { entry, leaders } of entries.values()) {
|
|
4948
|
+
for (const leader of leaders.keys()) {
|
|
4949
|
+
let set = peerToEntries.get(leader);
|
|
4950
|
+
if (!set) {
|
|
4951
|
+
set = [];
|
|
4952
|
+
peerToEntries.set(leader, set);
|
|
4953
|
+
}
|
|
4039
4954
|
|
|
4040
|
-
|
|
4041
|
-
|
|
4955
|
+
set.push(entry.hash);
|
|
4956
|
+
}
|
|
4042
4957
|
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
|
|
4047
|
-
|
|
4958
|
+
const pendingPrev = this._pendingDeletes.get(entry.hash);
|
|
4959
|
+
if (pendingPrev) {
|
|
4960
|
+
// If a background prune is already in-flight, an explicit prune request should
|
|
4961
|
+
// still respect the caller's timeout. Otherwise, tests (and user calls) can
|
|
4962
|
+
// block on the longer "checked prune" timeout derived from
|
|
4963
|
+
// `_respondToIHaveTimeout + waitForReplicatorTimeout`, which is intentionally
|
|
4964
|
+
// large for resiliency.
|
|
4965
|
+
if (explicitTimeout) {
|
|
4966
|
+
const timeoutMs = Math.max(0, Math.floor(options?.timeout ?? 0));
|
|
4967
|
+
promises.push(
|
|
4968
|
+
new Promise((resolve, reject) => {
|
|
4969
|
+
// Mirror the checked-prune error prefix so existing callers/tests can
|
|
4970
|
+
// match on the message substring.
|
|
4971
|
+
const timer = setTimeout(() => {
|
|
4972
|
+
reject(
|
|
4973
|
+
new Error(
|
|
4974
|
+
`Timeout for checked pruning after ${timeoutMs}ms (pending=true closed=${this.closed})`,
|
|
4975
|
+
),
|
|
4976
|
+
);
|
|
4977
|
+
}, timeoutMs);
|
|
4978
|
+
timer.unref?.();
|
|
4979
|
+
pendingPrev.promise.promise
|
|
4980
|
+
.then(resolve, reject)
|
|
4981
|
+
.finally(() => clearTimeout(timer));
|
|
4982
|
+
}),
|
|
4983
|
+
);
|
|
4984
|
+
} else {
|
|
4985
|
+
promises.push(pendingPrev.promise.promise);
|
|
4986
|
+
}
|
|
4987
|
+
continue;
|
|
4988
|
+
}
|
|
4048
4989
|
|
|
4049
|
-
|
|
4050
|
-
|
|
4990
|
+
const minReplicas = decodeReplicas(entry);
|
|
4991
|
+
const deferredPromise: DeferredPromise<void> = pDefer();
|
|
4051
4992
|
|
|
4052
4993
|
const clear = () => {
|
|
4053
4994
|
const pending = this._pendingDeletes.get(entry.hash);
|
|
@@ -4057,12 +4998,13 @@ export class SharedLog<
|
|
|
4057
4998
|
clearTimeout(timeout);
|
|
4058
4999
|
};
|
|
4059
5000
|
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
5001
|
+
const resolve = () => {
|
|
5002
|
+
clear();
|
|
5003
|
+
this.clearCheckedPruneRetry(entry.hash);
|
|
5004
|
+
cleanupTimer.push(
|
|
5005
|
+
setTimeout(async () => {
|
|
5006
|
+
this._gidPeersHistory.delete(entry.meta.gid);
|
|
5007
|
+
this.removePruneRequestSent(entry.hash);
|
|
4066
5008
|
this._requestIPruneResponseReplicatorSet.delete(entry.hash);
|
|
4067
5009
|
|
|
4068
5010
|
if (
|
|
@@ -4106,12 +5048,19 @@ export class SharedLog<
|
|
|
4106
5048
|
);
|
|
4107
5049
|
};
|
|
4108
5050
|
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
5051
|
+
const reject = (e: any) => {
|
|
5052
|
+
clear();
|
|
5053
|
+
const isCheckedPruneTimeout =
|
|
5054
|
+
e instanceof Error &&
|
|
5055
|
+
typeof e.message === "string" &&
|
|
5056
|
+
e.message.startsWith("Timeout for checked pruning");
|
|
5057
|
+
if (explicitTimeout || !isCheckedPruneTimeout) {
|
|
5058
|
+
this.clearCheckedPruneRetry(entry.hash);
|
|
5059
|
+
}
|
|
5060
|
+
this.removePruneRequestSent(entry.hash);
|
|
5061
|
+
this._requestIPruneResponseReplicatorSet.delete(entry.hash);
|
|
5062
|
+
deferredPromise.reject(e);
|
|
5063
|
+
};
|
|
4115
5064
|
|
|
4116
5065
|
let cursor: NumberFromType<R>[] | undefined = undefined;
|
|
4117
5066
|
|
|
@@ -4129,14 +5078,20 @@ export class SharedLog<
|
|
|
4129
5078
|
PRUNE_DEBOUNCE_INTERVAL * 2,
|
|
4130
5079
|
);
|
|
4131
5080
|
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
)
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
5081
|
+
const timeout = setTimeout(() => {
|
|
5082
|
+
// For internal/background prune flows (no explicit timeout), retry a few times
|
|
5083
|
+
// to avoid "permanently prunable" entries when `_pendingIHave` expires under
|
|
5084
|
+
// heavy load.
|
|
5085
|
+
if (!explicitTimeout) {
|
|
5086
|
+
this.scheduleCheckedPruneRetry({ entry, leaders });
|
|
5087
|
+
}
|
|
5088
|
+
reject(
|
|
5089
|
+
new Error(
|
|
5090
|
+
`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
|
|
5091
|
+
),
|
|
5092
|
+
);
|
|
5093
|
+
}, checkedPruneTimeoutMs);
|
|
5094
|
+
timeout.unref?.();
|
|
4140
5095
|
|
|
4141
5096
|
this._pendingDeletes.set(entry.hash, {
|
|
4142
5097
|
promise: deferredPromise,
|
|
@@ -4173,20 +5128,22 @@ export class SharedLog<
|
|
|
4173
5128
|
let existCounter = this._requestIPruneResponseReplicatorSet.get(
|
|
4174
5129
|
entry.hash,
|
|
4175
5130
|
);
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
5131
|
+
if (!existCounter) {
|
|
5132
|
+
existCounter = new Set();
|
|
5133
|
+
this._requestIPruneResponseReplicatorSet.set(
|
|
5134
|
+
entry.hash,
|
|
5135
|
+
existCounter,
|
|
5136
|
+
);
|
|
5137
|
+
}
|
|
5138
|
+
existCounter.add(publicKeyHash);
|
|
5139
|
+
// Seed provider hints so future remote reads can avoid extra round-trips.
|
|
5140
|
+
this.remoteBlocks.hintProviders(entry.hash, [publicKeyHash]);
|
|
4184
5141
|
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
5142
|
+
if (minReplicasValue <= existCounter.size) {
|
|
5143
|
+
resolve();
|
|
5144
|
+
}
|
|
5145
|
+
},
|
|
5146
|
+
});
|
|
4190
5147
|
|
|
4191
5148
|
promises.push(deferredPromise.promise);
|
|
4192
5149
|
}
|
|
@@ -4222,16 +5179,58 @@ export class SharedLog<
|
|
|
4222
5179
|
}
|
|
4223
5180
|
};
|
|
4224
5181
|
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
5182
|
+
for (const [k, v] of peerToEntries) {
|
|
5183
|
+
emitMessages(v, k);
|
|
5184
|
+
}
|
|
4228
5185
|
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
5186
|
+
// Keep remote `_pendingIHave` alive in the common "leader doesn't have entry yet"
|
|
5187
|
+
// case. This is intentionally disabled when an explicit timeout is provided to
|
|
5188
|
+
// preserve unit tests that assert remote `_pendingIHave` clears promptly.
|
|
5189
|
+
if (!explicitTimeout && peerToEntries.size > 0) {
|
|
5190
|
+
const respondToIHaveTimeout = Number(this._respondToIHaveTimeout ?? 0);
|
|
5191
|
+
const resendIntervalMs = Math.min(
|
|
5192
|
+
CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS,
|
|
5193
|
+
Math.max(
|
|
5194
|
+
CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS,
|
|
5195
|
+
Math.floor(respondToIHaveTimeout / 2) || 1_000,
|
|
5196
|
+
),
|
|
5197
|
+
);
|
|
5198
|
+
let inFlight = false;
|
|
5199
|
+
const timer = setInterval(() => {
|
|
5200
|
+
if (inFlight) return;
|
|
5201
|
+
if (this.closed) return;
|
|
5202
|
+
|
|
5203
|
+
const pendingByPeer: [string, string[]][] = [];
|
|
5204
|
+
for (const [peer, hashes] of peerToEntries) {
|
|
5205
|
+
const pending = hashes.filter((h) => this._pendingDeletes.has(h));
|
|
5206
|
+
if (pending.length > 0) {
|
|
5207
|
+
pendingByPeer.push([peer, pending]);
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
if (pendingByPeer.length === 0) {
|
|
5211
|
+
clearInterval(timer);
|
|
5212
|
+
return;
|
|
5213
|
+
}
|
|
5214
|
+
|
|
5215
|
+
inFlight = true;
|
|
5216
|
+
Promise.allSettled(
|
|
5217
|
+
pendingByPeer.map(([peer, hashes]) =>
|
|
5218
|
+
emitMessages(hashes, peer).catch(() => {}),
|
|
5219
|
+
),
|
|
5220
|
+
).finally(() => {
|
|
5221
|
+
inFlight = false;
|
|
5222
|
+
});
|
|
5223
|
+
}, resendIntervalMs);
|
|
5224
|
+
timer.unref?.();
|
|
5225
|
+
cleanupTimer.push(timer as any);
|
|
4232
5226
|
}
|
|
4233
|
-
|
|
4234
|
-
|
|
5227
|
+
|
|
5228
|
+
let cleanup = () => {
|
|
5229
|
+
for (const timer of cleanupTimer) {
|
|
5230
|
+
clearTimeout(timer);
|
|
5231
|
+
}
|
|
5232
|
+
this._closeController.signal.removeEventListener("abort", cleanup);
|
|
5233
|
+
};
|
|
4235
5234
|
|
|
4236
5235
|
Promise.allSettled(promises).finally(cleanup);
|
|
4237
5236
|
this._closeController.signal.addEventListener("abort", cleanup);
|
|
@@ -4303,12 +5302,21 @@ export class SharedLog<
|
|
|
4303
5302
|
* that we potentially need to share with other peers
|
|
4304
5303
|
*/
|
|
4305
5304
|
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
5305
|
+
if (this.closed) {
|
|
5306
|
+
return;
|
|
5307
|
+
}
|
|
4309
5308
|
|
|
4310
5309
|
await this.log.trim();
|
|
4311
5310
|
|
|
5311
|
+
const batchedChanges = Array.isArray(changeOrChanges[0])
|
|
5312
|
+
? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
|
|
5313
|
+
: [changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>];
|
|
5314
|
+
const changes = batchedChanges.flat();
|
|
5315
|
+
// On removed ranges (peer leaves / shrink), gid-level history can hide
|
|
5316
|
+
// per-entry gaps. Force a fresh delivery pass for reassigned entries.
|
|
5317
|
+
const forceFreshDelivery = changes.some((change) => change.type === "removed");
|
|
5318
|
+
const gidPeersHistorySnapshot = new Map<string, Set<string> | undefined>();
|
|
5319
|
+
|
|
4312
5320
|
const changed = false;
|
|
4313
5321
|
|
|
4314
5322
|
try {
|
|
@@ -4318,7 +5326,7 @@ export class SharedLog<
|
|
|
4318
5326
|
> = new Map();
|
|
4319
5327
|
|
|
4320
5328
|
for await (const entryReplicated of toRebalance<R>(
|
|
4321
|
-
|
|
5329
|
+
changes,
|
|
4322
5330
|
this.entryCoordinatesIndex,
|
|
4323
5331
|
this.recentlyRebalanced,
|
|
4324
5332
|
)) {
|
|
@@ -4326,7 +5334,16 @@ export class SharedLog<
|
|
|
4326
5334
|
break;
|
|
4327
5335
|
}
|
|
4328
5336
|
|
|
4329
|
-
let oldPeersSet
|
|
5337
|
+
let oldPeersSet: Set<string> | undefined;
|
|
5338
|
+
if (!forceFreshDelivery) {
|
|
5339
|
+
const gid = entryReplicated.gid;
|
|
5340
|
+
oldPeersSet = gidPeersHistorySnapshot.get(gid);
|
|
5341
|
+
if (!gidPeersHistorySnapshot.has(gid)) {
|
|
5342
|
+
const existing = this._gidPeersHistory.get(gid);
|
|
5343
|
+
oldPeersSet = existing ? new Set(existing) : undefined;
|
|
5344
|
+
gidPeersHistorySnapshot.set(gid, oldPeersSet);
|
|
5345
|
+
}
|
|
5346
|
+
}
|
|
4330
5347
|
let isLeader = false;
|
|
4331
5348
|
|
|
4332
5349
|
let currentPeers = await this.findLeaders(
|
|
@@ -4405,32 +5422,51 @@ export class SharedLog<
|
|
|
4405
5422
|
}
|
|
4406
5423
|
}
|
|
4407
5424
|
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
5425
|
+
async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
|
|
5426
|
+
logger.trace(
|
|
5427
|
+
`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
|
|
5428
|
+
evt.detail.topics.map((x) => x),
|
|
5429
|
+
)} '`,
|
|
5430
|
+
);
|
|
5431
|
+
if (!evt.detail.topics.includes(this.topic)) {
|
|
5432
|
+
return;
|
|
5433
|
+
}
|
|
4415
5434
|
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
evt.detail.topics,
|
|
4419
|
-
false,
|
|
4420
|
-
);
|
|
4421
|
-
}
|
|
5435
|
+
const fromHash = evt.detail.from.hashcode();
|
|
5436
|
+
this._replicationInfoBlockedPeers.add(fromHash);
|
|
4422
5437
|
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
)
|
|
4428
|
-
|
|
4429
|
-
|
|
5438
|
+
// Keep a per-peer timestamp watermark when we observe an unsubscribe. This
|
|
5439
|
+
// prevents late/out-of-order replication-info messages from re-introducing
|
|
5440
|
+
// stale segments for a peer that has already left the topic.
|
|
5441
|
+
const now = BigInt(+new Date());
|
|
5442
|
+
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
5443
|
+
if (!prev || prev < now) {
|
|
5444
|
+
this.latestReplicationInfoMessage.set(fromHash, now);
|
|
5445
|
+
}
|
|
5446
|
+
|
|
5447
|
+
return this.handleSubscriptionChange(
|
|
5448
|
+
evt.detail.from,
|
|
5449
|
+
evt.detail.topics,
|
|
5450
|
+
false,
|
|
5451
|
+
);
|
|
5452
|
+
}
|
|
5453
|
+
|
|
5454
|
+
async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
|
|
5455
|
+
logger.trace(
|
|
5456
|
+
`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
|
|
5457
|
+
evt.detail.topics.map((x) => x),
|
|
5458
|
+
)}'`,
|
|
5459
|
+
);
|
|
5460
|
+
if (!evt.detail.topics.includes(this.topic)) {
|
|
5461
|
+
return;
|
|
5462
|
+
}
|
|
5463
|
+
|
|
5464
|
+
this.remoteBlocks.onReachable(evt.detail.from);
|
|
5465
|
+
this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
|
|
4430
5466
|
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
5467
|
+
return this.handleSubscriptionChange(
|
|
5468
|
+
evt.detail.from,
|
|
5469
|
+
evt.detail.topics,
|
|
4434
5470
|
true,
|
|
4435
5471
|
);
|
|
4436
5472
|
}
|