@peerbit/pubsub 5.4.8 → 5.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-tree.d.ts +11 -1
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +228 -38
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +641 -141
- package/dist/src/index.js.map +1 -1
- package/package.json +5 -5
- package/src/fanout-tree.ts +301 -42
- package/src/index.ts +899 -190
package/src/index.ts
CHANGED
|
@@ -244,6 +244,31 @@ const TOPIC_ROOT_QUERY_ROUND_TIMEOUT_MS = 3_000;
|
|
|
244
244
|
const TOPIC_ROOT_HOST_SCAN_TIMEOUT_MS = DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS;
|
|
245
245
|
const TOPIC_ROOT_HOST_SCAN_CONCURRENCY = 8;
|
|
246
246
|
const DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS = 2_000;
|
|
247
|
+
const RELAY_ROOT_PREFLIGHT_TIMEOUT_MS = 2_000;
|
|
248
|
+
const RELAY_ROOT_PREFLIGHT_REUSE_MS = 2_000;
|
|
249
|
+
|
|
250
|
+
type RelayRootProbeResult = {
|
|
251
|
+
claim: TopicRootCandidateClaimRecord;
|
|
252
|
+
routes: string[];
|
|
253
|
+
status: "alive" | "failed" | "unknown";
|
|
254
|
+
observedAt: number;
|
|
255
|
+
};
|
|
256
|
+
type RelayRootProbeCohort = {
|
|
257
|
+
controller: AbortController;
|
|
258
|
+
users: Set<() => boolean>;
|
|
259
|
+
promise: Promise<ReadonlyMap<string, RelayRootProbeResult>>;
|
|
260
|
+
results?: ReadonlyMap<string, RelayRootProbeResult>;
|
|
261
|
+
lifecycleRevision: number;
|
|
262
|
+
};
|
|
263
|
+
type RelayRootRecoveryAttempt = {
|
|
264
|
+
cohort?: RelayRootProbeCohort;
|
|
265
|
+
policyDeadline?: number;
|
|
266
|
+
};
|
|
267
|
+
type TopicRootState = {
|
|
268
|
+
root?: string;
|
|
269
|
+
authoritative: boolean;
|
|
270
|
+
autoCandidate: boolean;
|
|
271
|
+
};
|
|
247
272
|
|
|
248
273
|
const DEFAULT_PUBSUB_FANOUT_CHANNEL_OPTIONS: Omit<
|
|
249
274
|
FanoutTreeChannelOptions,
|
|
@@ -341,6 +366,7 @@ type EnsureFanoutChannelOptions = {
|
|
|
341
366
|
root?: string;
|
|
342
367
|
rootCandidateGeneration?: string;
|
|
343
368
|
signal?: AbortSignal;
|
|
369
|
+
isCurrent?: () => boolean;
|
|
344
370
|
};
|
|
345
371
|
|
|
346
372
|
export type TopicControlPlaneComponents = DirectStreamComponents;
|
|
@@ -427,7 +453,7 @@ export class TopicControlPlane
|
|
|
427
453
|
private readonly hostShards: boolean;
|
|
428
454
|
private readonly shardRootCache = new Map<
|
|
429
455
|
string,
|
|
430
|
-
{ root: string
|
|
456
|
+
TopicRootState & { root: string }
|
|
431
457
|
>();
|
|
432
458
|
private readonly shardTopicCache = new Map<string, string>();
|
|
433
459
|
private readonly shardRefCounts = new Map<string, number>();
|
|
@@ -466,9 +492,9 @@ export class TopicControlPlane
|
|
|
466
492
|
string,
|
|
467
493
|
TopicRootCandidateClaimRecord
|
|
468
494
|
>();
|
|
469
|
-
|
|
470
|
-
//
|
|
471
|
-
//
|
|
495
|
+
private relayRootProbeCohort?: RelayRootProbeCohort;
|
|
496
|
+
// Locally observed unreachability makes only that origin's current lease
|
|
497
|
+
// ineffective locally. This is not a component-wide withdrawal. Keeping the verified
|
|
472
498
|
// bytes + replay floor prevents a relay from resurrecting the departed lease;
|
|
473
499
|
// the map is bounded by the same retained-origin cap as the claim map.
|
|
474
500
|
private readonly suppressedDepartedTopicRootCandidateClaims = new Map<
|
|
@@ -499,6 +525,38 @@ export class TopicControlPlane
|
|
|
499
525
|
private _onFanoutPeerUnreachable?: (
|
|
500
526
|
ev: CustomEvent<{ topic: string; root: string; publicKeyHash: string }>,
|
|
501
527
|
) => void;
|
|
528
|
+
private readonly onFanoutAttached = (
|
|
529
|
+
ev: CustomEvent<{ topic: string; root: string }>,
|
|
530
|
+
) => {
|
|
531
|
+
const st = this.fanoutChannels.get(ev.detail.topic);
|
|
532
|
+
if (!st || st.root !== ev.detail.root) return;
|
|
533
|
+
// A successful publish before attachment may have had no listeners.
|
|
534
|
+
// Coalesce attachment bursts, retaining one trailing pass during a send.
|
|
535
|
+
st.announceDirty = true;
|
|
536
|
+
if (st.announceTask) return;
|
|
537
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
538
|
+
st.announceTask = Promise.resolve()
|
|
539
|
+
.then(async () => {
|
|
540
|
+
await st.join;
|
|
541
|
+
while (
|
|
542
|
+
st.announceDirty &&
|
|
543
|
+
this.isFanoutChannelCurrent(ev.detail.topic, st.channel, revision)
|
|
544
|
+
) {
|
|
545
|
+
st.announceDirty = false;
|
|
546
|
+
await this.announceShardSubscriptions(ev.detail.topic, revision);
|
|
547
|
+
}
|
|
548
|
+
})
|
|
549
|
+
.catch(logErrorIfStarted)
|
|
550
|
+
.finally(() => {
|
|
551
|
+
st.announceTask = undefined;
|
|
552
|
+
if (
|
|
553
|
+
st.announceDirty &&
|
|
554
|
+
this.isFanoutChannelCurrent(ev.detail.topic, st.channel, revision)
|
|
555
|
+
) {
|
|
556
|
+
this.onFanoutAttached(ev);
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
};
|
|
502
560
|
|
|
503
561
|
private fanoutChannels = new Map<
|
|
504
562
|
string,
|
|
@@ -511,12 +569,15 @@ export class TopicControlPlane
|
|
|
511
569
|
ephemeral: boolean;
|
|
512
570
|
lastUsedAt: number;
|
|
513
571
|
idleCloseTimeout?: ReturnType<typeof setTimeout>;
|
|
572
|
+
announceDirty?: boolean;
|
|
573
|
+
announceTask?: Promise<void>;
|
|
514
574
|
}
|
|
515
575
|
>();
|
|
516
576
|
private ensureFanoutChannelInFlight = new Map<
|
|
517
577
|
string,
|
|
518
578
|
{
|
|
519
579
|
abortController: AbortController;
|
|
580
|
+
isCurrent: () => boolean;
|
|
520
581
|
candidateGeneration: string;
|
|
521
582
|
lifecycleRevision: number;
|
|
522
583
|
opening: Promise<void>;
|
|
@@ -742,6 +803,8 @@ export class TopicControlPlane
|
|
|
742
803
|
"fanout:peer-unreachable",
|
|
743
804
|
this._onFanoutPeerUnreachable as any,
|
|
744
805
|
);
|
|
806
|
+
this.fanout.addEventListener("fanout:joined", this.onFanoutAttached);
|
|
807
|
+
this.fanout.addEventListener("fanout:child-joined", this.onFanoutAttached);
|
|
745
808
|
await super.start();
|
|
746
809
|
if (this.autoTopicRootCandidates) {
|
|
747
810
|
let retryDelayMs = 0;
|
|
@@ -790,8 +853,13 @@ export class TopicControlPlane
|
|
|
790
853
|
this._onFanoutPeerUnreachable as any,
|
|
791
854
|
);
|
|
792
855
|
}
|
|
856
|
+
this.fanout.removeEventListener("fanout:joined", this.onFanoutAttached);
|
|
857
|
+
this.fanout.removeEventListener(
|
|
858
|
+
"fanout:child-joined",
|
|
859
|
+
this.onFanoutAttached,
|
|
860
|
+
);
|
|
793
861
|
for (const st of this.fanoutChannels.values()) {
|
|
794
|
-
|
|
862
|
+
this.clearFanoutIdleClose(st);
|
|
795
863
|
try {
|
|
796
864
|
st.channel.removeEventListener("data", st.onData as any);
|
|
797
865
|
} catch {
|
|
@@ -862,32 +930,34 @@ export class TopicControlPlane
|
|
|
862
930
|
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
863
931
|
immediate: hadSignedPeer !== hasSignedPeer,
|
|
864
932
|
});
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
}
|
|
878
|
-
if (existingPeer !== peer) {
|
|
879
|
-
peer.addEventListener("stream:outbound", sendWhenOutboundReady);
|
|
880
|
-
peer.addEventListener(
|
|
881
|
-
"close",
|
|
882
|
-
() =>
|
|
883
|
-
peer.removeEventListener(
|
|
884
|
-
"stream:outbound",
|
|
885
|
-
sendWhenOutboundReady,
|
|
886
|
-
),
|
|
887
|
-
{ once: true },
|
|
888
|
-
);
|
|
889
|
-
}
|
|
933
|
+
}
|
|
934
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
935
|
+
const sendWhenOutboundReady = () => {
|
|
936
|
+
if (
|
|
937
|
+
!this.isTopicControlPlaneActive(lifecycleRevision) ||
|
|
938
|
+
this.peers.get(publicKey.hashcode()) !== peer
|
|
939
|
+
)
|
|
940
|
+
return;
|
|
941
|
+
if (
|
|
942
|
+
this.autoTopicRootCandidates &&
|
|
943
|
+
peer.protocol === TOPIC_CONTROL_PLANE_PROTOCOL_V2_1
|
|
944
|
+
) {
|
|
945
|
+
void this.sendAutoTopicRootCandidates([peer]).catch(() => {});
|
|
890
946
|
}
|
|
947
|
+
void this.announceDirectSubscriptions(
|
|
948
|
+
[...this.subscriptions.keys()],
|
|
949
|
+
[peer],
|
|
950
|
+
).catch(logErrorIfStarted);
|
|
951
|
+
};
|
|
952
|
+
if (peer.isWritable) sendWhenOutboundReady();
|
|
953
|
+
if (existingPeer !== peer) {
|
|
954
|
+
peer.addEventListener("stream:outbound", sendWhenOutboundReady);
|
|
955
|
+
peer.addEventListener(
|
|
956
|
+
"close",
|
|
957
|
+
() =>
|
|
958
|
+
peer.removeEventListener("stream:outbound", sendWhenOutboundReady),
|
|
959
|
+
{ once: true },
|
|
960
|
+
);
|
|
891
961
|
}
|
|
892
962
|
return peer;
|
|
893
963
|
}
|
|
@@ -911,16 +981,10 @@ export class TopicControlPlane
|
|
|
911
981
|
this.signedTopicRootCandidateClaims.has(hash)
|
|
912
982
|
) {
|
|
913
983
|
const claim = this.signedTopicRootCandidateClaims.get(hash)!;
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
suppressed.timestamp !== claim.timestamp ||
|
|
919
|
-
!bytesEqual(suppressed.bytes, claim.bytes);
|
|
920
|
-
this.suppressedDepartedTopicRootCandidateClaims.set(hash, {
|
|
921
|
-
bytes: claim.bytes,
|
|
922
|
-
timestamp: claim.timestamp,
|
|
923
|
-
});
|
|
984
|
+
suppressedDepartedOrigin = this.suppressTopicRootCandidateClaim(
|
|
985
|
+
hash,
|
|
986
|
+
claim,
|
|
987
|
+
);
|
|
924
988
|
}
|
|
925
989
|
if (
|
|
926
990
|
this.autoTopicRootCandidates &&
|
|
@@ -973,6 +1037,10 @@ export class TopicControlPlane
|
|
|
973
1037
|
}
|
|
974
1038
|
|
|
975
1039
|
private clearSignedTopicRootCandidateState() {
|
|
1040
|
+
this.relayRootProbeCohort?.controller.abort(
|
|
1041
|
+
new AbortError("topic root candidate recovery stopped"),
|
|
1042
|
+
);
|
|
1043
|
+
this.relayRootProbeCohort = undefined;
|
|
976
1044
|
this.clearTopicRootCandidateClaimTimer();
|
|
977
1045
|
this.signedTopicRootCandidateClaims.clear();
|
|
978
1046
|
this.suppressedDepartedTopicRootCandidateClaims.clear();
|
|
@@ -1348,6 +1416,329 @@ export class TopicControlPlane
|
|
|
1348
1416
|
);
|
|
1349
1417
|
}
|
|
1350
1418
|
|
|
1419
|
+
private suppressTopicRootCandidateClaim(
|
|
1420
|
+
origin: string,
|
|
1421
|
+
claim: TopicRootCandidateClaimRecord,
|
|
1422
|
+
): boolean {
|
|
1423
|
+
if (
|
|
1424
|
+
!this.autoTopicRootCandidates ||
|
|
1425
|
+
origin === this.publicKeyHash ||
|
|
1426
|
+
this.peers.has(origin) ||
|
|
1427
|
+
this.signedTopicRootCandidateClaims.get(origin) !== claim ||
|
|
1428
|
+
this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim)
|
|
1429
|
+
)
|
|
1430
|
+
return false;
|
|
1431
|
+
this.suppressedDepartedTopicRootCandidateClaims.set(origin, {
|
|
1432
|
+
bytes: claim.bytes,
|
|
1433
|
+
timestamp: claim.timestamp,
|
|
1434
|
+
});
|
|
1435
|
+
return true;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
private topicRootRouteObservations(origin: string): string[] {
|
|
1439
|
+
return this.routes
|
|
1440
|
+
.getRouteHints(this.publicKeyHash, origin)
|
|
1441
|
+
.map((hint) =>
|
|
1442
|
+
JSON.stringify([hint.nextHop, hint.session, hint.updatedAt]),
|
|
1443
|
+
);
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
private createRelayRootProbeCohort(
|
|
1447
|
+
claims: Array<[string, TopicRootCandidateClaimRecord]>,
|
|
1448
|
+
): RelayRootProbeCohort {
|
|
1449
|
+
const previous = this.relayRootProbeCohort?.results;
|
|
1450
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
1451
|
+
const controller = new AbortController();
|
|
1452
|
+
const users = new Set<() => boolean>();
|
|
1453
|
+
const deadline = new AbortError("relay root reachability deadline reached");
|
|
1454
|
+
const timer = setTimeout(
|
|
1455
|
+
() => controller.abort(deadline),
|
|
1456
|
+
RELAY_ROOT_PREFLIGHT_TIMEOUT_MS,
|
|
1457
|
+
);
|
|
1458
|
+
timer.unref?.();
|
|
1459
|
+
let resolveRetry!: () => void;
|
|
1460
|
+
const retryBoundary = new Promise<void>((resolve) => {
|
|
1461
|
+
resolveRetry = resolve;
|
|
1462
|
+
});
|
|
1463
|
+
const retryTimer = setTimeout(
|
|
1464
|
+
resolveRetry,
|
|
1465
|
+
RELAY_ROOT_PREFLIGHT_TIMEOUT_MS / 2,
|
|
1466
|
+
);
|
|
1467
|
+
retryTimer.unref?.();
|
|
1468
|
+
const captured = claims.map(([origin, claim]) => ({
|
|
1469
|
+
origin,
|
|
1470
|
+
claim,
|
|
1471
|
+
routes: this.topicRootRouteObservations(origin),
|
|
1472
|
+
}));
|
|
1473
|
+
const cohort: RelayRootProbeCohort = {
|
|
1474
|
+
controller,
|
|
1475
|
+
users,
|
|
1476
|
+
lifecycleRevision,
|
|
1477
|
+
promise: Promise.resolve()
|
|
1478
|
+
.then(async () => {
|
|
1479
|
+
const outcomes = await Promise.all(
|
|
1480
|
+
captured.map(async ({ origin, claim, routes }) => {
|
|
1481
|
+
const cached = previous?.get(origin);
|
|
1482
|
+
if (
|
|
1483
|
+
cached?.claim === claim &&
|
|
1484
|
+
cached.status === "alive" &&
|
|
1485
|
+
performance.now() - cached.observedAt <
|
|
1486
|
+
RELAY_ROOT_PREFLIGHT_REUSE_MS
|
|
1487
|
+
)
|
|
1488
|
+
return [origin, { ...cached, routes }] as const;
|
|
1489
|
+
const winner = new AbortController();
|
|
1490
|
+
const signals = linkAbortSignals([
|
|
1491
|
+
controller.signal,
|
|
1492
|
+
winner.signal,
|
|
1493
|
+
]);
|
|
1494
|
+
let alive = false;
|
|
1495
|
+
let resolveAlive!: (status: "alive") => void;
|
|
1496
|
+
const firstSuccess = new Promise<"alive">((resolve) => {
|
|
1497
|
+
resolveAlive = resolve;
|
|
1498
|
+
});
|
|
1499
|
+
const probe = async (
|
|
1500
|
+
retry: boolean,
|
|
1501
|
+
): Promise<RelayRootProbeResult["status"]> => {
|
|
1502
|
+
const signal = signals.signal;
|
|
1503
|
+
let submitted = false;
|
|
1504
|
+
try {
|
|
1505
|
+
if (retry) await withAbort(retryBoundary, signal);
|
|
1506
|
+
throwIfAborted(signal);
|
|
1507
|
+
if (alive) return "alive";
|
|
1508
|
+
const message = await withAbort(
|
|
1509
|
+
this.createMessage(undefined, {
|
|
1510
|
+
mode: new AcknowledgeDelivery({
|
|
1511
|
+
to: [origin],
|
|
1512
|
+
redundancy: 1,
|
|
1513
|
+
}),
|
|
1514
|
+
priority: 1,
|
|
1515
|
+
responsePriority: 1,
|
|
1516
|
+
}),
|
|
1517
|
+
signal,
|
|
1518
|
+
);
|
|
1519
|
+
// Signers may ignore cancellation. Fence the send continuation.
|
|
1520
|
+
throwIfAborted(signal);
|
|
1521
|
+
if (alive) return "alive";
|
|
1522
|
+
if (
|
|
1523
|
+
!this.isTopicControlPlaneActive(lifecycleRevision) ||
|
|
1524
|
+
![...users].some((current) => current())
|
|
1525
|
+
) {
|
|
1526
|
+
return "unknown";
|
|
1527
|
+
}
|
|
1528
|
+
// No transport neighbor is local readiness, not proof of origin death.
|
|
1529
|
+
if (this.peers.size === 0) return "unknown";
|
|
1530
|
+
submitted = true;
|
|
1531
|
+
await withAbort(
|
|
1532
|
+
this.publishMessage(
|
|
1533
|
+
this.publicKey,
|
|
1534
|
+
message,
|
|
1535
|
+
undefined,
|
|
1536
|
+
undefined,
|
|
1537
|
+
signal,
|
|
1538
|
+
),
|
|
1539
|
+
signal,
|
|
1540
|
+
);
|
|
1541
|
+
throwIfAborted(signal);
|
|
1542
|
+
alive = true;
|
|
1543
|
+
resolveAlive("alive");
|
|
1544
|
+
return "alive";
|
|
1545
|
+
} catch (error) {
|
|
1546
|
+
if (
|
|
1547
|
+
submitted &&
|
|
1548
|
+
(error === deadline || error instanceof DeliveryError)
|
|
1549
|
+
)
|
|
1550
|
+
return "failed";
|
|
1551
|
+
else if (!signal.aborted) throw error;
|
|
1552
|
+
return "unknown";
|
|
1553
|
+
}
|
|
1554
|
+
};
|
|
1555
|
+
// Hedge once at the shared midpoint without discarding the first
|
|
1556
|
+
// authenticated ACK wait: a 1.5s first reply is still timely.
|
|
1557
|
+
let status: RelayRootProbeResult["status"];
|
|
1558
|
+
try {
|
|
1559
|
+
status = await Promise.race([
|
|
1560
|
+
firstSuccess,
|
|
1561
|
+
Promise.all([probe(false), probe(true)]).then((results) =>
|
|
1562
|
+
results.includes("unknown")
|
|
1563
|
+
? ("unknown" as const)
|
|
1564
|
+
: ("failed" as const),
|
|
1565
|
+
),
|
|
1566
|
+
]);
|
|
1567
|
+
} finally {
|
|
1568
|
+
winner.abort(new AbortError("origin probe settled"));
|
|
1569
|
+
signals.clear();
|
|
1570
|
+
}
|
|
1571
|
+
return [
|
|
1572
|
+
origin,
|
|
1573
|
+
{ claim, routes, status, observedAt: performance.now() },
|
|
1574
|
+
] as const;
|
|
1575
|
+
}),
|
|
1576
|
+
);
|
|
1577
|
+
cohort.results = new Map(outcomes);
|
|
1578
|
+
return cohort.results;
|
|
1579
|
+
})
|
|
1580
|
+
.finally(() => {
|
|
1581
|
+
clearTimeout(timer);
|
|
1582
|
+
clearTimeout(retryTimer);
|
|
1583
|
+
controller.abort(new AbortError("relay root probes settled"));
|
|
1584
|
+
}),
|
|
1585
|
+
};
|
|
1586
|
+
this.relayRootProbeCohort = cohort;
|
|
1587
|
+
return cohort;
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
private async preflightRelayRootCandidates(
|
|
1591
|
+
shardTopic: string,
|
|
1592
|
+
attempt: RelayRootRecoveryAttempt,
|
|
1593
|
+
signal: AbortSignal | undefined,
|
|
1594
|
+
isCurrent: () => boolean,
|
|
1595
|
+
localPolicyOnly = false,
|
|
1596
|
+
): Promise<boolean> {
|
|
1597
|
+
throwIfAborted(signal);
|
|
1598
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
1599
|
+
const current = () =>
|
|
1600
|
+
isCurrent() && this.isTopicControlPlaneActive(revision);
|
|
1601
|
+
if (!current()) throw new AbortError("obsolete relay root recovery");
|
|
1602
|
+
if (
|
|
1603
|
+
!this.autoTopicRootCandidates ||
|
|
1604
|
+
this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured()
|
|
1605
|
+
)
|
|
1606
|
+
return false;
|
|
1607
|
+
const claims = [...this.signedTopicRootCandidateClaims.entries()]
|
|
1608
|
+
.filter(
|
|
1609
|
+
([origin, claim]) =>
|
|
1610
|
+
origin !== this.publicKeyHash &&
|
|
1611
|
+
!this.peers.has(origin) &&
|
|
1612
|
+
!this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim),
|
|
1613
|
+
)
|
|
1614
|
+
.slice(0, TOPIC_ROOT_CANDIDATES_MAX);
|
|
1615
|
+
if (claims.length === 0) return false;
|
|
1616
|
+
const shared = this.relayRootProbeCohort;
|
|
1617
|
+
const cohort = (attempt.cohort ??=
|
|
1618
|
+
shared &&
|
|
1619
|
+
(shared.results
|
|
1620
|
+
? shared.users.size > 0
|
|
1621
|
+
: !shared.controller.signal.aborted) &&
|
|
1622
|
+
shared.lifecycleRevision === revision
|
|
1623
|
+
? shared
|
|
1624
|
+
: this.createRelayRootProbeCohort(claims));
|
|
1625
|
+
cohort.users.add(current);
|
|
1626
|
+
let held = true;
|
|
1627
|
+
const release = () => {
|
|
1628
|
+
if (!held) return;
|
|
1629
|
+
held = false;
|
|
1630
|
+
cohort.users.delete(current);
|
|
1631
|
+
if (cohort.users.size === 0 && !cohort.results) {
|
|
1632
|
+
cohort.controller.abort(
|
|
1633
|
+
new AbortError("relay root recovery cancelled"),
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1636
|
+
};
|
|
1637
|
+
signal?.addEventListener("abort", release, { once: true });
|
|
1638
|
+
try {
|
|
1639
|
+
const outcomes = await withAbort(cohort.promise, signal);
|
|
1640
|
+
throwIfAborted(signal);
|
|
1641
|
+
if (!current()) throw new AbortError("obsolete relay root recovery");
|
|
1642
|
+
|
|
1643
|
+
// Policy setters do not share the candidate generation. Re-read configured
|
|
1644
|
+
// roots/resolvers/trackers after observation; a stalled policy is unknown,
|
|
1645
|
+
// never evidence authorizing local suppression.
|
|
1646
|
+
attempt.policyDeadline ??=
|
|
1647
|
+
performance.now() + RELAY_ROOT_PREFLIGHT_TIMEOUT_MS;
|
|
1648
|
+
return await this.withRelayRootPolicyDeadline(
|
|
1649
|
+
attempt,
|
|
1650
|
+
signal,
|
|
1651
|
+
async (policySignal) => {
|
|
1652
|
+
const configured = await withAbort(
|
|
1653
|
+
localPolicyOnly
|
|
1654
|
+
? this.topicRootControlPlane.resolveLocalTopicRoot(shardTopic, {
|
|
1655
|
+
signal: policySignal,
|
|
1656
|
+
})
|
|
1657
|
+
: this.topicRootControlPlane.resolveTrackedTopicRoot(shardTopic, {
|
|
1658
|
+
signal: policySignal,
|
|
1659
|
+
}),
|
|
1660
|
+
policySignal,
|
|
1661
|
+
);
|
|
1662
|
+
throwIfAborted(policySignal);
|
|
1663
|
+
if (!current()) throw new AbortError("obsolete relay root recovery");
|
|
1664
|
+
this.shardRootCache.delete(shardTopic);
|
|
1665
|
+
if (
|
|
1666
|
+
configured ||
|
|
1667
|
+
this.topicRootControlPlane.getTopicRoot(shardTopic) ||
|
|
1668
|
+
!this.autoTopicRootCandidates ||
|
|
1669
|
+
this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured()
|
|
1670
|
+
)
|
|
1671
|
+
return true;
|
|
1672
|
+
let changed = false;
|
|
1673
|
+
let unknown = false;
|
|
1674
|
+
for (const [origin, claim] of claims) {
|
|
1675
|
+
if (
|
|
1676
|
+
this.peers.has(origin) ||
|
|
1677
|
+
this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim)
|
|
1678
|
+
)
|
|
1679
|
+
continue;
|
|
1680
|
+
const observed = outcomes.get(origin);
|
|
1681
|
+
if (
|
|
1682
|
+
this.signedTopicRootCandidateClaims.get(origin) !== claim ||
|
|
1683
|
+
observed?.claim !== claim
|
|
1684
|
+
) {
|
|
1685
|
+
unknown = true;
|
|
1686
|
+
continue;
|
|
1687
|
+
}
|
|
1688
|
+
const freshRoute =
|
|
1689
|
+
this.routes.isReachable(this.publicKeyHash, origin) &&
|
|
1690
|
+
this.topicRootRouteObservations(origin).some(
|
|
1691
|
+
(route) => !observed.routes.includes(route),
|
|
1692
|
+
);
|
|
1693
|
+
if (freshRoute || observed.status === "alive") continue;
|
|
1694
|
+
if (observed.status === "unknown") unknown = true;
|
|
1695
|
+
else
|
|
1696
|
+
changed =
|
|
1697
|
+
this.suppressTopicRootCandidateClaim(origin, claim) || changed;
|
|
1698
|
+
}
|
|
1699
|
+
if (changed)
|
|
1700
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
1701
|
+
immediate: true,
|
|
1702
|
+
});
|
|
1703
|
+
if (unknown)
|
|
1704
|
+
throw new DeliveryError(
|
|
1705
|
+
"Relay root reachability could not be established",
|
|
1706
|
+
);
|
|
1707
|
+
return true;
|
|
1708
|
+
},
|
|
1709
|
+
);
|
|
1710
|
+
} finally {
|
|
1711
|
+
signal?.removeEventListener("abort", release);
|
|
1712
|
+
release();
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
private async withRelayRootPolicyDeadline<T>(
|
|
1717
|
+
attempt: RelayRootRecoveryAttempt,
|
|
1718
|
+
signal: AbortSignal | undefined,
|
|
1719
|
+
resolve: (signal: AbortSignal | undefined) => Promise<T>,
|
|
1720
|
+
): Promise<T> {
|
|
1721
|
+
// Ordinary initial resolution retains its existing contract. Only work
|
|
1722
|
+
// following a probe shares the one additional policy budget, including retries.
|
|
1723
|
+
if (attempt.policyDeadline === undefined) return resolve(signal);
|
|
1724
|
+
throwIfAborted(signal);
|
|
1725
|
+
const remaining = attempt.policyDeadline - performance.now();
|
|
1726
|
+
const deadline = new AbortError(
|
|
1727
|
+
"relay root policy recheck deadline reached",
|
|
1728
|
+
);
|
|
1729
|
+
if (remaining <= 0) throw deadline;
|
|
1730
|
+
const controller = new AbortController();
|
|
1731
|
+
const timer = setTimeout(() => controller.abort(deadline), remaining);
|
|
1732
|
+
timer.unref?.();
|
|
1733
|
+
const signals = linkAbortSignals([signal, controller.signal]);
|
|
1734
|
+
try {
|
|
1735
|
+
return await withAbort(resolve(signals.signal), signals.signal);
|
|
1736
|
+
} finally {
|
|
1737
|
+
clearTimeout(timer);
|
|
1738
|
+
signals.clear();
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1351
1742
|
private canAdvanceTopicRootCandidateClaimReplayFloor(
|
|
1352
1743
|
origin: string,
|
|
1353
1744
|
timestamp: bigint,
|
|
@@ -1425,6 +1816,7 @@ export class TopicControlPlane
|
|
|
1425
1816
|
rawClaim: Uint8Array,
|
|
1426
1817
|
outerPeerHash: string,
|
|
1427
1818
|
expectedPeer?: PeerStreams,
|
|
1819
|
+
batchUpdate?: { immediate: boolean },
|
|
1428
1820
|
): Promise<boolean> {
|
|
1429
1821
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
1430
1822
|
if (
|
|
@@ -1470,7 +1862,12 @@ export class TopicControlPlane
|
|
|
1470
1862
|
return false;
|
|
1471
1863
|
}
|
|
1472
1864
|
|
|
1473
|
-
|
|
1865
|
+
if (batchUpdate) {
|
|
1866
|
+
if (this.pruneExpiredTopicRootCandidateClaims(now))
|
|
1867
|
+
batchUpdate.immediate = true;
|
|
1868
|
+
} else {
|
|
1869
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(now);
|
|
1870
|
+
}
|
|
1474
1871
|
const retained = this.signedTopicRootCandidateClaims.get(origin);
|
|
1475
1872
|
if (
|
|
1476
1873
|
outerPeerHash === origin &&
|
|
@@ -1490,7 +1887,9 @@ export class TopicControlPlane
|
|
|
1490
1887
|
// monotonic acceptance deadline. A current authenticated origin stream is
|
|
1491
1888
|
// sufficient liveness evidence; do not advance the floor or lease.
|
|
1492
1889
|
this.suppressedDepartedTopicRootCandidateClaims.delete(origin);
|
|
1493
|
-
|
|
1890
|
+
if (batchUpdate) batchUpdate.immediate = true;
|
|
1891
|
+
else
|
|
1892
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(now, { immediate: true });
|
|
1494
1893
|
return false;
|
|
1495
1894
|
}
|
|
1496
1895
|
|
|
@@ -1775,6 +2174,7 @@ export class TopicControlPlane
|
|
|
1775
2174
|
|
|
1776
2175
|
private async reconcileShardOverlays() {
|
|
1777
2176
|
if (!this.started || this.topicControlPlaneStopping) return;
|
|
2177
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
1778
2178
|
|
|
1779
2179
|
const byShard = new Map<string, string[]>();
|
|
1780
2180
|
for (const topic of this.subscriptions.keys()) {
|
|
@@ -1784,29 +2184,25 @@ export class TopicControlPlane
|
|
|
1784
2184
|
|
|
1785
2185
|
// Ensure shard overlays are joined using the current root mapping (may
|
|
1786
2186
|
// migrate channels if roots changed), then re-announce subscriptions.
|
|
1787
|
-
await Promise.
|
|
2187
|
+
const results = await Promise.allSettled(
|
|
1788
2188
|
[...byShard.entries()].map(async ([shardTopic, userTopics]) => {
|
|
1789
2189
|
if (userTopics.length === 0) return;
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
const msg = new Subscribe({
|
|
1793
|
-
topics: userTopics,
|
|
1794
|
-
requestSubscribers: true,
|
|
1795
|
-
});
|
|
1796
|
-
const embedded = await this.createMessage(
|
|
1797
|
-
this.encodePubSubMessage(msg),
|
|
1798
|
-
{
|
|
1799
|
-
mode: new AnyWhere(),
|
|
1800
|
-
priority: 1,
|
|
1801
|
-
skipRecipientValidation: true,
|
|
1802
|
-
} as any,
|
|
2190
|
+
const owners = userTopics.map(
|
|
2191
|
+
(topic) => [topic, this.subscriptions.get(topic)] as const,
|
|
1803
2192
|
);
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
2193
|
+
await this.ensureFanoutChannel(shardTopic, {
|
|
2194
|
+
ephemeral: false,
|
|
2195
|
+
isCurrent: () =>
|
|
2196
|
+
owners.some(
|
|
2197
|
+
([topic, owner]) =>
|
|
2198
|
+
owner && this.subscriptions.get(topic) === owner,
|
|
2199
|
+
),
|
|
2200
|
+
});
|
|
2201
|
+
await this.announceShardSubscriptions(shardTopic, lifecycleRevision);
|
|
1808
2202
|
}),
|
|
1809
2203
|
);
|
|
2204
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
2205
|
+
if (failure?.status === "rejected") throw failure.reason;
|
|
1810
2206
|
}
|
|
1811
2207
|
|
|
1812
2208
|
private isTrackedTopic(topic: string) {
|
|
@@ -1820,15 +2216,16 @@ export class TopicControlPlane
|
|
|
1820
2216
|
private untrackTopic(topic: string) {
|
|
1821
2217
|
const peers = this.topics.get(topic);
|
|
1822
2218
|
this.topics.delete(topic);
|
|
1823
|
-
|
|
1824
|
-
for (const peerHash of peers.keys()) {
|
|
2219
|
+
for (const peerHash of peers?.keys() ?? []) {
|
|
1825
2220
|
this.peerToTopic.get(peerHash)?.delete(topic);
|
|
1826
|
-
this.lastSubscriptionMessages.get(peerHash)?.delete(topic);
|
|
1827
2221
|
if (!this.peerToTopic.get(peerHash)?.size) {
|
|
1828
2222
|
this.peerToTopic.delete(peerHash);
|
|
1829
|
-
this.lastSubscriptionMessages.delete(peerHash);
|
|
1830
2223
|
}
|
|
1831
2224
|
}
|
|
2225
|
+
for (const [peerHash, messages] of this.lastSubscriptionMessages) {
|
|
2226
|
+
messages.delete(topic);
|
|
2227
|
+
if (messages.size === 0) this.lastSubscriptionMessages.delete(peerHash);
|
|
2228
|
+
}
|
|
1832
2229
|
}
|
|
1833
2230
|
|
|
1834
2231
|
private initializePeer(publicKey: PublicSignKey) {
|
|
@@ -1852,11 +2249,26 @@ export class TopicControlPlane
|
|
|
1852
2249
|
this.lastSubscriptionMessages.get(oldest)?.delete(topic);
|
|
1853
2250
|
if (!this.peerToTopic.get(oldest)?.size) {
|
|
1854
2251
|
this.peerToTopic.delete(oldest);
|
|
2252
|
+
}
|
|
2253
|
+
if (!this.lastSubscriptionMessages.get(oldest)?.size) {
|
|
1855
2254
|
this.lastSubscriptionMessages.delete(oldest);
|
|
1856
2255
|
}
|
|
1857
2256
|
}
|
|
1858
2257
|
}
|
|
1859
2258
|
|
|
2259
|
+
private pruneUnsubscribedWatermarks(topic: string) {
|
|
2260
|
+
let remaining = this.subscriberCacheMaxEntries;
|
|
2261
|
+
for (const [peerHash, messages] of [
|
|
2262
|
+
...this.lastSubscriptionMessages,
|
|
2263
|
+
].reverse()) {
|
|
2264
|
+
if (!messages.has(topic) || this.topics.get(topic)?.has(peerHash))
|
|
2265
|
+
continue;
|
|
2266
|
+
if (remaining-- > 0) continue;
|
|
2267
|
+
messages.delete(topic);
|
|
2268
|
+
if (messages.size === 0) this.lastSubscriptionMessages.delete(peerHash);
|
|
2269
|
+
}
|
|
2270
|
+
}
|
|
2271
|
+
|
|
1860
2272
|
private getSubscriptionOverlap(topics?: string[]) {
|
|
1861
2273
|
const subscriptions: string[] = [];
|
|
1862
2274
|
if (topics) {
|
|
@@ -1893,22 +2305,42 @@ export class TopicControlPlane
|
|
|
1893
2305
|
|
|
1894
2306
|
private scheduleFanoutIdleClose(topic: string) {
|
|
1895
2307
|
const st = this.fanoutChannels.get(topic);
|
|
1896
|
-
|
|
2308
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
2309
|
+
if (
|
|
2310
|
+
!st ||
|
|
2311
|
+
!st.ephemeral ||
|
|
2312
|
+
!this.isTopicControlPlaneActive(lifecycleRevision)
|
|
2313
|
+
)
|
|
2314
|
+
return;
|
|
1897
2315
|
this.clearFanoutIdleClose(st);
|
|
1898
2316
|
if (this.fanoutPublishIdleCloseMs <= 0) return;
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
2317
|
+
const timer = setTimeout(() => {
|
|
2318
|
+
// Cancellation can race a queued callback. It must still own this exact
|
|
2319
|
+
// timer, channel state and lifecycle before acting on the topic again.
|
|
2320
|
+
if (
|
|
2321
|
+
!this.isTopicControlPlaneActive(lifecycleRevision) ||
|
|
2322
|
+
this.fanoutChannels.get(topic) !== st ||
|
|
2323
|
+
st.idleCloseTimeout !== timer ||
|
|
2324
|
+
!st.ephemeral
|
|
2325
|
+
)
|
|
2326
|
+
return;
|
|
2327
|
+
st.idleCloseTimeout = undefined;
|
|
2328
|
+
const idleMs = Date.now() - st.lastUsedAt;
|
|
1903
2329
|
if (idleMs >= this.fanoutPublishIdleCloseMs) {
|
|
1904
2330
|
void this.closeFanoutChannel(topic);
|
|
1905
2331
|
return;
|
|
1906
2332
|
}
|
|
1907
2333
|
this.scheduleFanoutIdleClose(topic);
|
|
1908
2334
|
}, this.fanoutPublishIdleCloseMs);
|
|
2335
|
+
st.idleCloseTimeout = timer;
|
|
1909
2336
|
}
|
|
1910
2337
|
|
|
1911
|
-
private touchFanoutChannel(
|
|
2338
|
+
private touchFanoutChannel(
|
|
2339
|
+
topic: string,
|
|
2340
|
+
channel: FanoutChannel,
|
|
2341
|
+
lifecycleRevision: number,
|
|
2342
|
+
) {
|
|
2343
|
+
if (!this.isFanoutChannelCurrent(topic, channel, lifecycleRevision)) return;
|
|
1912
2344
|
const st = this.fanoutChannels.get(topic);
|
|
1913
2345
|
if (!st) return;
|
|
1914
2346
|
st.lastUsedAt = Date.now();
|
|
@@ -2162,7 +2594,7 @@ export class TopicControlPlane
|
|
|
2162
2594
|
private normalizePeerTopicRootState(
|
|
2163
2595
|
topic: string,
|
|
2164
2596
|
root: string,
|
|
2165
|
-
): { root: string
|
|
2597
|
+
): TopicRootState & { root: string } {
|
|
2166
2598
|
const deterministic =
|
|
2167
2599
|
this.topicRootControlPlane.resolveDeterministicTopicRoot(topic);
|
|
2168
2600
|
if (
|
|
@@ -2171,9 +2603,16 @@ export class TopicControlPlane
|
|
|
2171
2603
|
deterministic &&
|
|
2172
2604
|
root !== deterministic
|
|
2173
2605
|
) {
|
|
2174
|
-
return { root: deterministic, authoritative: false };
|
|
2606
|
+
return { root: deterministic, authoritative: false, autoCandidate: true };
|
|
2175
2607
|
}
|
|
2176
|
-
return {
|
|
2608
|
+
return {
|
|
2609
|
+
root,
|
|
2610
|
+
authoritative: true,
|
|
2611
|
+
autoCandidate:
|
|
2612
|
+
this.autoTopicRootCandidates &&
|
|
2613
|
+
topic.startsWith(this.shardTopicPrefix) &&
|
|
2614
|
+
root === deterministic,
|
|
2615
|
+
};
|
|
2177
2616
|
}
|
|
2178
2617
|
|
|
2179
2618
|
private async resolveTopicRootState(
|
|
@@ -2181,19 +2620,20 @@ export class TopicControlPlane
|
|
|
2181
2620
|
options?: TopicRootResolutionOptions & {
|
|
2182
2621
|
queryConnectedPeers?: boolean;
|
|
2183
2622
|
},
|
|
2184
|
-
): Promise<
|
|
2623
|
+
): Promise<TopicRootState> {
|
|
2185
2624
|
throwIfAborted(options?.signal);
|
|
2186
2625
|
const tracked = await this.topicRootControlPlane.resolveTrackedTopicRoot(
|
|
2187
2626
|
topic,
|
|
2188
2627
|
options,
|
|
2189
2628
|
);
|
|
2190
2629
|
if (tracked) {
|
|
2191
|
-
return { root: tracked, authoritative: true };
|
|
2630
|
+
return { root: tracked, authoritative: true, autoCandidate: false };
|
|
2192
2631
|
}
|
|
2193
2632
|
if (options?.queryConnectedPeers === false) {
|
|
2194
2633
|
return {
|
|
2195
2634
|
root: this.topicRootControlPlane.resolveDeterministicTopicRoot(topic),
|
|
2196
2635
|
authoritative: false,
|
|
2636
|
+
autoCandidate: this.autoTopicRootCandidates,
|
|
2197
2637
|
};
|
|
2198
2638
|
}
|
|
2199
2639
|
|
|
@@ -2269,6 +2709,7 @@ export class TopicControlPlane
|
|
|
2269
2709
|
return {
|
|
2270
2710
|
root: this.topicRootControlPlane.resolveDeterministicTopicRoot(topic),
|
|
2271
2711
|
authoritative: false,
|
|
2712
|
+
autoCandidate: this.autoTopicRootCandidates,
|
|
2272
2713
|
};
|
|
2273
2714
|
}
|
|
2274
2715
|
|
|
@@ -2292,7 +2733,11 @@ export class TopicControlPlane
|
|
|
2292
2733
|
options?: TopicRootResolutionOptions & {
|
|
2293
2734
|
queryConnectedPeers?: boolean;
|
|
2294
2735
|
},
|
|
2295
|
-
): Promise<{
|
|
2736
|
+
): Promise<{
|
|
2737
|
+
root: string;
|
|
2738
|
+
candidateGeneration: string;
|
|
2739
|
+
autoCandidate: boolean;
|
|
2740
|
+
}> {
|
|
2296
2741
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
2297
2742
|
for (;;) {
|
|
2298
2743
|
throwIfAborted(options?.signal);
|
|
@@ -2315,7 +2760,11 @@ export class TopicControlPlane
|
|
|
2315
2760
|
? undefined
|
|
2316
2761
|
: this.shardRootCache.get(shardTopic);
|
|
2317
2762
|
if (cached && (cached.authoritative || !hasConnectedTrackers)) {
|
|
2318
|
-
return {
|
|
2763
|
+
return {
|
|
2764
|
+
root: cached.root,
|
|
2765
|
+
candidateGeneration,
|
|
2766
|
+
autoCandidate: cached.autoCandidate,
|
|
2767
|
+
};
|
|
2319
2768
|
}
|
|
2320
2769
|
|
|
2321
2770
|
const resolved = await this.resolveTopicRootState(shardTopic, options);
|
|
@@ -2331,15 +2780,16 @@ export class TopicControlPlane
|
|
|
2331
2780
|
if (resolved.authoritative || !hasConnectedTrackers) {
|
|
2332
2781
|
this.shardRootCache.set(
|
|
2333
2782
|
shardTopic,
|
|
2334
|
-
resolved as {
|
|
2335
|
-
root: string;
|
|
2336
|
-
authoritative: boolean;
|
|
2337
|
-
},
|
|
2783
|
+
resolved as TopicRootState & { root: string },
|
|
2338
2784
|
);
|
|
2339
2785
|
} else {
|
|
2340
2786
|
this.shardRootCache.delete(shardTopic);
|
|
2341
2787
|
}
|
|
2342
|
-
return {
|
|
2788
|
+
return {
|
|
2789
|
+
root: resolved.root,
|
|
2790
|
+
candidateGeneration,
|
|
2791
|
+
autoCandidate: resolved.autoCandidate,
|
|
2792
|
+
};
|
|
2343
2793
|
}
|
|
2344
2794
|
}
|
|
2345
2795
|
|
|
@@ -2372,6 +2822,11 @@ export class TopicControlPlane
|
|
|
2372
2822
|
pubsubMessage: PubSubMessage,
|
|
2373
2823
|
signal?: AbortSignal,
|
|
2374
2824
|
) {
|
|
2825
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
2826
|
+
const isCurrent = () =>
|
|
2827
|
+
this.isTopicControlPlaneActive(revision) &&
|
|
2828
|
+
this.peers.get(peer.publicKey.hashcode()) === peer;
|
|
2829
|
+
if (!isCurrent()) return;
|
|
2375
2830
|
const embedded = await this.createMessage(
|
|
2376
2831
|
this.encodePubSubMessage(pubsubMessage),
|
|
2377
2832
|
{
|
|
@@ -2383,12 +2838,13 @@ export class TopicControlPlane
|
|
|
2383
2838
|
skipRecipientValidation: true,
|
|
2384
2839
|
} as any,
|
|
2385
2840
|
);
|
|
2841
|
+
if (!isCurrent()) return;
|
|
2386
2842
|
await this.publishMessage(
|
|
2387
2843
|
this.publicKey,
|
|
2388
2844
|
embedded,
|
|
2389
2845
|
[peer],
|
|
2390
2846
|
undefined,
|
|
2391
|
-
signal,
|
|
2847
|
+
signal ?? this.topicRootResolutionAbortController.signal,
|
|
2392
2848
|
);
|
|
2393
2849
|
}
|
|
2394
2850
|
|
|
@@ -2527,6 +2983,7 @@ export class TopicControlPlane
|
|
|
2527
2983
|
this.shardRootCache.set(shardTopic, {
|
|
2528
2984
|
root: resolved.root,
|
|
2529
2985
|
authoritative: true,
|
|
2986
|
+
autoCandidate: resolved.autoCandidate,
|
|
2530
2987
|
});
|
|
2531
2988
|
}
|
|
2532
2989
|
return resolved.root;
|
|
@@ -2534,20 +2991,55 @@ export class TopicControlPlane
|
|
|
2534
2991
|
|
|
2535
2992
|
private async resolveQueryableTopicRoot(
|
|
2536
2993
|
topic: string,
|
|
2994
|
+
isCurrent: () => boolean = () => true,
|
|
2537
2995
|
): Promise<string | undefined> {
|
|
2538
2996
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
2539
2997
|
const lifecycleSignal = this.topicRootResolutionAbortController.signal;
|
|
2998
|
+
const recovery: RelayRootRecoveryAttempt = {};
|
|
2540
2999
|
return this.withTopicRootCandidateResolution(
|
|
2541
3000
|
async ({ candidateGeneration, signal }) => {
|
|
2542
3001
|
throwIfAborted(signal);
|
|
2543
3002
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
3003
|
+
if (!isCurrent()) throw new AbortError("obsolete topic root query");
|
|
3004
|
+
// Incoming queries retain canonical directory semantics: never ask
|
|
3005
|
+
// trackers here, since a tracker may itself query this peer.
|
|
3006
|
+
const resolveCanonical = () =>
|
|
3007
|
+
this.withRelayRootPolicyDeadline(
|
|
3008
|
+
recovery,
|
|
2547
3009
|
signal,
|
|
2548
|
-
|
|
2549
|
-
|
|
3010
|
+
async (policySignal) => {
|
|
3011
|
+
const local =
|
|
3012
|
+
await this.topicRootControlPlane.resolveLocalTopicRoot(topic, {
|
|
3013
|
+
signal: policySignal,
|
|
3014
|
+
});
|
|
3015
|
+
return {
|
|
3016
|
+
root:
|
|
3017
|
+
local ??
|
|
3018
|
+
this.topicRootControlPlane.resolveDeterministicTopicRoot(
|
|
3019
|
+
topic,
|
|
3020
|
+
),
|
|
3021
|
+
autoCandidate: !local && this.autoTopicRootCandidates,
|
|
3022
|
+
};
|
|
3023
|
+
},
|
|
3024
|
+
);
|
|
3025
|
+
let resolved = await resolveCanonical();
|
|
3026
|
+
if (resolved.autoCandidate && topic.startsWith(this.shardTopicPrefix)) {
|
|
3027
|
+
if (
|
|
3028
|
+
await this.preflightRelayRootCandidates(
|
|
3029
|
+
topic,
|
|
3030
|
+
recovery,
|
|
3031
|
+
signal,
|
|
3032
|
+
isCurrent,
|
|
3033
|
+
true,
|
|
3034
|
+
)
|
|
3035
|
+
) {
|
|
3036
|
+
throwIfAborted(signal);
|
|
3037
|
+
resolved = await resolveCanonical();
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
const root = resolved.root;
|
|
2550
3041
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
3042
|
+
if (!isCurrent()) throw new AbortError("obsolete topic root query");
|
|
2551
3043
|
if (root !== this.publicKeyHash) {
|
|
2552
3044
|
return root;
|
|
2553
3045
|
}
|
|
@@ -2677,8 +3169,11 @@ export class TopicControlPlane
|
|
|
2677
3169
|
): Promise<void> {
|
|
2678
3170
|
const topic = shardTopic.toString();
|
|
2679
3171
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
3172
|
+
const recovery: RelayRootRecoveryAttempt = {};
|
|
2680
3173
|
for (;;) {
|
|
2681
3174
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
3175
|
+
if (options?.isCurrent?.() === false)
|
|
3176
|
+
throw new AbortError("obsolete fanout channel owner");
|
|
2682
3177
|
if (options?.signal?.aborted) {
|
|
2683
3178
|
throw (
|
|
2684
3179
|
options.signal.reason ??
|
|
@@ -2689,13 +3184,20 @@ export class TopicControlPlane
|
|
|
2689
3184
|
const active = this.ensureFanoutChannelInFlight.get(topic);
|
|
2690
3185
|
if (
|
|
2691
3186
|
active?.candidateGeneration === candidateGeneration &&
|
|
2692
|
-
active.lifecycleRevision === lifecycleRevision
|
|
3187
|
+
active.lifecycleRevision === lifecycleRevision &&
|
|
3188
|
+
active.isCurrent()
|
|
2693
3189
|
) {
|
|
2694
3190
|
try {
|
|
2695
3191
|
await withAbort(active.opening, options?.signal);
|
|
2696
3192
|
} catch (error) {
|
|
2697
3193
|
if (options?.signal?.aborted) throw error;
|
|
2698
3194
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
3195
|
+
if (
|
|
3196
|
+
candidateGeneration === this.getTopicRootCandidateGeneration() &&
|
|
3197
|
+
!active.abortController.signal.aborted &&
|
|
3198
|
+
active.isCurrent()
|
|
3199
|
+
)
|
|
3200
|
+
throw error;
|
|
2699
3201
|
}
|
|
2700
3202
|
await withAbort(active.settled, options?.signal);
|
|
2701
3203
|
if (this.ensureFanoutChannelInFlight.get(topic) === active) {
|
|
@@ -2729,11 +3231,21 @@ export class TopicControlPlane
|
|
|
2729
3231
|
});
|
|
2730
3232
|
}
|
|
2731
3233
|
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
3234
|
+
// Publish the opening identity before even a synchronously resolved root
|
|
3235
|
+
// can enter its guarded recovery/creation continuation.
|
|
3236
|
+
const opening = Promise.resolve().then(() =>
|
|
3237
|
+
this.ensureFanoutChannelOnce(
|
|
3238
|
+
topic,
|
|
3239
|
+
{ ...options, signal: abortController.signal },
|
|
3240
|
+
lifecycleRevision,
|
|
3241
|
+
candidateGeneration,
|
|
3242
|
+
recovery,
|
|
3243
|
+
() =>
|
|
3244
|
+
this.ensureFanoutChannelInFlight.get(topic)?.abortController ===
|
|
3245
|
+
abortController &&
|
|
3246
|
+
!abortController.signal.aborted &&
|
|
3247
|
+
options?.isCurrent?.() !== false,
|
|
3248
|
+
),
|
|
2737
3249
|
);
|
|
2738
3250
|
let resolveSettled!: () => void;
|
|
2739
3251
|
const settled = new Promise<void>((resolve) => {
|
|
@@ -2741,6 +3253,7 @@ export class TopicControlPlane
|
|
|
2741
3253
|
});
|
|
2742
3254
|
const inFlight = {
|
|
2743
3255
|
abortController,
|
|
3256
|
+
isCurrent: () => options?.isCurrent?.() !== false,
|
|
2744
3257
|
candidateGeneration,
|
|
2745
3258
|
lifecycleRevision,
|
|
2746
3259
|
opening,
|
|
@@ -2785,6 +3298,8 @@ export class TopicControlPlane
|
|
|
2785
3298
|
options: EnsureFanoutChannelOptions | undefined,
|
|
2786
3299
|
lifecycleRevision: number,
|
|
2787
3300
|
candidateGeneration: string,
|
|
3301
|
+
recovery: RelayRootRecoveryAttempt = {},
|
|
3302
|
+
isOpeningCurrent: () => boolean = () => true,
|
|
2788
3303
|
): Promise<void> {
|
|
2789
3304
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2790
3305
|
if (candidateGeneration !== this.getTopicRootCandidateGeneration()) return;
|
|
@@ -2806,15 +3321,19 @@ export class TopicControlPlane
|
|
|
2806
3321
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2807
3322
|
let resolvedGeneration = candidateGeneration;
|
|
2808
3323
|
let root = suppliedRoot;
|
|
3324
|
+
let autoCandidate = false;
|
|
3325
|
+
const resolveRoot = () =>
|
|
3326
|
+
this.withRelayRootPolicyDeadline(recovery, options?.signal, (signal) =>
|
|
3327
|
+
this.resolveShardRootState(t, { signal }),
|
|
3328
|
+
);
|
|
2809
3329
|
const existing = this.fanoutChannels.get(t);
|
|
2810
3330
|
if (existing) {
|
|
2811
3331
|
if (!root) {
|
|
2812
|
-
const resolved = await
|
|
2813
|
-
signal: options?.signal,
|
|
2814
|
-
});
|
|
3332
|
+
const resolved = await resolveRoot();
|
|
2815
3333
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2816
3334
|
root = resolved.root;
|
|
2817
3335
|
resolvedGeneration = resolved.candidateGeneration;
|
|
3336
|
+
autoCandidate = resolved.autoCandidate;
|
|
2818
3337
|
}
|
|
2819
3338
|
if (
|
|
2820
3339
|
resolvedGeneration !== candidateGeneration ||
|
|
@@ -2855,21 +3374,45 @@ export class TopicControlPlane
|
|
|
2855
3374
|
}
|
|
2856
3375
|
|
|
2857
3376
|
if (!root) {
|
|
2858
|
-
const resolved = await
|
|
2859
|
-
signal: options?.signal,
|
|
2860
|
-
});
|
|
3377
|
+
const resolved = await resolveRoot();
|
|
2861
3378
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2862
3379
|
root = resolved.root;
|
|
2863
3380
|
resolvedGeneration = resolved.candidateGeneration;
|
|
3381
|
+
autoCandidate = resolved.autoCandidate;
|
|
3382
|
+
}
|
|
3383
|
+
if (
|
|
3384
|
+
autoCandidate &&
|
|
3385
|
+
(await this.preflightRelayRootCandidates(
|
|
3386
|
+
t,
|
|
3387
|
+
recovery,
|
|
3388
|
+
options?.signal,
|
|
3389
|
+
isOpeningCurrent,
|
|
3390
|
+
))
|
|
3391
|
+
) {
|
|
3392
|
+
throwIfAborted(options?.signal);
|
|
3393
|
+
if (candidateGeneration !== this.getTopicRootCandidateGeneration())
|
|
3394
|
+
return;
|
|
3395
|
+
const resolved = await resolveRoot();
|
|
3396
|
+
root = resolved.root;
|
|
3397
|
+
resolvedGeneration = resolved.candidateGeneration;
|
|
3398
|
+
autoCandidate = resolved.autoCandidate;
|
|
3399
|
+
}
|
|
3400
|
+
if (
|
|
3401
|
+
suppliedRoot !== undefined ||
|
|
3402
|
+
!this.autoTopicRootCandidates ||
|
|
3403
|
+
autoCandidate
|
|
3404
|
+
) {
|
|
3405
|
+
root = await this.confirmDirectShardRoot(
|
|
3406
|
+
t,
|
|
3407
|
+
root,
|
|
3408
|
+
resolvedGeneration,
|
|
3409
|
+
lifecycleRevision,
|
|
3410
|
+
options?.signal,
|
|
3411
|
+
);
|
|
2864
3412
|
}
|
|
2865
|
-
root = await this.confirmDirectShardRoot(
|
|
2866
|
-
t,
|
|
2867
|
-
root,
|
|
2868
|
-
resolvedGeneration,
|
|
2869
|
-
lifecycleRevision,
|
|
2870
|
-
options?.signal,
|
|
2871
|
-
);
|
|
2872
3413
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
3414
|
+
if (!isOpeningCurrent())
|
|
3415
|
+
throw new AbortError("obsolete fanout channel owner");
|
|
2873
3416
|
if (
|
|
2874
3417
|
resolvedGeneration !== candidateGeneration ||
|
|
2875
3418
|
candidateGeneration !== this.getTopicRootCandidateGeneration()
|
|
@@ -3053,7 +3596,7 @@ export class TopicControlPlane
|
|
|
3053
3596
|
join
|
|
3054
3597
|
.then(() => {
|
|
3055
3598
|
const st = this.fanoutChannels.get(t);
|
|
3056
|
-
if (st === channelState && st.ephemeral) {
|
|
3599
|
+
if (st === channelState && st.ephemeral && isCurrent()) {
|
|
3057
3600
|
this.scheduleFanoutIdleClose(t);
|
|
3058
3601
|
}
|
|
3059
3602
|
})
|
|
@@ -3197,6 +3740,117 @@ export class TopicControlPlane
|
|
|
3197
3740
|
return this.debounceSubscribeAggregator.add({ key: topic });
|
|
3198
3741
|
}
|
|
3199
3742
|
|
|
3743
|
+
private async announceShardSubscriptions(
|
|
3744
|
+
shardTopic: string,
|
|
3745
|
+
lifecycleRevision: number,
|
|
3746
|
+
) {
|
|
3747
|
+
const st = this.fanoutChannels.get(shardTopic);
|
|
3748
|
+
if (!st) return;
|
|
3749
|
+
let topics = [...this.subscriptions.entries()].filter(
|
|
3750
|
+
([topic]) => this.getShardTopicForUserTopic(topic) === shardTopic,
|
|
3751
|
+
);
|
|
3752
|
+
while (
|
|
3753
|
+
topics.length > 0 &&
|
|
3754
|
+
this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)
|
|
3755
|
+
) {
|
|
3756
|
+
const embedded = await this.createMessage(
|
|
3757
|
+
this.encodePubSubMessage(
|
|
3758
|
+
new Subscribe({
|
|
3759
|
+
topics: topics.map(([topic]) => topic),
|
|
3760
|
+
requestSubscribers: true,
|
|
3761
|
+
}),
|
|
3762
|
+
),
|
|
3763
|
+
{
|
|
3764
|
+
mode: new AnyWhere(),
|
|
3765
|
+
priority: 1,
|
|
3766
|
+
skipRecipientValidation: true,
|
|
3767
|
+
} as any,
|
|
3768
|
+
);
|
|
3769
|
+
if (
|
|
3770
|
+
!this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)
|
|
3771
|
+
)
|
|
3772
|
+
return;
|
|
3773
|
+
const current = topics.filter(
|
|
3774
|
+
([topic, sub]) => this.subscriptions.get(topic) === sub,
|
|
3775
|
+
);
|
|
3776
|
+
if (current.length !== topics.length) {
|
|
3777
|
+
// Never announce a subscription removed while signing. This retry
|
|
3778
|
+
// only shrinks the original batch, so churn cannot extend it forever.
|
|
3779
|
+
topics = current;
|
|
3780
|
+
continue;
|
|
3781
|
+
}
|
|
3782
|
+
await st.channel.publish(toUint8Array(embedded.bytes()));
|
|
3783
|
+
if (
|
|
3784
|
+
this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)
|
|
3785
|
+
) {
|
|
3786
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
3787
|
+
}
|
|
3788
|
+
return;
|
|
3789
|
+
}
|
|
3790
|
+
}
|
|
3791
|
+
|
|
3792
|
+
private async announceDirectSubscriptions(
|
|
3793
|
+
topics: string[],
|
|
3794
|
+
peers = [...this.peers.values()],
|
|
3795
|
+
subscribed = true,
|
|
3796
|
+
) {
|
|
3797
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
3798
|
+
const signal = this.topicRootResolutionAbortController.signal;
|
|
3799
|
+
const subscriptions = new Map(
|
|
3800
|
+
topics.map((topic) => [topic, this.subscriptions.get(topic)]),
|
|
3801
|
+
);
|
|
3802
|
+
const isCurrentTopic = (topic: string) =>
|
|
3803
|
+
subscribed
|
|
3804
|
+
? subscriptions.get(topic) != null &&
|
|
3805
|
+
this.subscriptions.get(topic) === subscriptions.get(topic)
|
|
3806
|
+
: !this.subscriptions.has(topic);
|
|
3807
|
+
await Promise.all(
|
|
3808
|
+
peers
|
|
3809
|
+
.map(async (peer) => {
|
|
3810
|
+
const isCurrent = () =>
|
|
3811
|
+
this.isTopicControlPlaneActive(revision) &&
|
|
3812
|
+
this.peers.get(peer.publicKey.hashcode()) === peer &&
|
|
3813
|
+
peer.isWritable;
|
|
3814
|
+
let currentTopics = topics.filter(isCurrentTopic);
|
|
3815
|
+
while (isCurrent() && currentTopics.length > 0) {
|
|
3816
|
+
const embedded = await this.createMessage(
|
|
3817
|
+
this.encodePubSubMessage(
|
|
3818
|
+
subscribed
|
|
3819
|
+
? new Subscribe({
|
|
3820
|
+
topics: currentTopics,
|
|
3821
|
+
requestSubscribers: true,
|
|
3822
|
+
})
|
|
3823
|
+
: new Unsubscribe({ topics: currentTopics }),
|
|
3824
|
+
),
|
|
3825
|
+
{
|
|
3826
|
+
mode: new SilentDelivery({
|
|
3827
|
+
to: [peer.publicKey.hashcode()],
|
|
3828
|
+
redundancy: 1,
|
|
3829
|
+
}),
|
|
3830
|
+
priority: 1,
|
|
3831
|
+
skipRecipientValidation: true,
|
|
3832
|
+
} as any,
|
|
3833
|
+
);
|
|
3834
|
+
if (!isCurrent()) return;
|
|
3835
|
+
const remaining = currentTopics.filter(isCurrentTopic);
|
|
3836
|
+
if (remaining.length !== currentTopics.length) {
|
|
3837
|
+
currentTopics = remaining;
|
|
3838
|
+
continue;
|
|
3839
|
+
}
|
|
3840
|
+
await this.publishMessage(
|
|
3841
|
+
this.publicKey,
|
|
3842
|
+
embedded,
|
|
3843
|
+
[peer],
|
|
3844
|
+
undefined,
|
|
3845
|
+
signal,
|
|
3846
|
+
);
|
|
3847
|
+
return;
|
|
3848
|
+
}
|
|
3849
|
+
})
|
|
3850
|
+
.map((task) => task.catch(logErrorIfStarted)),
|
|
3851
|
+
);
|
|
3852
|
+
}
|
|
3853
|
+
|
|
3200
3854
|
private async _subscribe(
|
|
3201
3855
|
topics: { key: string; counter: number }[],
|
|
3202
3856
|
lifecycleRevision = this.topicControlPlaneLifecycleRevision,
|
|
@@ -3206,7 +3860,6 @@ export class TopicControlPlane
|
|
|
3206
3860
|
if (topics.length === 0) return;
|
|
3207
3861
|
|
|
3208
3862
|
const byShard = new Map<string, string[]>();
|
|
3209
|
-
const joins: Promise<void>[] = [];
|
|
3210
3863
|
for (const { key: topic, counter } of topics) {
|
|
3211
3864
|
let prev = this.subscriptions.get(topic);
|
|
3212
3865
|
if (prev) {
|
|
@@ -3224,46 +3877,35 @@ export class TopicControlPlane
|
|
|
3224
3877
|
shardTopic,
|
|
3225
3878
|
(this.shardRefCounts.get(shardTopic) ?? 0) + 1,
|
|
3226
3879
|
);
|
|
3227
|
-
joins.push(this.ensureFanoutChannel(shardTopic));
|
|
3228
3880
|
}
|
|
3229
3881
|
|
|
3230
|
-
|
|
3231
|
-
|
|
3882
|
+
// Neighbours can use different root policies. Discover them independently
|
|
3883
|
+
// of shard joins, without adopting or overriding either peer's root policy.
|
|
3884
|
+
void this.announceDirectSubscriptions([...byShard.values()].flat()).catch(
|
|
3885
|
+
logErrorIfStarted,
|
|
3886
|
+
);
|
|
3232
3887
|
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3888
|
+
const results = await Promise.allSettled(
|
|
3889
|
+
[...byShard.keys()].map(async (shardTopic) => {
|
|
3890
|
+
const owners = byShard
|
|
3891
|
+
.get(shardTopic)!
|
|
3892
|
+
.map((topic) => [topic, this.subscriptions.get(topic)] as const);
|
|
3893
|
+
await this.ensureFanoutChannel(shardTopic, {
|
|
3894
|
+
isCurrent: () =>
|
|
3895
|
+
owners.some(
|
|
3896
|
+
([topic, owner]) =>
|
|
3897
|
+
owner && this.subscriptions.get(topic) === owner,
|
|
3898
|
+
),
|
|
3243
3899
|
});
|
|
3244
|
-
|
|
3245
|
-
this.encodePubSubMessage(msg),
|
|
3246
|
-
{
|
|
3247
|
-
mode: new AnyWhere(),
|
|
3248
|
-
priority: 1,
|
|
3249
|
-
skipRecipientValidation: true,
|
|
3250
|
-
} as any,
|
|
3251
|
-
);
|
|
3252
|
-
if (
|
|
3253
|
-
!this.isFanoutChannelCurrent(
|
|
3254
|
-
shardTopic,
|
|
3255
|
-
st.channel,
|
|
3256
|
-
lifecycleRevision,
|
|
3257
|
-
)
|
|
3258
|
-
)
|
|
3259
|
-
return;
|
|
3260
|
-
await st.channel.publish(toUint8Array(embedded.bytes()));
|
|
3261
|
-
if (
|
|
3262
|
-
this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)
|
|
3263
|
-
)
|
|
3264
|
-
this.touchFanoutChannel(shardTopic);
|
|
3900
|
+
await this.announceShardSubscriptions(shardTopic, lifecycleRevision);
|
|
3265
3901
|
}),
|
|
3266
3902
|
);
|
|
3903
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
3904
|
+
if (failure?.status === "rejected") {
|
|
3905
|
+
if (this.isTopicControlPlaneActive(lifecycleRevision))
|
|
3906
|
+
this.scheduleReconcileShardOverlays();
|
|
3907
|
+
throw failure.reason;
|
|
3908
|
+
}
|
|
3267
3909
|
}
|
|
3268
3910
|
|
|
3269
3911
|
async unsubscribe(
|
|
@@ -3304,6 +3946,12 @@ export class TopicControlPlane
|
|
|
3304
3946
|
const next = (this.shardRefCounts.get(shardTopic) ?? 0) - 1;
|
|
3305
3947
|
if (next <= 0) {
|
|
3306
3948
|
this.shardRefCounts.delete(shardTopic);
|
|
3949
|
+
const opening = this.ensureFanoutChannelInFlight.get(shardTopic);
|
|
3950
|
+
if (opening && !opening.isCurrent()) {
|
|
3951
|
+
opening.abortController.abort(
|
|
3952
|
+
new AbortError("last shard subscription removed"),
|
|
3953
|
+
);
|
|
3954
|
+
}
|
|
3307
3955
|
} else {
|
|
3308
3956
|
this.shardRefCounts.set(shardTopic, next);
|
|
3309
3957
|
}
|
|
@@ -3324,12 +3972,18 @@ export class TopicControlPlane
|
|
|
3324
3972
|
|
|
3325
3973
|
const byShard = new Map<string, string[]>();
|
|
3326
3974
|
for (const { key: topic } of topics) {
|
|
3327
|
-
//
|
|
3975
|
+
// A pending reopen must not erase the previous departure: its Subscribe
|
|
3976
|
+
// shares the transport session and would otherwise only refresh the cache.
|
|
3328
3977
|
if (this.subscriptions.has(topic)) continue;
|
|
3329
3978
|
const shardTopic = this.getShardTopicForUserTopic(topic);
|
|
3330
3979
|
byShard.set(shardTopic, [...(byShard.get(shardTopic) ?? []), topic]);
|
|
3331
3980
|
}
|
|
3332
3981
|
|
|
3982
|
+
void this.announceDirectSubscriptions(
|
|
3983
|
+
[...byShard.values()].flat(),
|
|
3984
|
+
undefined,
|
|
3985
|
+
false,
|
|
3986
|
+
).catch(logErrorIfStarted);
|
|
3333
3987
|
await Promise.all(
|
|
3334
3988
|
[...byShard.entries()].map(async ([shardTopic, userTopics]) => {
|
|
3335
3989
|
if (userTopics.length === 0) return;
|
|
@@ -3365,7 +4019,11 @@ export class TopicControlPlane
|
|
|
3365
4019
|
lifecycleRevision,
|
|
3366
4020
|
)
|
|
3367
4021
|
)
|
|
3368
|
-
this.touchFanoutChannel(
|
|
4022
|
+
this.touchFanoutChannel(
|
|
4023
|
+
shardTopic,
|
|
4024
|
+
st.channel,
|
|
4025
|
+
lifecycleRevision,
|
|
4026
|
+
);
|
|
3369
4027
|
}
|
|
3370
4028
|
} catch {
|
|
3371
4029
|
// best-effort
|
|
@@ -3396,6 +4054,7 @@ export class TopicControlPlane
|
|
|
3396
4054
|
batches: { session: bigint; timestamp: bigint; topics: string[] }[],
|
|
3397
4055
|
) {
|
|
3398
4056
|
if (!this.started) throw new NotStartedError();
|
|
4057
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
3399
4058
|
|
|
3400
4059
|
const byShard = new Map<
|
|
3401
4060
|
string,
|
|
@@ -3446,7 +4105,7 @@ export class TopicControlPlane
|
|
|
3446
4105
|
const st = this.fanoutChannels.get(shardTopic);
|
|
3447
4106
|
if (st) {
|
|
3448
4107
|
void st.channel.publishMaybe(toUint8Array(embedded.bytes()));
|
|
3449
|
-
this.touchFanoutChannel(shardTopic);
|
|
4108
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
3450
4109
|
}
|
|
3451
4110
|
} catch {
|
|
3452
4111
|
// best-effort
|
|
@@ -3460,6 +4119,7 @@ export class TopicControlPlane
|
|
|
3460
4119
|
shardTopic: string,
|
|
3461
4120
|
) {
|
|
3462
4121
|
if (!this.started) throw new NotStartedError();
|
|
4122
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
3463
4123
|
try {
|
|
3464
4124
|
const msg = new PeerUnavailable({
|
|
3465
4125
|
publicKeyHash,
|
|
@@ -3476,7 +4136,7 @@ export class TopicControlPlane
|
|
|
3476
4136
|
const st = this.fanoutChannels.get(shardTopic);
|
|
3477
4137
|
if (st) {
|
|
3478
4138
|
void st.channel.publishMaybe(toUint8Array(embedded.bytes()));
|
|
3479
|
-
this.touchFanoutChannel(shardTopic);
|
|
4139
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
3480
4140
|
}
|
|
3481
4141
|
} catch {
|
|
3482
4142
|
// best-effort
|
|
@@ -3543,6 +4203,7 @@ export class TopicControlPlane
|
|
|
3543
4203
|
to?: PublicSignKey,
|
|
3544
4204
|
): Promise<void> {
|
|
3545
4205
|
if (!this.started) throw new NotStartedError();
|
|
4206
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
3546
4207
|
if (topic == null) throw new Error("ERR_NOT_VALID_TOPIC");
|
|
3547
4208
|
if (topic.length === 0) return;
|
|
3548
4209
|
|
|
@@ -3594,7 +4255,7 @@ export class TopicControlPlane
|
|
|
3594
4255
|
} else {
|
|
3595
4256
|
await st.channel.publish(payload);
|
|
3596
4257
|
}
|
|
3597
|
-
this.touchFanoutChannel(shardTopic);
|
|
4258
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
3598
4259
|
}),
|
|
3599
4260
|
);
|
|
3600
4261
|
}
|
|
@@ -3612,6 +4273,7 @@ export class TopicControlPlane
|
|
|
3612
4273
|
WithExtraSigners & { signal?: AbortSignal },
|
|
3613
4274
|
): Promise<Uint8Array | undefined> {
|
|
3614
4275
|
if (!this.started) throw new NotStartedError();
|
|
4276
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
3615
4277
|
|
|
3616
4278
|
const topicsAll =
|
|
3617
4279
|
(options as { topics: string[] }).topics?.map((x) => x.toString()) || [];
|
|
@@ -3719,7 +4381,7 @@ export class TopicControlPlane
|
|
|
3719
4381
|
throw new Error(`Fanout channel missing for shard: ${shardTopic}`);
|
|
3720
4382
|
}
|
|
3721
4383
|
await withAbort(st.channel.publish(payload), options?.signal);
|
|
3722
|
-
this.touchFanoutChannel(shardTopic);
|
|
4384
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
3723
4385
|
}),
|
|
3724
4386
|
);
|
|
3725
4387
|
|
|
@@ -3819,6 +4481,18 @@ export class TopicControlPlane
|
|
|
3819
4481
|
|
|
3820
4482
|
public override onPeerUnreachable(publicKeyHash: string) {
|
|
3821
4483
|
super.onPeerUnreachable(publicKeyHash);
|
|
4484
|
+
const claim = this.signedTopicRootCandidateClaims.get(publicKeyHash);
|
|
4485
|
+
if (
|
|
4486
|
+
this.isTopicControlPlaneActive(this.topicControlPlaneLifecycleRevision) &&
|
|
4487
|
+
!this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured() &&
|
|
4488
|
+
claim &&
|
|
4489
|
+
!this.routes.isReachable(this.publicKeyHash, publicKeyHash) &&
|
|
4490
|
+
this.suppressTopicRootCandidateClaim(publicKeyHash, claim)
|
|
4491
|
+
) {
|
|
4492
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
4493
|
+
immediate: true,
|
|
4494
|
+
});
|
|
4495
|
+
}
|
|
3822
4496
|
const key = this.peerKeyHashToPublicKey.get(publicKeyHash);
|
|
3823
4497
|
if (!key) {
|
|
3824
4498
|
return;
|
|
@@ -4177,16 +4851,24 @@ export class TopicControlPlane
|
|
|
4177
4851
|
stream: PeerStreams;
|
|
4178
4852
|
}): Promise<void> {
|
|
4179
4853
|
const { pubsubMessage, message, from, stream } = input;
|
|
4180
|
-
|
|
4854
|
+
if (
|
|
4855
|
+
(pubsubMessage instanceof Subscribe ||
|
|
4856
|
+
pubsubMessage instanceof Unsubscribe) &&
|
|
4857
|
+
this.peers.get(stream.publicKey.hashcode()) !== stream
|
|
4858
|
+
)
|
|
4859
|
+
return;
|
|
4860
|
+
const isBoundDirectControl =
|
|
4181
4861
|
pubsubMessage instanceof TopicRootCandidates ||
|
|
4182
4862
|
pubsubMessage instanceof TopicRootCandidateClaims ||
|
|
4183
4863
|
pubsubMessage instanceof TopicRootQuery ||
|
|
4184
|
-
pubsubMessage instanceof TopicRootQueryResponse
|
|
4185
|
-
|
|
4864
|
+
pubsubMessage instanceof TopicRootQueryResponse ||
|
|
4865
|
+
pubsubMessage instanceof Subscribe ||
|
|
4866
|
+
pubsubMessage instanceof Unsubscribe;
|
|
4867
|
+
const directRootSigner = isBoundDirectControl
|
|
4186
4868
|
? message.header.signatures?.publicKeys[0]
|
|
4187
4869
|
: undefined;
|
|
4188
4870
|
if (
|
|
4189
|
-
|
|
4871
|
+
isBoundDirectControl &&
|
|
4190
4872
|
(!directRootSigner || !directRootSigner.equals(stream.publicKey))
|
|
4191
4873
|
) {
|
|
4192
4874
|
return;
|
|
@@ -4209,6 +4891,8 @@ export class TopicControlPlane
|
|
|
4209
4891
|
return;
|
|
4210
4892
|
}
|
|
4211
4893
|
const imported: Uint8Array[] = [];
|
|
4894
|
+
const batchUpdate = { immediate: false };
|
|
4895
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
4212
4896
|
const outerPeerHash = stream.publicKey.hashcode();
|
|
4213
4897
|
for (const claim of pubsubMessage.claims) {
|
|
4214
4898
|
if (
|
|
@@ -4216,13 +4900,24 @@ export class TopicControlPlane
|
|
|
4216
4900
|
claim,
|
|
4217
4901
|
outerPeerHash,
|
|
4218
4902
|
stream,
|
|
4903
|
+
batchUpdate,
|
|
4219
4904
|
)
|
|
4220
4905
|
) {
|
|
4221
4906
|
imported.push(claim);
|
|
4222
4907
|
}
|
|
4223
4908
|
}
|
|
4909
|
+
if (
|
|
4910
|
+
!this.isTopicControlPlaneActive(revision) ||
|
|
4911
|
+
this.peers.get(outerPeerHash) !== stream
|
|
4912
|
+
)
|
|
4913
|
+
return;
|
|
4914
|
+
if (imported.length > 0 || batchUpdate.immediate) {
|
|
4915
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(
|
|
4916
|
+
BigInt(Date.now()),
|
|
4917
|
+
batchUpdate,
|
|
4918
|
+
);
|
|
4919
|
+
}
|
|
4224
4920
|
if (imported.length > 0) {
|
|
4225
|
-
this.rebuildAutoTopicRootCandidatesFromClaims();
|
|
4226
4921
|
// Relay the exact nested claims; no relay signature is substituted for
|
|
4227
4922
|
// their origin signatures.
|
|
4228
4923
|
void this.sendSignedTopicRootCandidateClaims(
|
|
@@ -4236,7 +4931,10 @@ export class TopicControlPlane
|
|
|
4236
4931
|
}
|
|
4237
4932
|
|
|
4238
4933
|
if (pubsubMessage instanceof TopicRootQuery) {
|
|
4239
|
-
const root = await this.resolveQueryableTopicRoot(
|
|
4934
|
+
const root = await this.resolveQueryableTopicRoot(
|
|
4935
|
+
pubsubMessage.topic,
|
|
4936
|
+
() => this.peers.get(from.hashcode()) === stream,
|
|
4937
|
+
);
|
|
4240
4938
|
await this.sendDirectControlMessage(
|
|
4241
4939
|
stream,
|
|
4242
4940
|
new TopicRootQueryResponse({
|
|
@@ -4322,6 +5020,11 @@ export class TopicControlPlane
|
|
|
4322
5020
|
return;
|
|
4323
5021
|
}
|
|
4324
5022
|
|
|
5023
|
+
if (pubsubMessage instanceof Unsubscribe) {
|
|
5024
|
+
this.processUnsubscribeMessage(message, pubsubMessage, from);
|
|
5025
|
+
return;
|
|
5026
|
+
}
|
|
5027
|
+
|
|
4325
5028
|
if (pubsubMessage instanceof GetSubscribers) {
|
|
4326
5029
|
const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
|
|
4327
5030
|
if (overlap.length === 0) return;
|
|
@@ -4347,6 +5050,45 @@ export class TopicControlPlane
|
|
|
4347
5050
|
}
|
|
4348
5051
|
}
|
|
4349
5052
|
|
|
5053
|
+
private processUnsubscribeMessage(
|
|
5054
|
+
message: DataMessage,
|
|
5055
|
+
unsubscribe: Unsubscribe,
|
|
5056
|
+
from: PublicSignKey,
|
|
5057
|
+
) {
|
|
5058
|
+
const senderKey = from.hashcode();
|
|
5059
|
+
const relevantTopics = unsubscribe.topics.filter((topic) =>
|
|
5060
|
+
this.isTrackedTopic(topic),
|
|
5061
|
+
);
|
|
5062
|
+
if (
|
|
5063
|
+
!relevantTopics.length ||
|
|
5064
|
+
!this.subscriptionMessageIsLatest(message, unsubscribe, relevantTopics)
|
|
5065
|
+
)
|
|
5066
|
+
return;
|
|
5067
|
+
const changed: string[] = [];
|
|
5068
|
+
for (const topic of relevantTopics) {
|
|
5069
|
+
if (this.topics.get(topic)?.delete(senderKey)) {
|
|
5070
|
+
changed.push(topic);
|
|
5071
|
+
this.peerToTopic.get(senderKey)?.delete(topic);
|
|
5072
|
+
}
|
|
5073
|
+
}
|
|
5074
|
+
if (!this.peerToTopic.get(senderKey)?.size) {
|
|
5075
|
+
this.peerToTopic.delete(senderKey);
|
|
5076
|
+
}
|
|
5077
|
+
// Independent direct/shard paths may deliver an older Subscribe later.
|
|
5078
|
+
// Retain recent unsubscribe floors, bounded separately from live subscribers.
|
|
5079
|
+
const messages = this.lastSubscriptionMessages.get(senderKey)!;
|
|
5080
|
+
this.lastSubscriptionMessages.delete(senderKey);
|
|
5081
|
+
this.lastSubscriptionMessages.set(senderKey, messages);
|
|
5082
|
+
for (const topic of relevantTopics) this.pruneUnsubscribedWatermarks(topic);
|
|
5083
|
+
if (changed.length > 0) {
|
|
5084
|
+
this.dispatchEvent(
|
|
5085
|
+
new CustomEvent<UnsubcriptionEvent>("unsubscribe", {
|
|
5086
|
+
detail: new UnsubcriptionEvent(from, changed, "remote-unsubscribe"),
|
|
5087
|
+
}),
|
|
5088
|
+
);
|
|
5089
|
+
}
|
|
5090
|
+
}
|
|
5091
|
+
|
|
4350
5092
|
private async processShardPubSubMessage(input: {
|
|
4351
5093
|
pubsubMessage: PubSubMessage;
|
|
4352
5094
|
message: DataMessage;
|
|
@@ -4433,41 +5175,7 @@ export class TopicControlPlane
|
|
|
4433
5175
|
}
|
|
4434
5176
|
|
|
4435
5177
|
if (pubsubMessage instanceof Unsubscribe) {
|
|
4436
|
-
|
|
4437
|
-
const senderKey = sender.hashcode();
|
|
4438
|
-
const relevantTopics = pubsubMessage.topics.filter((t) =>
|
|
4439
|
-
this.isTrackedTopic(t),
|
|
4440
|
-
);
|
|
4441
|
-
|
|
4442
|
-
if (
|
|
4443
|
-
relevantTopics.length > 0 &&
|
|
4444
|
-
this.subscriptionMessageIsLatest(message, pubsubMessage, relevantTopics)
|
|
4445
|
-
) {
|
|
4446
|
-
const changed: string[] = [];
|
|
4447
|
-
for (const topic of relevantTopics) {
|
|
4448
|
-
const peers = this.topics.get(topic);
|
|
4449
|
-
if (!peers) continue;
|
|
4450
|
-
if (peers.delete(senderKey)) {
|
|
4451
|
-
changed.push(topic);
|
|
4452
|
-
this.peerToTopic.get(senderKey)?.delete(topic);
|
|
4453
|
-
}
|
|
4454
|
-
}
|
|
4455
|
-
if (!this.peerToTopic.get(senderKey)?.size) {
|
|
4456
|
-
this.peerToTopic.delete(senderKey);
|
|
4457
|
-
this.lastSubscriptionMessages.delete(senderKey);
|
|
4458
|
-
}
|
|
4459
|
-
if (changed.length > 0) {
|
|
4460
|
-
this.dispatchEvent(
|
|
4461
|
-
new CustomEvent<UnsubcriptionEvent>("unsubscribe", {
|
|
4462
|
-
detail: new UnsubcriptionEvent(
|
|
4463
|
-
sender,
|
|
4464
|
-
changed,
|
|
4465
|
-
"remote-unsubscribe",
|
|
4466
|
-
),
|
|
4467
|
-
}),
|
|
4468
|
-
);
|
|
4469
|
-
}
|
|
4470
|
-
}
|
|
5178
|
+
this.processUnsubscribeMessage(message, pubsubMessage, from);
|
|
4471
5179
|
return;
|
|
4472
5180
|
}
|
|
4473
5181
|
|
|
@@ -4586,6 +5294,7 @@ export class TopicControlPlane
|
|
|
4586
5294
|
!(pubsubMessage instanceof TopicRootQuery) &&
|
|
4587
5295
|
!(pubsubMessage instanceof GetSubscribers) &&
|
|
4588
5296
|
!(pubsubMessage instanceof Subscribe) &&
|
|
5297
|
+
!(pubsubMessage instanceof Unsubscribe) &&
|
|
4589
5298
|
!(pubsubMessage instanceof TopicRootQueryResponse)
|
|
4590
5299
|
) {
|
|
4591
5300
|
return true;
|