@peerbit/pubsub 5.2.11 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts +3 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js +90 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.js +712 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js +1591 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js +297 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts +73 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.js +183 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.js.map +1 -0
- package/dist/benchmark/fanout-tree-sim-lib.d.ts +101 -0
- package/dist/benchmark/fanout-tree-sim-lib.d.ts.map +1 -1
- package/dist/benchmark/fanout-tree-sim-lib.js +540 -38
- package/dist/benchmark/fanout-tree-sim-lib.js.map +1 -1
- package/dist/benchmark/fanout-tree-sim.d.ts +0 -9
- package/dist/benchmark/fanout-tree-sim.d.ts.map +1 -1
- package/dist/benchmark/fanout-tree-sim.js +369 -28
- package/dist/benchmark/fanout-tree-sim.js.map +1 -1
- package/dist/benchmark/index.js +20 -0
- package/dist/benchmark/index.js.map +1 -1
- package/dist/benchmark/pubsub-topic-sim-lib.d.ts +4 -0
- package/dist/benchmark/pubsub-topic-sim-lib.d.ts.map +1 -1
- package/dist/benchmark/pubsub-topic-sim-lib.js +58 -3
- package/dist/benchmark/pubsub-topic-sim-lib.js.map +1 -1
- package/dist/benchmark/pubsub-topic-sim.js +18 -0
- package/dist/benchmark/pubsub-topic-sim.js.map +1 -1
- package/dist/src/debounced-set.d.ts +1 -1
- package/dist/src/debounced-set.d.ts.map +1 -1
- package/dist/src/debounced-set.js +5 -1
- package/dist/src/debounced-set.js.map +1 -1
- package/dist/src/fanout-tree-codec.d.ts +330 -0
- package/dist/src/fanout-tree-codec.d.ts.map +1 -0
- package/dist/src/fanout-tree-codec.js +1209 -0
- package/dist/src/fanout-tree-codec.js.map +1 -0
- package/dist/src/fanout-tree-parent-upgrade.d.ts +117 -0
- package/dist/src/fanout-tree-parent-upgrade.d.ts.map +1 -0
- package/dist/src/fanout-tree-parent-upgrade.js +135 -0
- package/dist/src/fanout-tree-parent-upgrade.js.map +1 -0
- package/dist/src/fanout-tree.d.ts +241 -73
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +2224 -1050
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +57 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +293 -41
- package/dist/src/index.js.map +1 -1
- package/dist/src/topic-root-control-plane.d.ts +13 -0
- package/dist/src/topic-root-control-plane.d.ts.map +1 -1
- package/dist/src/topic-root-control-plane.js +45 -0
- package/dist/src/topic-root-control-plane.js.map +1 -1
- package/package.json +7 -7
- package/src/debounced-set.ts +5 -0
- package/src/fanout-tree-codec.ts +1580 -0
- package/src/fanout-tree-parent-upgrade.ts +353 -0
- package/src/fanout-tree.ts +4467 -2204
- package/src/index.ts +392 -56
- package/src/topic-root-control-plane.ts +48 -0
package/dist/src/fanout-tree.js
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
|
-
import { getPublicKeyFromPeerId, ready, sha256Sync, toBase64 } from "@peerbit/crypto";
|
|
2
1
|
import { multiaddr } from "@multiformats/multiaddr";
|
|
2
|
+
import { getPublicKeyFromPeerId, ready, sha256Sync, toBase64, } from "@peerbit/crypto";
|
|
3
3
|
import { DirectStream, dontThrowIfDeliveryError, } from "@peerbit/stream";
|
|
4
4
|
import { AnyWhere, DataMessage, } from "@peerbit/stream-interface";
|
|
5
5
|
import { AbortError, TimeoutError, delay } from "@peerbit/time";
|
|
6
6
|
import { anySignal } from "any-signal";
|
|
7
7
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
8
8
|
import { normalizeUint8Array, toUint8Array } from "./bytes.js";
|
|
9
|
+
import { JOIN_REJECT_LOW_BID, JOIN_REJECT_NO_CAPACITY, JOIN_REJECT_NOT_ATTACHED, JOIN_REJECT_REDIRECT_ADDR_MAX, JOIN_REJECT_REDIRECT_MAX, MSG_DATA, MSG_END, MSG_FETCH_REQ, MSG_IHAVE, MSG_JOIN_ACCEPT, MSG_JOIN_REJECT, MSG_JOIN_REQ, MSG_KICK, MSG_LEAVE, MSG_PARENT_PROBE_REPLY, MSG_PARENT_PROBE_REQ, MSG_PROVIDER_ANNOUNCE, MSG_PROVIDER_NOTIFY, MSG_PROVIDER_QUERY, MSG_PROVIDER_REPLY, MSG_PROVIDER_SUBSCRIBE, MSG_PROVIDER_UNSUBSCRIBE, MSG_PUBLISH_PROXY, MSG_REPAIR_REQ, MSG_ROUTE_QUERY, MSG_ROUTE_REPLY, MSG_TRACKER_ANNOUNCE, MSG_TRACKER_FEEDBACK, MSG_TRACKER_QUERY, MSG_TRACKER_REPLY, MSG_UNICAST, MSG_UNICAST_ACK, PARENT_PROBE_FLAG_ACCEPTING, PARENT_PROBE_FLAG_OVERLOADED, PARENT_PROBE_FLAG_REPAIRING, PARENT_PROBE_FLAG_ROOTED, TRACKER_FEEDBACK_DIAL_FAILED, TRACKER_FEEDBACK_JOINED, TRACKER_FEEDBACK_JOIN_REJECT, TRACKER_FEEDBACK_JOIN_TIMEOUT, UNICAST_ACK_DEFAULT_TIMEOUT_MS, clampU16, readU32BE, tsFanoutWireCodec, writeU32BE, } from "./fanout-tree-codec.js";
|
|
10
|
+
import { evaluateParentUpgradeGate, normalizeParentUpgradePolicy, recordParentUpgradeSkip, } from "./fanout-tree-parent-upgrade.js";
|
|
9
11
|
import { TopicRootControlPlane } from "./topic-root-control-plane.js";
|
|
10
12
|
const CONTROL_PRIORITY = 10;
|
|
11
13
|
const DATA_PRIORITY = 1;
|
|
12
14
|
const ROUTE_PROXY_TIMEOUT_MS = 10_000;
|
|
15
|
+
// Backstop bounds for the route-proxy machinery under query storms (#983):
|
|
16
|
+
// each pending entry retains a live timer closure and a Set of child hashes.
|
|
17
|
+
const ROUTE_PROXY_MAX_PENDING = 2_048;
|
|
18
|
+
const ROUTE_PROXY_MAX_WAITERS = 512;
|
|
13
19
|
const ROUTE_CACHE_MAX_ENTRIES_HARD_CAP = 100_000;
|
|
14
20
|
const PEER_HINT_MAX_ENTRIES_HARD_CAP = 100_000;
|
|
15
21
|
// Best-effort address cache for join candidates learned via trackers/redirects.
|
|
@@ -35,8 +41,10 @@ const OVERLOAD_KICK_MAX_PER_EVENT = 4;
|
|
|
35
41
|
const DATA_WRITE_FAIL_KICK_STREAK_THRESHOLD = 3;
|
|
36
42
|
const DATA_WRITE_FAIL_KICK_COOLDOWN_MS = 2_000;
|
|
37
43
|
const DATA_WRITE_FAIL_KICK_MAX_PER_EVENT = 4;
|
|
38
|
-
const
|
|
39
|
-
const
|
|
44
|
+
const PARENT_REPAIR_DEAD_STREAK_THRESHOLD = 16;
|
|
45
|
+
const PARENT_REPAIR_DEAD_MIN_LIVENESS_MS = 15_000;
|
|
46
|
+
const REPAIR_RETRY_MIN_MS = 1_000;
|
|
47
|
+
const REPAIR_RETRY_INTERVAL_FACTOR = 5;
|
|
40
48
|
const JOIN_REJECT_REDIRECT_QUEUE_MAX = 64;
|
|
41
49
|
// When a relay loses its parent, pause before trying to rejoin so its children can
|
|
42
50
|
// attach elsewhere (helps avoid disconnected components stabilizing).
|
|
@@ -45,622 +53,16 @@ const JOIN_REJECT_REDIRECT_QUEUE_MAX = 64;
|
|
|
45
53
|
// This gives kicked children a window to reattach elsewhere (helps avoid stable
|
|
46
54
|
// disconnected components). Leaf nodes (no children) rejoin immediately.
|
|
47
55
|
const RELAY_REJOIN_COOLDOWN_MS = 3_000;
|
|
48
|
-
const
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const MSG_UNICAST = 12;
|
|
55
|
-
const MSG_ROUTE_QUERY = 13;
|
|
56
|
-
const MSG_ROUTE_REPLY = 14;
|
|
57
|
-
const MSG_PUBLISH_PROXY = 15;
|
|
58
|
-
const MSG_LEAVE = 16;
|
|
59
|
-
const MSG_UNICAST_ACK = 17;
|
|
60
|
-
const MSG_REPAIR_REQ = 20;
|
|
61
|
-
const MSG_FETCH_REQ = 21;
|
|
62
|
-
const MSG_IHAVE = 22;
|
|
63
|
-
const MSG_TRACKER_ANNOUNCE = 30;
|
|
64
|
-
const MSG_TRACKER_QUERY = 31;
|
|
65
|
-
const MSG_TRACKER_REPLY = 32;
|
|
66
|
-
const MSG_TRACKER_FEEDBACK = 33;
|
|
67
|
-
const MSG_PROVIDER_ANNOUNCE = 34;
|
|
68
|
-
const MSG_PROVIDER_QUERY = 35;
|
|
69
|
-
const MSG_PROVIDER_REPLY = 36;
|
|
70
|
-
const MSG_PROVIDER_SUBSCRIBE = 37;
|
|
71
|
-
const MSG_PROVIDER_UNSUBSCRIBE = 38;
|
|
72
|
-
const MSG_PROVIDER_NOTIFY = 39;
|
|
73
|
-
const JOIN_REJECT_NOT_ATTACHED = 1;
|
|
74
|
-
const JOIN_REJECT_NO_CAPACITY = 2;
|
|
75
|
-
const JOIN_REJECT_LOW_BID = 3;
|
|
76
|
-
const TRACKER_FEEDBACK_JOINED = 1;
|
|
77
|
-
const TRACKER_FEEDBACK_DIAL_FAILED = 2;
|
|
78
|
-
const TRACKER_FEEDBACK_JOIN_TIMEOUT = 3;
|
|
79
|
-
const TRACKER_FEEDBACK_JOIN_REJECT = 4;
|
|
80
|
-
const textEncoder = new TextEncoder();
|
|
81
|
-
const textDecoder = new TextDecoder();
|
|
82
|
-
const writeU32BE = (buf, offset, value) => {
|
|
83
|
-
buf[offset + 0] = (value >>> 24) & 0xff;
|
|
84
|
-
buf[offset + 1] = (value >>> 16) & 0xff;
|
|
85
|
-
buf[offset + 2] = (value >>> 8) & 0xff;
|
|
86
|
-
buf[offset + 3] = value & 0xff;
|
|
87
|
-
};
|
|
88
|
-
const readU32BE = (buf, offset) => ((buf[offset + 0] << 24) |
|
|
89
|
-
(buf[offset + 1] << 16) |
|
|
90
|
-
(buf[offset + 2] << 8) |
|
|
91
|
-
buf[offset + 3]) >>> 0;
|
|
92
|
-
const writeU64BE = (buf, offset, value) => {
|
|
93
|
-
let v = value & 0xffffffffffffffffn;
|
|
94
|
-
for (let i = 7; i >= 0; i--) {
|
|
95
|
-
buf[offset + i] = Number(v & 0xffn);
|
|
96
|
-
v >>= 8n;
|
|
97
|
-
}
|
|
98
|
-
};
|
|
99
|
-
const readU64BE = (buf, offset) => {
|
|
100
|
-
let v = 0n;
|
|
101
|
-
for (let i = 0; i < 8; i++) {
|
|
102
|
-
v = (v << 8n) | BigInt(buf[offset + i]);
|
|
103
|
-
}
|
|
104
|
-
return v;
|
|
105
|
-
};
|
|
106
|
-
const writeU16BE = (buf, offset, value) => {
|
|
107
|
-
buf[offset + 0] = (value >>> 8) & 0xff;
|
|
108
|
-
buf[offset + 1] = value & 0xff;
|
|
109
|
-
};
|
|
110
|
-
const readU16BE = (buf, offset) => ((buf[offset + 0] << 8) | buf[offset + 1]) >>> 0;
|
|
111
|
-
const encodeJoinReq = (channelKey, reqId, bidPerByte) => {
|
|
112
|
-
const buf = new Uint8Array(1 + 32 + 4 + 4);
|
|
113
|
-
buf[0] = MSG_JOIN_REQ;
|
|
114
|
-
buf.set(channelKey, 1);
|
|
115
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
116
|
-
writeU32BE(buf, 37, bidPerByte >>> 0);
|
|
117
|
-
return buf;
|
|
118
|
-
};
|
|
119
|
-
const MAX_ROUTE_HOPS = 32;
|
|
120
|
-
const UNICAST_FLAG_ACK = 1;
|
|
121
|
-
const UNICAST_ACK_DEFAULT_TIMEOUT_MS = 30_000;
|
|
122
|
-
const encodeJoinAccept = (channelKey, reqId, level, parentRouteFromRoot) => {
|
|
123
|
-
const routeBytes = [];
|
|
124
|
-
let bytes = 1 + 32 + 4 + 2 + 1;
|
|
125
|
-
let count = 0;
|
|
126
|
-
for (const hop of parentRouteFromRoot ?? []) {
|
|
127
|
-
if (count >= MAX_ROUTE_HOPS)
|
|
128
|
-
break;
|
|
129
|
-
if (typeof hop !== "string")
|
|
130
|
-
continue;
|
|
131
|
-
const hb = textEncoder.encode(hop);
|
|
132
|
-
if (hb.length === 0 || hb.length > 255)
|
|
133
|
-
continue;
|
|
134
|
-
routeBytes.push(hb);
|
|
135
|
-
bytes += 1 + hb.length;
|
|
136
|
-
count += 1;
|
|
137
|
-
}
|
|
138
|
-
const buf = new Uint8Array(bytes);
|
|
139
|
-
buf[0] = MSG_JOIN_ACCEPT;
|
|
140
|
-
buf.set(channelKey, 1);
|
|
141
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
142
|
-
writeU16BE(buf, 37, level & 0xffff);
|
|
143
|
-
buf[39] = count & 0xff;
|
|
144
|
-
let offset = 40;
|
|
145
|
-
for (const hb of routeBytes) {
|
|
146
|
-
buf[offset++] = hb.length & 0xff;
|
|
147
|
-
buf.set(hb, offset);
|
|
148
|
-
offset += hb.length;
|
|
149
|
-
}
|
|
150
|
-
return buf;
|
|
151
|
-
};
|
|
152
|
-
const encodeJoinReject = (channelKey, reqId, reason, redirects) => {
|
|
153
|
-
const encoded = [];
|
|
154
|
-
for (const r of redirects ?? []) {
|
|
155
|
-
if (!r?.hash)
|
|
156
|
-
continue;
|
|
157
|
-
const hashBytes = textEncoder.encode(r.hash);
|
|
158
|
-
if (hashBytes.length === 0 || hashBytes.length > 255)
|
|
159
|
-
continue;
|
|
160
|
-
const addrs = (r.addrs ?? [])
|
|
161
|
-
.filter((a) => a instanceof Uint8Array)
|
|
162
|
-
.filter((a) => a.length > 0 && a.length <= 0xffff)
|
|
163
|
-
.slice(0, JOIN_REJECT_REDIRECT_ADDR_MAX);
|
|
164
|
-
if (addrs.length === 0)
|
|
165
|
-
continue;
|
|
166
|
-
encoded.push({ hashBytes, addrs });
|
|
167
|
-
if (encoded.length >= JOIN_REJECT_REDIRECT_MAX)
|
|
168
|
-
break;
|
|
169
|
-
}
|
|
170
|
-
if (encoded.length === 0) {
|
|
171
|
-
const buf = new Uint8Array(1 + 32 + 4 + 1);
|
|
172
|
-
buf[0] = MSG_JOIN_REJECT;
|
|
173
|
-
buf.set(channelKey, 1);
|
|
174
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
175
|
-
buf[37] = reason & 0xff;
|
|
176
|
-
return buf;
|
|
177
|
-
}
|
|
178
|
-
let bytes = 1 + 32 + 4 + 1 + 1;
|
|
179
|
-
for (const e of encoded) {
|
|
180
|
-
bytes += 1 + e.hashBytes.length + 1;
|
|
181
|
-
for (const a of e.addrs)
|
|
182
|
-
bytes += 2 + a.length;
|
|
183
|
-
}
|
|
184
|
-
const buf = new Uint8Array(bytes);
|
|
185
|
-
buf[0] = MSG_JOIN_REJECT;
|
|
186
|
-
buf.set(channelKey, 1);
|
|
187
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
188
|
-
buf[37] = reason & 0xff;
|
|
189
|
-
buf[38] = Math.max(0, Math.min(255, encoded.length)) & 0xff;
|
|
190
|
-
let offset = 39;
|
|
191
|
-
for (const e of encoded) {
|
|
192
|
-
buf[offset++] = e.hashBytes.length & 0xff;
|
|
193
|
-
buf.set(e.hashBytes, offset);
|
|
194
|
-
offset += e.hashBytes.length;
|
|
195
|
-
buf[offset++] = Math.max(0, Math.min(255, e.addrs.length)) & 0xff;
|
|
196
|
-
for (const a of e.addrs) {
|
|
197
|
-
writeU16BE(buf, offset, a.length);
|
|
198
|
-
offset += 2;
|
|
199
|
-
buf.set(a, offset);
|
|
200
|
-
offset += a.length;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return buf;
|
|
204
|
-
};
|
|
205
|
-
const encodeKick = (channelKey) => {
|
|
206
|
-
const buf = new Uint8Array(1 + 32);
|
|
207
|
-
buf[0] = MSG_KICK;
|
|
208
|
-
buf.set(channelKey, 1);
|
|
209
|
-
return buf;
|
|
210
|
-
};
|
|
211
|
-
const encodeEnd = (channelKey, lastSeqExclusive) => {
|
|
212
|
-
const buf = new Uint8Array(1 + 32 + 4);
|
|
213
|
-
buf[0] = MSG_END;
|
|
214
|
-
buf.set(channelKey, 1);
|
|
215
|
-
writeU32BE(buf, 33, lastSeqExclusive >>> 0);
|
|
216
|
-
return buf;
|
|
217
|
-
};
|
|
218
|
-
const encodeRepairReq = (channelKey, reqId, missingSeqs) => {
|
|
219
|
-
const count = Math.max(0, Math.min(255, missingSeqs.length));
|
|
220
|
-
const buf = new Uint8Array(1 + 32 + 4 + 1 + count * 4);
|
|
221
|
-
buf[0] = MSG_REPAIR_REQ;
|
|
222
|
-
buf.set(channelKey, 1);
|
|
223
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
224
|
-
buf[37] = count & 0xff;
|
|
225
|
-
for (let i = 0; i < count; i++) {
|
|
226
|
-
writeU32BE(buf, 38 + i * 4, missingSeqs[i] >>> 0);
|
|
227
|
-
}
|
|
228
|
-
return buf;
|
|
229
|
-
};
|
|
230
|
-
const encodeFetchReq = (channelKey, reqId, missingSeqs) => {
|
|
231
|
-
const count = Math.max(0, Math.min(255, missingSeqs.length));
|
|
232
|
-
const buf = new Uint8Array(1 + 32 + 4 + 1 + count * 4);
|
|
233
|
-
buf[0] = MSG_FETCH_REQ;
|
|
234
|
-
buf.set(channelKey, 1);
|
|
235
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
236
|
-
buf[37] = count & 0xff;
|
|
237
|
-
for (let i = 0; i < count; i++) {
|
|
238
|
-
writeU32BE(buf, 38 + i * 4, missingSeqs[i] >>> 0);
|
|
239
|
-
}
|
|
240
|
-
return buf;
|
|
241
|
-
};
|
|
242
|
-
const encodeIHave = (channelKey, haveFrom, haveToExclusive) => {
|
|
243
|
-
const buf = new Uint8Array(1 + 32 + 4 + 4);
|
|
244
|
-
buf[0] = MSG_IHAVE;
|
|
245
|
-
buf.set(channelKey, 1);
|
|
246
|
-
writeU32BE(buf, 33, haveFrom >>> 0);
|
|
247
|
-
writeU32BE(buf, 37, haveToExclusive >>> 0);
|
|
248
|
-
return buf;
|
|
249
|
-
};
|
|
250
|
-
const encodeData = (payload) => {
|
|
251
|
-
const buf = new Uint8Array(1 + payload.length);
|
|
252
|
-
buf[0] = MSG_DATA;
|
|
253
|
-
buf.set(payload, 1);
|
|
254
|
-
return buf;
|
|
255
|
-
};
|
|
256
|
-
const encodePublishProxy = (channelKey, payload) => {
|
|
257
|
-
const buf = new Uint8Array(1 + 32 + payload.length);
|
|
258
|
-
buf[0] = MSG_PUBLISH_PROXY;
|
|
259
|
-
buf.set(channelKey, 1);
|
|
260
|
-
buf.set(payload, 33);
|
|
261
|
-
return buf;
|
|
262
|
-
};
|
|
263
|
-
const encodeLeave = (channelKey) => {
|
|
264
|
-
const buf = new Uint8Array(1 + 32);
|
|
265
|
-
buf[0] = MSG_LEAVE;
|
|
266
|
-
buf.set(channelKey, 1);
|
|
267
|
-
return buf;
|
|
268
|
-
};
|
|
269
|
-
const encodeUnicast = (channelKey, route, payload, options) => {
|
|
270
|
-
const wantsAck = options?.ackToken != null;
|
|
271
|
-
const flags = wantsAck ? UNICAST_FLAG_ACK : 0;
|
|
272
|
-
const toRouteBytes = [];
|
|
273
|
-
let bytes = 1 + 32 + 1 + (wantsAck ? 8 : 0) + 1;
|
|
274
|
-
let toCount = 0;
|
|
275
|
-
for (const hop of route ?? []) {
|
|
276
|
-
if (toCount >= MAX_ROUTE_HOPS)
|
|
277
|
-
break;
|
|
278
|
-
if (typeof hop !== "string")
|
|
279
|
-
continue;
|
|
280
|
-
const hb = textEncoder.encode(hop);
|
|
281
|
-
if (hb.length === 0 || hb.length > 255)
|
|
282
|
-
continue;
|
|
283
|
-
toRouteBytes.push(hb);
|
|
284
|
-
bytes += 1 + hb.length;
|
|
285
|
-
toCount += 1;
|
|
286
|
-
}
|
|
287
|
-
const replyRouteBytes = [];
|
|
288
|
-
let replyCount = 0;
|
|
289
|
-
if (wantsAck) {
|
|
290
|
-
bytes += 1; // replyRouteCount
|
|
291
|
-
for (const hop of options?.replyRoute ?? []) {
|
|
292
|
-
if (replyCount >= MAX_ROUTE_HOPS)
|
|
293
|
-
break;
|
|
294
|
-
if (typeof hop !== "string")
|
|
295
|
-
continue;
|
|
296
|
-
const hb = textEncoder.encode(hop);
|
|
297
|
-
if (hb.length === 0 || hb.length > 255)
|
|
298
|
-
continue;
|
|
299
|
-
replyRouteBytes.push(hb);
|
|
300
|
-
bytes += 1 + hb.length;
|
|
301
|
-
replyCount += 1;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
bytes += payload.byteLength;
|
|
305
|
-
const buf = new Uint8Array(bytes);
|
|
306
|
-
buf[0] = MSG_UNICAST;
|
|
307
|
-
buf.set(channelKey, 1);
|
|
308
|
-
buf[33] = flags & 0xff;
|
|
309
|
-
let offset = 34;
|
|
310
|
-
if (wantsAck) {
|
|
311
|
-
writeU64BE(buf, offset, options.ackToken);
|
|
312
|
-
offset += 8;
|
|
313
|
-
}
|
|
314
|
-
buf[offset++] = toCount & 0xff;
|
|
315
|
-
for (const hb of toRouteBytes) {
|
|
316
|
-
buf[offset++] = hb.length & 0xff;
|
|
317
|
-
buf.set(hb, offset);
|
|
318
|
-
offset += hb.length;
|
|
319
|
-
}
|
|
320
|
-
if (wantsAck) {
|
|
321
|
-
buf[offset++] = replyCount & 0xff;
|
|
322
|
-
for (const hb of replyRouteBytes) {
|
|
323
|
-
buf[offset++] = hb.length & 0xff;
|
|
324
|
-
buf.set(hb, offset);
|
|
325
|
-
offset += hb.length;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
buf.set(payload, offset);
|
|
329
|
-
return buf;
|
|
330
|
-
};
|
|
331
|
-
const encodeUnicastAck = (channelKey, ackToken, route) => {
|
|
332
|
-
const routeBytes = [];
|
|
333
|
-
let bytes = 1 + 32 + 8 + 1;
|
|
334
|
-
let count = 0;
|
|
335
|
-
for (const hop of route ?? []) {
|
|
336
|
-
if (count >= MAX_ROUTE_HOPS)
|
|
337
|
-
break;
|
|
338
|
-
if (typeof hop !== "string")
|
|
339
|
-
continue;
|
|
340
|
-
const hb = textEncoder.encode(hop);
|
|
341
|
-
if (hb.length === 0 || hb.length > 255)
|
|
342
|
-
continue;
|
|
343
|
-
routeBytes.push(hb);
|
|
344
|
-
bytes += 1 + hb.length;
|
|
345
|
-
count += 1;
|
|
346
|
-
}
|
|
347
|
-
const buf = new Uint8Array(bytes);
|
|
348
|
-
buf[0] = MSG_UNICAST_ACK;
|
|
349
|
-
buf.set(channelKey, 1);
|
|
350
|
-
writeU64BE(buf, 33, ackToken);
|
|
351
|
-
buf[41] = count & 0xff;
|
|
352
|
-
let offset = 42;
|
|
353
|
-
for (const hb of routeBytes) {
|
|
354
|
-
buf[offset++] = hb.length & 0xff;
|
|
355
|
-
buf.set(hb, offset);
|
|
356
|
-
offset += hb.length;
|
|
357
|
-
}
|
|
358
|
-
return buf;
|
|
359
|
-
};
|
|
360
|
-
const encodeRouteQuery = (channelKey, reqId, targetHash) => {
|
|
361
|
-
const targetBytes = textEncoder.encode(targetHash);
|
|
362
|
-
const targetLen = Math.max(0, Math.min(255, targetBytes.length));
|
|
363
|
-
const buf = new Uint8Array(1 + 32 + 4 + 1 + targetLen);
|
|
364
|
-
buf[0] = MSG_ROUTE_QUERY;
|
|
365
|
-
buf.set(channelKey, 1);
|
|
366
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
367
|
-
buf[37] = targetLen & 0xff;
|
|
368
|
-
buf.set(targetBytes.subarray(0, targetLen), 38);
|
|
369
|
-
return buf;
|
|
370
|
-
};
|
|
371
|
-
const encodeRouteReply = (channelKey, reqId, route) => {
|
|
372
|
-
const routeBytes = [];
|
|
373
|
-
let bytes = 1 + 32 + 4 + 1;
|
|
374
|
-
let count = 0;
|
|
375
|
-
for (const hop of route ?? []) {
|
|
376
|
-
if (count >= MAX_ROUTE_HOPS)
|
|
377
|
-
break;
|
|
378
|
-
if (typeof hop !== "string")
|
|
379
|
-
continue;
|
|
380
|
-
const hb = textEncoder.encode(hop);
|
|
381
|
-
if (hb.length === 0 || hb.length > 255)
|
|
382
|
-
continue;
|
|
383
|
-
routeBytes.push(hb);
|
|
384
|
-
bytes += 1 + hb.length;
|
|
385
|
-
count += 1;
|
|
386
|
-
}
|
|
387
|
-
const buf = new Uint8Array(bytes);
|
|
388
|
-
buf[0] = MSG_ROUTE_REPLY;
|
|
389
|
-
buf.set(channelKey, 1);
|
|
390
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
391
|
-
buf[37] = count & 0xff;
|
|
392
|
-
let offset = 38;
|
|
393
|
-
for (const hb of routeBytes) {
|
|
394
|
-
buf[offset++] = hb.length & 0xff;
|
|
395
|
-
buf.set(hb, offset);
|
|
396
|
-
offset += hb.length;
|
|
397
|
-
}
|
|
398
|
-
return buf;
|
|
399
|
-
};
|
|
400
|
-
const decodeRoute = (data, offsetStart, routeCount) => {
|
|
401
|
-
let offset = offsetStart;
|
|
402
|
-
const route = [];
|
|
403
|
-
for (let i = 0; i < routeCount; i++) {
|
|
404
|
-
if (offset + 1 > data.length)
|
|
405
|
-
break;
|
|
406
|
-
const len = data[offset++];
|
|
407
|
-
if (len === 0)
|
|
408
|
-
break;
|
|
409
|
-
if (offset + len > data.length)
|
|
410
|
-
break;
|
|
411
|
-
if (route.length < MAX_ROUTE_HOPS) {
|
|
412
|
-
route.push(textDecoder.decode(data.subarray(offset, offset + len)));
|
|
413
|
-
}
|
|
414
|
-
offset += len;
|
|
415
|
-
}
|
|
416
|
-
return { route, offset };
|
|
417
|
-
};
|
|
418
|
-
const clampU16 = (v) => Math.max(0, Math.min(0xffff, v | 0));
|
|
419
|
-
const encodeTrackerAnnounce = (channelKey, ttlMs, level, maxChildren, freeSlots, bidPerByte, addrs) => {
|
|
420
|
-
const addrCount = Math.max(0, Math.min(255, addrs.length));
|
|
421
|
-
let bytes = 1 + 32 + 4 + 2 + 2 + 2 + 4 + 1;
|
|
422
|
-
const addrBytes = new Array(addrCount);
|
|
423
|
-
for (let i = 0; i < addrCount; i++) {
|
|
424
|
-
const b = addrs[i].bytes;
|
|
425
|
-
addrBytes[i] = b;
|
|
426
|
-
bytes += 2 + b.length;
|
|
427
|
-
}
|
|
428
|
-
const buf = new Uint8Array(bytes);
|
|
429
|
-
buf[0] = MSG_TRACKER_ANNOUNCE;
|
|
430
|
-
buf.set(channelKey, 1);
|
|
431
|
-
writeU32BE(buf, 33, Math.max(0, Math.floor(ttlMs)) >>> 0);
|
|
432
|
-
writeU16BE(buf, 37, clampU16(level));
|
|
433
|
-
writeU16BE(buf, 39, clampU16(maxChildren));
|
|
434
|
-
writeU16BE(buf, 41, clampU16(freeSlots));
|
|
435
|
-
writeU32BE(buf, 43, Math.max(0, Math.floor(bidPerByte)) >>> 0);
|
|
436
|
-
buf[47] = addrCount & 0xff;
|
|
437
|
-
let offset = 48;
|
|
438
|
-
for (let i = 0; i < addrCount; i++) {
|
|
439
|
-
const b = addrBytes[i];
|
|
440
|
-
writeU16BE(buf, offset, b.length);
|
|
441
|
-
offset += 2;
|
|
442
|
-
buf.set(b, offset);
|
|
443
|
-
offset += b.length;
|
|
444
|
-
}
|
|
445
|
-
return buf;
|
|
446
|
-
};
|
|
447
|
-
const encodeTrackerQuery = (channelKey, reqId, want) => {
|
|
448
|
-
const buf = new Uint8Array(1 + 32 + 4 + 2);
|
|
449
|
-
buf[0] = MSG_TRACKER_QUERY;
|
|
450
|
-
buf.set(channelKey, 1);
|
|
451
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
452
|
-
writeU16BE(buf, 37, clampU16(want));
|
|
453
|
-
return buf;
|
|
454
|
-
};
|
|
455
|
-
const encodeTrackerReply = (channelKey, reqId, entries) => {
|
|
456
|
-
const count = Math.max(0, Math.min(255, entries.length));
|
|
457
|
-
let bytes = 1 + 32 + 4 + 1;
|
|
458
|
-
const encoded = [];
|
|
459
|
-
for (let i = 0; i < count; i++) {
|
|
460
|
-
const e = entries[i];
|
|
461
|
-
const hashBytes = textEncoder.encode(e.hash);
|
|
462
|
-
if (hashBytes.length > 255)
|
|
463
|
-
continue;
|
|
464
|
-
const addrCount = Math.max(0, Math.min(255, e.addrs.length));
|
|
465
|
-
const addrs = e.addrs.slice(0, addrCount);
|
|
466
|
-
bytes += 1 + hashBytes.length + 2 + 2 + 4 + 1;
|
|
467
|
-
for (const a of addrs)
|
|
468
|
-
bytes += 2 + a.length;
|
|
469
|
-
encoded.push({
|
|
470
|
-
hashBytes,
|
|
471
|
-
level: e.level,
|
|
472
|
-
freeSlots: e.freeSlots,
|
|
473
|
-
bidPerByte: e.bidPerByte,
|
|
474
|
-
addrs,
|
|
475
|
-
});
|
|
476
|
-
}
|
|
477
|
-
const buf = new Uint8Array(bytes);
|
|
478
|
-
buf[0] = MSG_TRACKER_REPLY;
|
|
479
|
-
buf.set(channelKey, 1);
|
|
480
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
481
|
-
buf[37] = Math.max(0, Math.min(255, encoded.length)) & 0xff;
|
|
482
|
-
let offset = 38;
|
|
483
|
-
for (const e of encoded) {
|
|
484
|
-
buf[offset++] = e.hashBytes.length & 0xff;
|
|
485
|
-
buf.set(e.hashBytes, offset);
|
|
486
|
-
offset += e.hashBytes.length;
|
|
487
|
-
writeU16BE(buf, offset, clampU16(e.level));
|
|
488
|
-
offset += 2;
|
|
489
|
-
writeU16BE(buf, offset, clampU16(e.freeSlots));
|
|
490
|
-
offset += 2;
|
|
491
|
-
writeU32BE(buf, offset, Math.max(0, Math.floor(e.bidPerByte)) >>> 0);
|
|
492
|
-
offset += 4;
|
|
493
|
-
buf[offset++] = Math.max(0, Math.min(255, e.addrs.length)) & 0xff;
|
|
494
|
-
for (const a of e.addrs) {
|
|
495
|
-
writeU16BE(buf, offset, a.length);
|
|
496
|
-
offset += 2;
|
|
497
|
-
buf.set(a, offset);
|
|
498
|
-
offset += a.length;
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
return buf;
|
|
502
|
-
};
|
|
503
|
-
const encodeTrackerFeedback = (channelKey, candidateHash, event, reason) => {
|
|
504
|
-
const hashBytes = textEncoder.encode(candidateHash);
|
|
505
|
-
const hashLen = Math.max(0, Math.min(255, hashBytes.length));
|
|
506
|
-
const buf = new Uint8Array(1 + 32 + 1 + hashLen + 1 + 1);
|
|
507
|
-
buf[0] = MSG_TRACKER_FEEDBACK;
|
|
508
|
-
buf.set(channelKey, 1);
|
|
509
|
-
buf[33] = hashLen & 0xff;
|
|
510
|
-
buf.set(hashBytes.subarray(0, hashLen), 34);
|
|
511
|
-
buf[34 + hashLen] = event & 0xff;
|
|
512
|
-
buf[34 + hashLen + 1] = reason & 0xff;
|
|
513
|
-
return buf;
|
|
514
|
-
};
|
|
515
|
-
const encodeProviderAnnounce = (namespaceKey, ttlMs, addrs) => {
|
|
516
|
-
const addrCount = Math.max(0, Math.min(255, addrs.length));
|
|
517
|
-
let bytes = 1 + 32 + 4 + 1;
|
|
518
|
-
const addrBytes = new Array(addrCount);
|
|
519
|
-
for (let i = 0; i < addrCount; i++) {
|
|
520
|
-
const b = addrs[i].bytes;
|
|
521
|
-
addrBytes[i] = b;
|
|
522
|
-
bytes += 2 + b.length;
|
|
523
|
-
}
|
|
524
|
-
const buf = new Uint8Array(bytes);
|
|
525
|
-
buf[0] = MSG_PROVIDER_ANNOUNCE;
|
|
526
|
-
buf.set(namespaceKey, 1);
|
|
527
|
-
writeU32BE(buf, 33, Math.max(0, Math.floor(ttlMs)) >>> 0);
|
|
528
|
-
buf[37] = addrCount & 0xff;
|
|
529
|
-
let offset = 38;
|
|
530
|
-
for (let i = 0; i < addrCount; i++) {
|
|
531
|
-
const b = addrBytes[i];
|
|
532
|
-
writeU16BE(buf, offset, b.length);
|
|
533
|
-
offset += 2;
|
|
534
|
-
buf.set(b, offset);
|
|
535
|
-
offset += b.length;
|
|
536
|
-
}
|
|
537
|
-
return buf;
|
|
538
|
-
};
|
|
539
|
-
const encodeProviderQuery = (namespaceKey, reqId, want, seed) => {
|
|
540
|
-
const buf = new Uint8Array(1 + 32 + 4 + 2 + 4);
|
|
541
|
-
buf[0] = MSG_PROVIDER_QUERY;
|
|
542
|
-
buf.set(namespaceKey, 1);
|
|
543
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
544
|
-
writeU16BE(buf, 37, clampU16(want));
|
|
545
|
-
writeU32BE(buf, 39, seed >>> 0);
|
|
546
|
-
return buf;
|
|
547
|
-
};
|
|
548
|
-
const encodeProviderEntries = (entries) => {
|
|
549
|
-
const count = Math.max(0, Math.min(255, entries.length));
|
|
550
|
-
let bytes = 1;
|
|
551
|
-
const encoded = [];
|
|
552
|
-
for (let i = 0; i < count; i++) {
|
|
553
|
-
const e = entries[i];
|
|
554
|
-
const hashBytes = textEncoder.encode(e.hash);
|
|
555
|
-
if (hashBytes.length > 255)
|
|
556
|
-
continue;
|
|
557
|
-
const addrCount = Math.max(0, Math.min(255, e.addrs.length));
|
|
558
|
-
const addrs = e.addrs.slice(0, addrCount);
|
|
559
|
-
bytes += 1 + hashBytes.length + 1;
|
|
560
|
-
for (const a of addrs)
|
|
561
|
-
bytes += 2 + a.length;
|
|
562
|
-
encoded.push({ hashBytes, addrs });
|
|
563
|
-
}
|
|
564
|
-
return { bytes, encoded };
|
|
565
|
-
};
|
|
566
|
-
const encodeProviderReply = (namespaceKey, reqId, entries) => {
|
|
567
|
-
const { bytes: entryBytes, encoded } = encodeProviderEntries(entries);
|
|
568
|
-
let bytes = 1 + 32 + 4 + entryBytes;
|
|
569
|
-
const buf = new Uint8Array(bytes);
|
|
570
|
-
buf[0] = MSG_PROVIDER_REPLY;
|
|
571
|
-
buf.set(namespaceKey, 1);
|
|
572
|
-
writeU32BE(buf, 33, reqId >>> 0);
|
|
573
|
-
buf[37] = Math.max(0, Math.min(255, encoded.length)) & 0xff;
|
|
574
|
-
let offset = 38;
|
|
575
|
-
for (const e of encoded) {
|
|
576
|
-
buf[offset++] = e.hashBytes.length & 0xff;
|
|
577
|
-
buf.set(e.hashBytes, offset);
|
|
578
|
-
offset += e.hashBytes.length;
|
|
579
|
-
buf[offset++] = Math.max(0, Math.min(255, e.addrs.length)) & 0xff;
|
|
580
|
-
for (const a of e.addrs) {
|
|
581
|
-
writeU16BE(buf, offset, a.length);
|
|
582
|
-
offset += 2;
|
|
583
|
-
buf.set(a, offset);
|
|
584
|
-
offset += a.length;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
return buf;
|
|
588
|
-
};
|
|
589
|
-
const encodeProviderSubscribe = (namespaceKey, want, ttlMs) => {
|
|
590
|
-
const buf = new Uint8Array(1 + 32 + 2 + 4);
|
|
591
|
-
buf[0] = MSG_PROVIDER_SUBSCRIBE;
|
|
592
|
-
buf.set(namespaceKey, 1);
|
|
593
|
-
writeU16BE(buf, 33, clampU16(want));
|
|
594
|
-
writeU32BE(buf, 35, Math.max(0, Math.floor(ttlMs)) >>> 0);
|
|
595
|
-
return buf;
|
|
596
|
-
};
|
|
597
|
-
const encodeProviderUnsubscribe = (namespaceKey) => {
|
|
598
|
-
const buf = new Uint8Array(1 + 32);
|
|
599
|
-
buf[0] = MSG_PROVIDER_UNSUBSCRIBE;
|
|
600
|
-
buf.set(namespaceKey, 1);
|
|
601
|
-
return buf;
|
|
602
|
-
};
|
|
603
|
-
const encodeProviderNotify = (namespaceKey, entries) => {
|
|
604
|
-
const { bytes: entryBytes, encoded } = encodeProviderEntries(entries);
|
|
605
|
-
let bytes = 1 + 32 + entryBytes;
|
|
606
|
-
const buf = new Uint8Array(bytes);
|
|
607
|
-
buf[0] = MSG_PROVIDER_NOTIFY;
|
|
608
|
-
buf.set(namespaceKey, 1);
|
|
609
|
-
buf[33] = Math.max(0, Math.min(255, encoded.length)) & 0xff;
|
|
610
|
-
let offset = 34;
|
|
611
|
-
for (const e of encoded) {
|
|
612
|
-
buf[offset++] = e.hashBytes.length & 0xff;
|
|
613
|
-
buf.set(e.hashBytes, offset);
|
|
614
|
-
offset += e.hashBytes.length;
|
|
615
|
-
buf[offset++] = Math.max(0, Math.min(255, e.addrs.length)) & 0xff;
|
|
616
|
-
for (const a of e.addrs) {
|
|
617
|
-
writeU16BE(buf, offset, a.length);
|
|
618
|
-
offset += 2;
|
|
619
|
-
buf.set(a, offset);
|
|
620
|
-
offset += a.length;
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
return buf;
|
|
624
|
-
};
|
|
625
|
-
const decodeProviderEntries = (data, offsetStart, maxCount) => {
|
|
626
|
-
let offset = offsetStart;
|
|
627
|
-
const providers = [];
|
|
628
|
-
const limit = Math.min(maxCount, 255);
|
|
629
|
-
for (let i = 0; i < limit; i++) {
|
|
630
|
-
if (offset + 1 > data.length)
|
|
631
|
-
break;
|
|
632
|
-
const hashLen = data[offset++];
|
|
633
|
-
if (offset + hashLen > data.length)
|
|
634
|
-
break;
|
|
635
|
-
const hash = textDecoder.decode(data.subarray(offset, offset + hashLen));
|
|
636
|
-
offset += hashLen;
|
|
637
|
-
if (offset + 1 > data.length)
|
|
638
|
-
break;
|
|
639
|
-
const addrCount = data[offset++];
|
|
640
|
-
const addrs = [];
|
|
641
|
-
const addrMax = Math.min(addrCount, 16);
|
|
642
|
-
for (let j = 0; j < addrMax; j++) {
|
|
643
|
-
if (offset + 2 > data.length)
|
|
644
|
-
break;
|
|
645
|
-
const len = readU16BE(data, offset);
|
|
646
|
-
offset += 2;
|
|
647
|
-
if (offset + len > data.length)
|
|
648
|
-
break;
|
|
649
|
-
const bytes = data.subarray(offset, offset + len);
|
|
650
|
-
offset += len;
|
|
651
|
-
try {
|
|
652
|
-
addrs.push(multiaddr(bytes));
|
|
653
|
-
}
|
|
654
|
-
catch {
|
|
655
|
-
// ignore invalid
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
if (!hash)
|
|
659
|
-
continue;
|
|
660
|
-
providers.push({ hash, addrs });
|
|
56
|
+
const PARENT_UPGRADE_ROOT_RESERVATION_TTL_MS = 5_000;
|
|
57
|
+
const stableUnitInterval = (input) => {
|
|
58
|
+
let hash = 0x811c9dc5;
|
|
59
|
+
for (let i = 0; i < input.length; i++) {
|
|
60
|
+
hash ^= input.charCodeAt(i);
|
|
61
|
+
hash = Math.imul(hash, 0x01000193);
|
|
661
62
|
}
|
|
662
|
-
return
|
|
63
|
+
return (hash >>> 0) / 0x100000000;
|
|
663
64
|
};
|
|
65
|
+
const textEncoder = new TextEncoder();
|
|
664
66
|
const createDeferred = () => {
|
|
665
67
|
let resolve;
|
|
666
68
|
let reject;
|
|
@@ -700,6 +102,44 @@ const createEmptyMetrics = () => ({
|
|
|
700
102
|
reparentDisconnect: 0,
|
|
701
103
|
reparentStale: 0,
|
|
702
104
|
reparentKicked: 0,
|
|
105
|
+
reparentUpgrade: 0,
|
|
106
|
+
reparentUpgradeSkipLeaf: 0,
|
|
107
|
+
reparentUpgradeSkipRepair: 0,
|
|
108
|
+
reparentUpgradeSkipData: 0,
|
|
109
|
+
reparentUpgradeSkipCooldown: 0,
|
|
110
|
+
reparentUpgradeSkipQuiet: 0,
|
|
111
|
+
reparentUpgradeSkipBudget: 0,
|
|
112
|
+
reparentUpgradeSkipCandidateLevel: 0,
|
|
113
|
+
reparentUpgradeSkipCandidateSlots: 0,
|
|
114
|
+
reparentUpgradeSkipCandidatePressure: 0,
|
|
115
|
+
reparentUpgradeSkipRootPressure: 0,
|
|
116
|
+
reparentUpgradeSkipProbeNoReply: 0,
|
|
117
|
+
reparentUpgradeSkipProbeNotRooted: 0,
|
|
118
|
+
reparentUpgradeSkipProbeRepair: 0,
|
|
119
|
+
reparentUpgradeSkipProbeLag: 0,
|
|
120
|
+
reparentUpgradeSkipProbeOverloaded: 0,
|
|
121
|
+
reparentUpgradeSkipProbeCooldown: 0,
|
|
122
|
+
parentProbeReqSent: 0,
|
|
123
|
+
parentProbeReqReceived: 0,
|
|
124
|
+
parentProbeReplySent: 0,
|
|
125
|
+
parentProbeReplyReceived: 0,
|
|
126
|
+
parentUpgradeRootReservationCreated: 0,
|
|
127
|
+
parentUpgradeRootReservationConsumed: 0,
|
|
128
|
+
parentUpgradeRootReservationRejected: 0,
|
|
129
|
+
parentUpgradeRootReservationMarginRejected: 0,
|
|
130
|
+
parentUpgradeRootReservationBlocked: 0,
|
|
131
|
+
parentUpgradeRootReservationExpired: 0,
|
|
132
|
+
parentShadowStart: 0,
|
|
133
|
+
parentShadowObserve: 0,
|
|
134
|
+
parentShadowPromote: 0,
|
|
135
|
+
parentShadowReset: 0,
|
|
136
|
+
parentShadowRejectNoReply: 0,
|
|
137
|
+
parentShadowRejectNotRooted: 0,
|
|
138
|
+
parentShadowRejectCapacity: 0,
|
|
139
|
+
parentShadowRejectRepair: 0,
|
|
140
|
+
parentShadowRejectLag: 0,
|
|
141
|
+
parentShadowRejectOverloaded: 0,
|
|
142
|
+
parentShadowRejectLevel: 0,
|
|
703
143
|
endSent: 0,
|
|
704
144
|
endReceived: 0,
|
|
705
145
|
repairReqSent: 0,
|
|
@@ -727,6 +167,8 @@ const createEmptyMetrics = () => ({
|
|
|
727
167
|
routeProxyQueries: 0,
|
|
728
168
|
routeProxyTimeouts: 0,
|
|
729
169
|
routeProxyFanout: 0,
|
|
170
|
+
routeProxyCoalesced: 0,
|
|
171
|
+
routeProxyRejected: 0,
|
|
730
172
|
});
|
|
731
173
|
const isDataId = (id) => id.length === 32 &&
|
|
732
174
|
id[0] === ID_PREFIX[0] &&
|
|
@@ -748,20 +190,33 @@ export class FanoutTree extends DirectStream {
|
|
|
748
190
|
underlayPeerDisconnectHandler;
|
|
749
191
|
pendingProviderQueryBySuffixKey = new Map();
|
|
750
192
|
providerAnnounceBySuffixKey = new Map();
|
|
193
|
+
parentUpgradeShadowInFlightSuffixKey;
|
|
751
194
|
defaultUploadOverheadBytes = 128;
|
|
752
195
|
random;
|
|
753
196
|
unicastAckNodeTag32;
|
|
754
197
|
unicastAckSeq = 0;
|
|
755
198
|
topicRootControlPlane;
|
|
199
|
+
// Native fanout components of the rust-core DirectStream engine
|
|
200
|
+
// (`@peerbit/network-rust`). When set, every frame encode/decode of the
|
|
201
|
+
// `/peerbit/fanout-tree/0.5.0` codec plus the parent-upgrade policy/gate
|
|
202
|
+
// decisions run natively (byte- and decision-identical, covered by the
|
|
203
|
+
// golden parity suite); the channel state machine, timers and events
|
|
204
|
+
// stay host-side. Unset (the default) leaves every code path as-is.
|
|
205
|
+
nativeFanout;
|
|
206
|
+
codec;
|
|
756
207
|
constructor(components, opts) {
|
|
757
|
-
const { random, topicRootControlPlane, ...rest } = (opts ??
|
|
208
|
+
const { random, topicRootControlPlane, ...rest } = (opts ??
|
|
209
|
+
{});
|
|
758
210
|
super(components, FANOUT_PROTOCOLS, {
|
|
759
211
|
canRelayMessage: false,
|
|
760
212
|
...rest,
|
|
761
213
|
});
|
|
762
214
|
this.random = typeof random === "function" ? random : Math.random;
|
|
215
|
+
this.nativeFanout = this.rustCore?.fanout;
|
|
216
|
+
this.codec = this.nativeFanout ?? tsFanoutWireCodec;
|
|
763
217
|
this.unicastAckNodeTag32 = readU32BE(sha256Sync(textEncoder.encode(this.publicKeyHash)), 0);
|
|
764
|
-
this.topicRootControlPlane =
|
|
218
|
+
this.topicRootControlPlane =
|
|
219
|
+
topicRootControlPlane ?? new TopicRootControlPlane();
|
|
765
220
|
const onPeerDisconnect = (ev) => {
|
|
766
221
|
const peerId = ev?.detail;
|
|
767
222
|
if (!peerId)
|
|
@@ -878,6 +333,24 @@ export class FanoutTree extends DirectStream {
|
|
|
878
333
|
}
|
|
879
334
|
return cached;
|
|
880
335
|
}
|
|
336
|
+
/** Decoded provider entries (raw multiaddr bytes) -> candidates. Invalid
|
|
337
|
+
* multiaddrs are dropped exactly like the historical inline parsing. */
|
|
338
|
+
toProviderCandidates(entries) {
|
|
339
|
+
const candidates = [];
|
|
340
|
+
for (const entry of entries) {
|
|
341
|
+
const addrs = [];
|
|
342
|
+
for (const bytes of entry.addrs) {
|
|
343
|
+
try {
|
|
344
|
+
addrs.push(multiaddr(bytes));
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
// ignore invalid multiaddrs
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
candidates.push({ hash: entry.hash, addrs });
|
|
351
|
+
}
|
|
352
|
+
return candidates;
|
|
353
|
+
}
|
|
881
354
|
rememberProviderCandidates(id, candidates, expiresAt) {
|
|
882
355
|
if (candidates.length === 0)
|
|
883
356
|
return;
|
|
@@ -942,7 +415,7 @@ export class FanoutTree extends DirectStream {
|
|
|
942
415
|
addrs: candidate.addrs.map((a) => a.bytes),
|
|
943
416
|
expiresAt: now + 60_000,
|
|
944
417
|
}));
|
|
945
|
-
void this._sendControl(hash, encodeProviderNotify(id.key, entries)).catch(dontThrowIfDeliveryError);
|
|
418
|
+
void this._sendControl(hash, this.codec.encodeProviderNotify(id.key, entries)).catch(dontThrowIfDeliveryError);
|
|
946
419
|
}
|
|
947
420
|
this.pruneProviderWatchersIfEmpty(id.suffixKey);
|
|
948
421
|
}
|
|
@@ -1064,7 +537,7 @@ export class FanoutTree extends DirectStream {
|
|
|
1064
537
|
return;
|
|
1065
538
|
const ttlMs = ttlOverrideMs != null ? ttlOverrideMs : state.ttlMs;
|
|
1066
539
|
const addrs = this.getSelfAnnounceAddrs();
|
|
1067
|
-
const bytes = encodeProviderAnnounce(state.id.key, ttlMs, addrs);
|
|
540
|
+
const bytes = this.codec.encodeProviderAnnounce(state.id.key, ttlMs, addrs);
|
|
1068
541
|
await this._sendControlMany(peers, bytes);
|
|
1069
542
|
}
|
|
1070
543
|
nextProviderReqId(suffixKey) {
|
|
@@ -1094,7 +567,7 @@ export class FanoutTree extends DirectStream {
|
|
|
1094
567
|
const shuffleInPlace = (arr, seed32) => {
|
|
1095
568
|
if (arr.length <= 1)
|
|
1096
569
|
return;
|
|
1097
|
-
if (
|
|
570
|
+
if (seed32 >>> 0 === 0) {
|
|
1098
571
|
for (let i = arr.length - 1; i > 0; i--) {
|
|
1099
572
|
const j = Math.floor(this.random() * (i + 1));
|
|
1100
573
|
const tmp = arr[i];
|
|
@@ -1143,7 +616,7 @@ export class FanoutTree extends DirectStream {
|
|
|
1143
616
|
? await this.ensureBootstrapPeers(bootstraps, dialTimeoutMs, signal, bootstrapMaxPeers)
|
|
1144
617
|
: [];
|
|
1145
618
|
const perTrackerTimeout = Math.max(0, Math.floor(options.queryTimeoutMs ?? 1_000));
|
|
1146
|
-
const overallTimeout = Math.max(0, Math.floor(options.timeoutMs ??
|
|
619
|
+
const overallTimeout = Math.max(0, Math.floor(options.timeoutMs ?? 8_000));
|
|
1147
620
|
const deadlineAt = overallTimeout > 0 ? Date.now() + overallTimeout : 0;
|
|
1148
621
|
const byReq = this.pendingProviderQueryBySuffixKey.get(id.suffixKey) ||
|
|
1149
622
|
new Map();
|
|
@@ -1153,9 +626,13 @@ export class FanoutTree extends DirectStream {
|
|
|
1153
626
|
const p = new Promise((resolve) => {
|
|
1154
627
|
byReq.set(reqId, { resolve });
|
|
1155
628
|
});
|
|
1156
|
-
void this._sendControl(trackerHash, encodeProviderQuery(id.key, reqId, want, seed));
|
|
1157
|
-
const remainingMs = deadlineAt > 0
|
|
1158
|
-
|
|
629
|
+
void this._sendControl(trackerHash, this.codec.encodeProviderQuery(id.key, reqId, want, seed));
|
|
630
|
+
const remainingMs = deadlineAt > 0
|
|
631
|
+
? Math.max(0, deadlineAt - Date.now())
|
|
632
|
+
: perTrackerTimeout;
|
|
633
|
+
const timeoutMs = deadlineAt > 0
|
|
634
|
+
? Math.min(perTrackerTimeout, remainingMs)
|
|
635
|
+
: perTrackerTimeout;
|
|
1159
636
|
const res = await Promise.race([
|
|
1160
637
|
p,
|
|
1161
638
|
delay(timeoutMs, { signal }).then(() => null),
|
|
@@ -1252,7 +729,7 @@ export class FanoutTree extends DirectStream {
|
|
|
1252
729
|
this.providerWatchesBySuffixKey.delete(id.suffixKey);
|
|
1253
730
|
}
|
|
1254
731
|
for (const trackerHash of watch.trackerPeers) {
|
|
1255
|
-
void this._sendControl(trackerHash, encodeProviderUnsubscribe(id.key)).catch(dontThrowIfDeliveryError);
|
|
732
|
+
void this._sendControl(trackerHash, this.codec.encodeProviderUnsubscribe(id.key)).catch(dontThrowIfDeliveryError);
|
|
1256
733
|
}
|
|
1257
734
|
watch.trackerPeers = [];
|
|
1258
735
|
};
|
|
@@ -1272,12 +749,15 @@ export class FanoutTree extends DirectStream {
|
|
|
1272
749
|
: [];
|
|
1273
750
|
watch.trackerPeers = trackerPeers;
|
|
1274
751
|
for (const trackerHash of trackerPeers) {
|
|
1275
|
-
void this._sendControl(trackerHash, encodeProviderSubscribe(id.key, watch.want, watch.ttlMs)).catch(dontThrowIfDeliveryError);
|
|
752
|
+
void this._sendControl(trackerHash, this.codec.encodeProviderSubscribe(id.key, watch.want, watch.ttlMs)).catch(dontThrowIfDeliveryError);
|
|
1276
753
|
}
|
|
1277
754
|
}
|
|
1278
755
|
catch (error) {
|
|
1279
756
|
if (error instanceof AbortError ||
|
|
1280
|
-
(error &&
|
|
757
|
+
(error &&
|
|
758
|
+
typeof error === "object" &&
|
|
759
|
+
"name" in error &&
|
|
760
|
+
error.name === "AbortError")) {
|
|
1281
761
|
return;
|
|
1282
762
|
}
|
|
1283
763
|
}
|
|
@@ -1337,9 +817,9 @@ export class FanoutTree extends DirectStream {
|
|
|
1337
817
|
const neighborRepair = opts.neighborRepair === true;
|
|
1338
818
|
const neighborRepairPeers = Math.max(0, Math.floor(opts.neighborRepairPeers ?? 2));
|
|
1339
819
|
const neighborMeshPeers = Math.max(0, Math.floor(opts.neighborMeshPeers ?? Math.max(0, neighborRepairPeers * 2)));
|
|
1340
|
-
const neighborAnnounceIntervalMs = Math.max(0, Math.floor(opts.neighborAnnounceIntervalMs ??
|
|
1341
|
-
const neighborMeshRefreshIntervalMs = Math.max(0, Math.floor(opts.neighborMeshRefreshIntervalMs ??
|
|
1342
|
-
const neighborHaveTtlMs = Math.max(0, Math.floor(opts.neighborHaveTtlMs ??
|
|
820
|
+
const neighborAnnounceIntervalMs = Math.max(0, Math.floor(opts.neighborAnnounceIntervalMs ?? 800));
|
|
821
|
+
const neighborMeshRefreshIntervalMs = Math.max(0, Math.floor(opts.neighborMeshRefreshIntervalMs ?? 10_000));
|
|
822
|
+
const neighborHaveTtlMs = Math.max(0, Math.floor(opts.neighborHaveTtlMs ?? 8_000));
|
|
1343
823
|
const neighborRepairBudgetBps = Math.max(0, Math.floor(opts.neighborRepairBudgetBps ?? 0));
|
|
1344
824
|
const neighborRepairBurstMs = Math.max(0, Math.floor(opts.neighborRepairBurstMs ?? 1_000));
|
|
1345
825
|
const neighborRepairTokenCapacity = neighborRepairBudgetBps > 0 && neighborRepairBurstMs > 0
|
|
@@ -1364,6 +844,14 @@ export class FanoutTree extends DirectStream {
|
|
|
1364
844
|
parent: undefined,
|
|
1365
845
|
children: new Map(),
|
|
1366
846
|
rejoinCooldownUntil: 0,
|
|
847
|
+
parentUpgradeCount: 0,
|
|
848
|
+
parentUpgradeLastAt: 0,
|
|
849
|
+
parentUpgradeBackoffMs: 0,
|
|
850
|
+
parentUpgradeBackoffUntil: 0,
|
|
851
|
+
parentUpgradeRetryAfterSeq: -1,
|
|
852
|
+
parentUpgradeStaleRootProbeRound: 0,
|
|
853
|
+
parentUpgradeTrackerNoCapacityUntil: 0,
|
|
854
|
+
parentProbeRejectBackoffMsByHash: new Map(),
|
|
1367
855
|
joinedAtLeastOnce: opts.role === "root",
|
|
1368
856
|
routeFromRoot: opts.role === "root" ? [this.publicKeyHash] : undefined,
|
|
1369
857
|
routeByPeer: new Map(),
|
|
@@ -1413,22 +901,33 @@ export class FanoutTree extends DirectStream {
|
|
|
1413
901
|
: undefined,
|
|
1414
902
|
nextExpectedSeq: 0,
|
|
1415
903
|
missingSeqs: new Set(),
|
|
904
|
+
repairRequestedAtBySeq: new Map(),
|
|
1416
905
|
endSeqExclusive: -1,
|
|
1417
906
|
lastRepairSentAt: 0,
|
|
907
|
+
parentRepairUnansweredStreak: 0,
|
|
1418
908
|
lastParentDataAt: 0,
|
|
909
|
+
lastParentUpgradeActivityAt: 0,
|
|
1419
910
|
receivedAnyParentData: false,
|
|
911
|
+
parentDataLatencySamples: 0,
|
|
912
|
+
parentDataLatencyEwmaMs: 0,
|
|
913
|
+
parentDataLatencyMaxMs: 0,
|
|
1420
914
|
channelPeers: new Map(),
|
|
1421
915
|
knownCandidateAddrs: new Map(),
|
|
1422
916
|
lazyPeers: new Set(),
|
|
1423
917
|
haveByPeer: new Map(),
|
|
1424
918
|
maxSeqSeen: -1,
|
|
1425
919
|
lastIHaveSentAt: 0,
|
|
920
|
+
cachedAnnounceTrackerPeers: undefined,
|
|
1426
921
|
lastIHaveSentMaxSeq: -1,
|
|
1427
922
|
pendingJoin: new Map(),
|
|
1428
923
|
pendingTrackerQuery: new Map(),
|
|
924
|
+
pendingParentProbe: new Map(),
|
|
925
|
+
parentUpgradeReservationsByHash: new Map(),
|
|
926
|
+
parentProbeRejectUntilByHash: new Map(),
|
|
1429
927
|
pendingRouteQuery: new Map(),
|
|
1430
928
|
pendingUnicastAck: new Map(),
|
|
1431
929
|
pendingRouteProxy: new Map(),
|
|
930
|
+
routeProxyByTarget: new Map(),
|
|
1432
931
|
bootstrapOverride: undefined,
|
|
1433
932
|
bootstrapDialTimeoutMs: 10_000,
|
|
1434
933
|
bootstrapMaxPeers: 0,
|
|
@@ -1487,20 +986,128 @@ export class FanoutTree extends DirectStream {
|
|
|
1487
986
|
}
|
|
1488
987
|
ch.pendingUnicastAck.clear();
|
|
1489
988
|
}
|
|
989
|
+
resetParentDataLatency(ch) {
|
|
990
|
+
ch.parentDataLatencySamples = 0;
|
|
991
|
+
ch.parentDataLatencyEwmaMs = 0;
|
|
992
|
+
ch.parentDataLatencyMaxMs = 0;
|
|
993
|
+
}
|
|
994
|
+
noteParentDataLatency(ch, timestamp) {
|
|
995
|
+
const sentAt = Number(timestamp);
|
|
996
|
+
if (!Number.isFinite(sentAt) || sentAt <= 0)
|
|
997
|
+
return;
|
|
998
|
+
const latencyMs = Date.now() - sentAt;
|
|
999
|
+
if (!Number.isFinite(latencyMs) || latencyMs < 0 || latencyMs > 60_000) {
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
const samples = Math.min(0xffff, ch.parentDataLatencySamples + 1);
|
|
1003
|
+
ch.parentDataLatencySamples = samples;
|
|
1004
|
+
ch.parentDataLatencyEwmaMs =
|
|
1005
|
+
samples <= 1
|
|
1006
|
+
? latencyMs
|
|
1007
|
+
: ch.parentDataLatencyEwmaMs * 0.875 + latencyMs * 0.125;
|
|
1008
|
+
ch.parentDataLatencyMaxMs = Math.max(ch.parentDataLatencyMaxMs, latencyMs);
|
|
1009
|
+
}
|
|
1010
|
+
clearParentUpgradeGrace(ch, notifyPreviousParent = false, notifyCandidateParent = false) {
|
|
1011
|
+
const grace = ch.parentUpgradeGrace;
|
|
1012
|
+
if (!grace)
|
|
1013
|
+
return [];
|
|
1014
|
+
clearTimeout(grace.timer);
|
|
1015
|
+
ch.parentUpgradeGrace = undefined;
|
|
1016
|
+
const sends = [];
|
|
1017
|
+
if (notifyPreviousParent && grace.previousParent !== ch.parent) {
|
|
1018
|
+
sends.push(this._sendControl(grace.previousParent, this.codec.encodeLeave(ch.id.key)).catch(() => { }));
|
|
1019
|
+
}
|
|
1020
|
+
if (notifyCandidateParent && grace.candidateParent !== ch.parent) {
|
|
1021
|
+
sends.push(this._sendControl(grace.candidateParent, this.codec.encodeLeave(ch.id.key)).catch(() => { }));
|
|
1022
|
+
}
|
|
1023
|
+
return sends;
|
|
1024
|
+
}
|
|
1025
|
+
commitParentUpgradeGrace(ch) {
|
|
1026
|
+
const grace = ch.parentUpgradeGrace;
|
|
1027
|
+
if (!grace)
|
|
1028
|
+
return;
|
|
1029
|
+
clearTimeout(grace.timer);
|
|
1030
|
+
ch.parentUpgradeGrace = undefined;
|
|
1031
|
+
if (ch.closed || ch.parent !== grace.candidateParent)
|
|
1032
|
+
return;
|
|
1033
|
+
void this._sendControl(grace.previousParent, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
1034
|
+
}
|
|
1035
|
+
rollbackParentUpgradeGrace(ch) {
|
|
1036
|
+
const grace = ch.parentUpgradeGrace;
|
|
1037
|
+
if (!grace)
|
|
1038
|
+
return;
|
|
1039
|
+
clearTimeout(grace.timer);
|
|
1040
|
+
ch.parentUpgradeGrace = undefined;
|
|
1041
|
+
if (ch.closed || ch.parent !== grace.candidateParent)
|
|
1042
|
+
return;
|
|
1043
|
+
ch.parent = grace.previousParent;
|
|
1044
|
+
ch.level = grace.previousLevel;
|
|
1045
|
+
ch.routeFromRoot = grace.previousRouteFromRoot
|
|
1046
|
+
? [...grace.previousRouteFromRoot]
|
|
1047
|
+
: undefined;
|
|
1048
|
+
ch.lastParentDataAt = grace.previousLastParentDataAt;
|
|
1049
|
+
ch.lastParentUpgradeActivityAt =
|
|
1050
|
+
grace.previousLastParentUpgradeActivityAt;
|
|
1051
|
+
ch.receivedAnyParentData = grace.previousReceivedAnyParentData;
|
|
1052
|
+
this.resetParentDataLatency(ch);
|
|
1053
|
+
this.touchPeerHint(ch, grace.previousParent);
|
|
1054
|
+
void this._sendControl(grace.candidateParent, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
1055
|
+
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
1056
|
+
}
|
|
1057
|
+
startParentUpgradeGrace(ch, options) {
|
|
1058
|
+
if (ch.closed || options.previousParent === options.candidateParent)
|
|
1059
|
+
return;
|
|
1060
|
+
void Promise.all(this.clearParentUpgradeGrace(ch, true, true)).catch(() => { });
|
|
1061
|
+
const timeoutMs = Math.max(0, Math.floor(options.timeoutMs));
|
|
1062
|
+
if (timeoutMs <= 0) {
|
|
1063
|
+
void this._sendControl(options.previousParent, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
const timer = setTimeout(() => this.rollbackParentUpgradeGrace(ch), timeoutMs);
|
|
1067
|
+
ch.parentUpgradeGrace = {
|
|
1068
|
+
previousParent: options.previousParent,
|
|
1069
|
+
candidateParent: options.candidateParent,
|
|
1070
|
+
previousLevel: options.previousLevel,
|
|
1071
|
+
previousRouteFromRoot: options.previousRouteFromRoot
|
|
1072
|
+
? [...options.previousRouteFromRoot]
|
|
1073
|
+
: undefined,
|
|
1074
|
+
previousLastParentDataAt: options.previousLastParentDataAt,
|
|
1075
|
+
previousLastParentUpgradeActivityAt: options.previousLastParentUpgradeActivityAt,
|
|
1076
|
+
previousReceivedAnyParentData: options.previousReceivedAnyParentData,
|
|
1077
|
+
timer,
|
|
1078
|
+
candidateFirstMessages: options.candidateFirstMessages,
|
|
1079
|
+
previousFirstMessages: options.previousFirstMessages,
|
|
1080
|
+
minCandidateFirstMessages: options.minCandidateFirstMessages,
|
|
1081
|
+
minCandidateAdvantageMessages: options.minCandidateAdvantageMessages,
|
|
1082
|
+
maxPreviousFirstMessages: options.maxPreviousFirstMessages,
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1490
1085
|
detachFromParent(ch) {
|
|
1086
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
1087
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
1088
|
+
}
|
|
1089
|
+
void Promise.all(this.clearParentUpgradeGrace(ch, true, true)).catch(() => { });
|
|
1491
1090
|
ch.parent = undefined;
|
|
1492
1091
|
ch.level = Number.POSITIVE_INFINITY;
|
|
1493
1092
|
ch.routeFromRoot = undefined;
|
|
1494
1093
|
ch.routeByPeer.clear();
|
|
1495
1094
|
ch.lastParentDataAt = 0;
|
|
1095
|
+
ch.lastParentUpgradeActivityAt = 0;
|
|
1496
1096
|
ch.receivedAnyParentData = false;
|
|
1097
|
+
this.resetParentDataLatency(ch);
|
|
1497
1098
|
ch.pendingJoin.clear();
|
|
1099
|
+
ch.pendingParentProbe.clear();
|
|
1100
|
+
ch.parentShadow = undefined;
|
|
1101
|
+
ch.parentUpgradeBackoffMs = 0;
|
|
1102
|
+
ch.parentUpgradeBackoffUntil = 0;
|
|
1103
|
+
ch.parentUpgradeRetryAfterSeq = -1;
|
|
1104
|
+
ch.parentUpgradeStaleRootProbeRound = 0;
|
|
1105
|
+
ch.parentUpgradeTrackerNoCapacityUntil = 0;
|
|
1106
|
+
ch.parentProbeRejectUntilByHash.clear();
|
|
1107
|
+
ch.parentProbeRejectBackoffMsByHash.clear();
|
|
1498
1108
|
ch.pendingRouteQuery.clear();
|
|
1499
1109
|
this.abortPendingUnicastAcks(ch, new AbortError("fanout channel detached"));
|
|
1500
|
-
|
|
1501
|
-
clearTimeout(pending.timer);
|
|
1502
|
-
}
|
|
1503
|
-
ch.pendingRouteProxy.clear();
|
|
1110
|
+
this.clearRouteProxies(ch);
|
|
1504
1111
|
}
|
|
1505
1112
|
onPeerDisconnectedFromUnderlay(peerHash) {
|
|
1506
1113
|
if (!peerHash)
|
|
@@ -1558,19 +1165,24 @@ export class FanoutTree extends DirectStream {
|
|
|
1558
1165
|
const kickChildren = options?.kickChildren !== false;
|
|
1559
1166
|
const pendingSends = [];
|
|
1560
1167
|
if (notifyParent && !ch.isRoot && ch.parent) {
|
|
1561
|
-
pendingSends.push(this._sendControl(ch.parent, encodeLeave(ch.id.key)).catch(() => { }));
|
|
1168
|
+
pendingSends.push(this._sendControl(ch.parent, this.codec.encodeLeave(ch.id.key)).catch(() => { }));
|
|
1562
1169
|
}
|
|
1170
|
+
pendingSends.push(...this.clearParentUpgradeGrace(ch, notifyParent && !ch.isRoot, false));
|
|
1563
1171
|
if (kickChildren && ch.children.size > 0) {
|
|
1564
1172
|
pendingSends.push(this.kickChildren(ch).catch(() => { }));
|
|
1565
1173
|
}
|
|
1566
1174
|
ch.pendingJoin.clear();
|
|
1567
1175
|
ch.pendingTrackerQuery.clear();
|
|
1176
|
+
ch.pendingParentProbe.clear();
|
|
1177
|
+
ch.parentProbeRejectUntilByHash.clear();
|
|
1178
|
+
ch.parentProbeRejectBackoffMsByHash.clear();
|
|
1179
|
+
ch.parentShadow = undefined;
|
|
1180
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
1181
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
1182
|
+
}
|
|
1568
1183
|
ch.pendingRouteQuery.clear();
|
|
1569
1184
|
this.abortPendingUnicastAcks(ch, new AbortError("fanout channel closed"));
|
|
1570
|
-
|
|
1571
|
-
clearTimeout(pending.timer);
|
|
1572
|
-
}
|
|
1573
|
-
ch.pendingRouteProxy.clear();
|
|
1185
|
+
this.clearRouteProxies(ch);
|
|
1574
1186
|
ch.parent = undefined;
|
|
1575
1187
|
ch.children.clear();
|
|
1576
1188
|
ch.lazyPeers.clear();
|
|
@@ -1603,6 +1215,11 @@ export class FanoutTree extends DirectStream {
|
|
|
1603
1215
|
routeCacheMaxEntries: ch.routeCacheMaxEntries,
|
|
1604
1216
|
};
|
|
1605
1217
|
}
|
|
1218
|
+
/**
|
|
1219
|
+
* Returns diagnostic counters for tests and simulation harnesses.
|
|
1220
|
+
*
|
|
1221
|
+
* @internal
|
|
1222
|
+
*/
|
|
1606
1223
|
getChannelMetrics(topic, root) {
|
|
1607
1224
|
const id = this.getChannelId(topic, root);
|
|
1608
1225
|
return this.getMetricsForSuffixKey(id.suffixKey);
|
|
@@ -1706,7 +1323,9 @@ export class FanoutTree extends DirectStream {
|
|
|
1706
1323
|
updatedAt: Date.now(),
|
|
1707
1324
|
};
|
|
1708
1325
|
}
|
|
1709
|
-
|
|
1326
|
+
// Speculative probe: a miss here says nothing about cache quality (the
|
|
1327
|
+
// caller falls through to resolveRouteToken, which counts its own miss).
|
|
1328
|
+
const route = this.getCachedRoute(ch, targetHash, { recordMiss: false });
|
|
1710
1329
|
if (!route)
|
|
1711
1330
|
return undefined;
|
|
1712
1331
|
const entry = ch.routeByPeer.get(targetHash);
|
|
@@ -1751,8 +1370,7 @@ export class FanoutTree extends DirectStream {
|
|
|
1751
1370
|
ch.knownCandidateAddrs.delete(hash);
|
|
1752
1371
|
ch.knownCandidateAddrs.set(hash, limited);
|
|
1753
1372
|
while (ch.knownCandidateAddrs.size > KNOWN_CANDIDATE_ADDRS_MAX_ENTRIES) {
|
|
1754
|
-
const oldest = ch.knownCandidateAddrs.keys().next()
|
|
1755
|
-
.value;
|
|
1373
|
+
const oldest = ch.knownCandidateAddrs.keys().next().value;
|
|
1756
1374
|
if (!oldest)
|
|
1757
1375
|
break;
|
|
1758
1376
|
ch.knownCandidateAddrs.delete(oldest);
|
|
@@ -1833,23 +1451,27 @@ export class FanoutTree extends DirectStream {
|
|
|
1833
1451
|
ch.metrics.routeCacheEvictions += 1;
|
|
1834
1452
|
}
|
|
1835
1453
|
}
|
|
1836
|
-
getCachedRoute(ch, targetHash) {
|
|
1454
|
+
getCachedRoute(ch, targetHash, opts) {
|
|
1455
|
+
const recordMiss = opts?.recordMiss !== false;
|
|
1837
1456
|
this.pruneRouteCache(ch);
|
|
1838
1457
|
const entry = ch.routeByPeer.get(targetHash);
|
|
1839
1458
|
if (!entry) {
|
|
1840
|
-
|
|
1459
|
+
if (recordMiss)
|
|
1460
|
+
ch.metrics.routeCacheMisses += 1;
|
|
1841
1461
|
return undefined;
|
|
1842
1462
|
}
|
|
1843
1463
|
const now = Date.now();
|
|
1844
1464
|
if (ch.routeCacheTtlMs > 0 && now - entry.updatedAt > ch.routeCacheTtlMs) {
|
|
1845
1465
|
ch.routeByPeer.delete(targetHash);
|
|
1846
|
-
|
|
1466
|
+
if (recordMiss)
|
|
1467
|
+
ch.metrics.routeCacheMisses += 1;
|
|
1847
1468
|
ch.metrics.routeCacheExpirations += 1;
|
|
1848
1469
|
return undefined;
|
|
1849
1470
|
}
|
|
1850
1471
|
if (!this.isRouteValidForChannel(ch, entry.route)) {
|
|
1851
1472
|
ch.routeByPeer.delete(targetHash);
|
|
1852
|
-
|
|
1473
|
+
if (recordMiss)
|
|
1474
|
+
ch.metrics.routeCacheMisses += 1;
|
|
1853
1475
|
return undefined;
|
|
1854
1476
|
}
|
|
1855
1477
|
// LRU touch
|
|
@@ -1883,14 +1505,92 @@ export class FanoutTree extends DirectStream {
|
|
|
1883
1505
|
if (!proxy)
|
|
1884
1506
|
return;
|
|
1885
1507
|
ch.pendingRouteProxy.delete(proxyReqId);
|
|
1508
|
+
const targetKey = `${proxy.direction}:${proxy.targetHash}`;
|
|
1509
|
+
if (ch.routeProxyByTarget.get(targetKey) === proxyReqId) {
|
|
1510
|
+
ch.routeProxyByTarget.delete(targetKey);
|
|
1511
|
+
}
|
|
1886
1512
|
clearTimeout(proxy.timer);
|
|
1513
|
+
const reply = (requester, downstreamReqId) => {
|
|
1514
|
+
void this._sendControl(requester, this.codec.encodeRouteReply(ch.id.key, downstreamReqId, route)).catch(() => { });
|
|
1515
|
+
};
|
|
1887
1516
|
if (proxy.localResolve) {
|
|
1888
1517
|
proxy.localResolve(route);
|
|
1889
|
-
return;
|
|
1890
1518
|
}
|
|
1891
|
-
|
|
1519
|
+
else {
|
|
1520
|
+
reply(proxy.requester, proxy.downstreamReqId);
|
|
1521
|
+
}
|
|
1522
|
+
if (proxy.waiters) {
|
|
1523
|
+
for (const w of proxy.waiters) {
|
|
1524
|
+
if ("localResolve" in w) {
|
|
1525
|
+
w.localResolve(route);
|
|
1526
|
+
}
|
|
1527
|
+
else {
|
|
1528
|
+
reply(w.requester, w.downstreamReqId);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
/**
|
|
1534
|
+
* Coalesce onto an in-flight same-target/same-direction proxy search, if one
|
|
1535
|
+
* exists. Returns true when the requester was attached as a waiter (or
|
|
1536
|
+
* answered empty on waiter overflow) and no new search should start.
|
|
1537
|
+
*/
|
|
1538
|
+
coalesceRouteProxy(ch, targetHash, direction, waiter) {
|
|
1539
|
+
const targetKey = `${direction}:${targetHash}`;
|
|
1540
|
+
const existingId = ch.routeProxyByTarget.get(targetKey);
|
|
1541
|
+
if (existingId == null)
|
|
1542
|
+
return false;
|
|
1543
|
+
const existing = ch.pendingRouteProxy.get(existingId);
|
|
1544
|
+
if (!existing) {
|
|
1545
|
+
ch.routeProxyByTarget.delete(targetKey);
|
|
1546
|
+
return false;
|
|
1547
|
+
}
|
|
1548
|
+
if ((existing.waiters?.length ?? 0) >= ROUTE_PROXY_MAX_WAITERS) {
|
|
1549
|
+
ch.metrics.routeProxyRejected += 1;
|
|
1550
|
+
if ("localResolve" in waiter) {
|
|
1551
|
+
waiter.localResolve(undefined);
|
|
1552
|
+
}
|
|
1553
|
+
else {
|
|
1554
|
+
void this._sendControl(waiter.requester, this.codec.encodeRouteReply(ch.id.key, waiter.downstreamReqId)).catch(() => { });
|
|
1555
|
+
}
|
|
1556
|
+
return true;
|
|
1557
|
+
}
|
|
1558
|
+
(existing.waiters ??= []).push(waiter);
|
|
1559
|
+
ch.metrics.routeProxyCoalesced += 1;
|
|
1560
|
+
return true;
|
|
1561
|
+
}
|
|
1562
|
+
/**
|
|
1563
|
+
* Drop all in-flight route-proxy state for a channel (detach/kick/close).
|
|
1564
|
+
* Local resolvers are settled with `undefined` so root-origin
|
|
1565
|
+
* `resolveRouteToken()` callers never hang; remote requesters time out on
|
|
1566
|
+
* their own deadlines, as before.
|
|
1567
|
+
*/
|
|
1568
|
+
clearRouteProxies(ch) {
|
|
1569
|
+
for (const pending of ch.pendingRouteProxy.values()) {
|
|
1570
|
+
clearTimeout(pending.timer);
|
|
1571
|
+
pending.localResolve?.(undefined);
|
|
1572
|
+
if (pending.waiters) {
|
|
1573
|
+
for (const w of pending.waiters) {
|
|
1574
|
+
if ("localResolve" in w)
|
|
1575
|
+
w.localResolve(undefined);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
ch.pendingRouteProxy.clear();
|
|
1580
|
+
ch.routeProxyByTarget.clear();
|
|
1892
1581
|
}
|
|
1893
1582
|
proxyRouteQuery(ch, requester, downstreamReqId, targetHash, candidates, timeoutMs = ROUTE_PROXY_TIMEOUT_MS) {
|
|
1583
|
+
// Direction matters for coalescing correctness: an upstream lookup and the
|
|
1584
|
+
// root's downstream flood for the same target may legitimately transit the
|
|
1585
|
+
// same node concurrently (e.g. the target lives in the requester's own
|
|
1586
|
+
// branch), and merging them would make them wait on each other.
|
|
1587
|
+
const direction = candidates.length === 1 && candidates[0] === ch.parent ? "up" : "down";
|
|
1588
|
+
if (this.coalesceRouteProxy(ch, targetHash, direction, {
|
|
1589
|
+
requester,
|
|
1590
|
+
downstreamReqId,
|
|
1591
|
+
})) {
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1894
1594
|
const unique = [];
|
|
1895
1595
|
const seen = new Set();
|
|
1896
1596
|
for (const candidate of candidates) {
|
|
@@ -1903,7 +1603,12 @@ export class FanoutTree extends DirectStream {
|
|
|
1903
1603
|
unique.push(candidate);
|
|
1904
1604
|
}
|
|
1905
1605
|
if (unique.length === 0) {
|
|
1906
|
-
void this._sendControl(requester, encodeRouteReply(ch.id.key, downstreamReqId)).catch(() => { });
|
|
1606
|
+
void this._sendControl(requester, this.codec.encodeRouteReply(ch.id.key, downstreamReqId)).catch(() => { });
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
1609
|
+
if (ch.pendingRouteProxy.size >= ROUTE_PROXY_MAX_PENDING) {
|
|
1610
|
+
ch.metrics.routeProxyRejected += 1;
|
|
1611
|
+
void this._sendControl(requester, this.codec.encodeRouteReply(ch.id.key, downstreamReqId)).catch(() => { });
|
|
1907
1612
|
return;
|
|
1908
1613
|
}
|
|
1909
1614
|
ch.metrics.routeProxyQueries += 1;
|
|
@@ -1918,9 +1623,11 @@ export class FanoutTree extends DirectStream {
|
|
|
1918
1623
|
downstreamReqId,
|
|
1919
1624
|
timer,
|
|
1920
1625
|
expectedReplies: new Set(unique),
|
|
1626
|
+
targetHash,
|
|
1627
|
+
direction,
|
|
1921
1628
|
});
|
|
1922
|
-
|
|
1923
|
-
|
|
1629
|
+
ch.routeProxyByTarget.set(`${direction}:${targetHash}`, proxyReqId);
|
|
1630
|
+
void this._sendControlMany(unique, this.codec.encodeRouteQuery(ch.id.key, proxyReqId, targetHash)).catch(() => {
|
|
1924
1631
|
this.completeRouteProxy(ch, proxyReqId);
|
|
1925
1632
|
});
|
|
1926
1633
|
}
|
|
@@ -1950,14 +1657,47 @@ export class FanoutTree extends DirectStream {
|
|
|
1950
1657
|
const timeoutMs = Math.max(1, Math.floor(options?.timeoutMs ?? 3_000));
|
|
1951
1658
|
return await new Promise((resolve, reject) => {
|
|
1952
1659
|
let settled = false;
|
|
1660
|
+
let proxyReqId;
|
|
1953
1661
|
const onAbort = () => {
|
|
1954
1662
|
if (settled)
|
|
1955
1663
|
return;
|
|
1956
1664
|
settled = true;
|
|
1957
|
-
|
|
1958
|
-
|
|
1665
|
+
clearTimeout(backstop);
|
|
1666
|
+
if (proxyReqId != null) {
|
|
1667
|
+
// Tear the search down only if nobody else coalesced onto it;
|
|
1668
|
+
// otherwise let it finish for the waiters (our localResolve
|
|
1669
|
+
// is a no-op once settled).
|
|
1670
|
+
const entry = ch.pendingRouteProxy.get(proxyReqId);
|
|
1671
|
+
if (entry && (!entry.waiters || entry.waiters.length === 0)) {
|
|
1672
|
+
this.completeRouteProxy(ch, proxyReqId);
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1959
1675
|
reject(new AbortError());
|
|
1960
1676
|
};
|
|
1677
|
+
const localResolve = (route) => {
|
|
1678
|
+
if (settled)
|
|
1679
|
+
return;
|
|
1680
|
+
settled = true;
|
|
1681
|
+
clearTimeout(backstop);
|
|
1682
|
+
if (options?.signal) {
|
|
1683
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
1684
|
+
}
|
|
1685
|
+
if (this.isRouteValidForChannel(ch, route)) {
|
|
1686
|
+
this.cacheRoute(ch, route);
|
|
1687
|
+
resolve([...route]);
|
|
1688
|
+
return;
|
|
1689
|
+
}
|
|
1690
|
+
resolve(undefined);
|
|
1691
|
+
};
|
|
1692
|
+
// Per-caller timeout: when coalesced, the shared in-flight search
|
|
1693
|
+
// may outlive this caller's own deadline.
|
|
1694
|
+
const backstop = setTimeout(() => localResolve(undefined), timeoutMs);
|
|
1695
|
+
if (options?.signal) {
|
|
1696
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
1697
|
+
}
|
|
1698
|
+
if (this.coalesceRouteProxy(ch, targetHash, "down", { localResolve })) {
|
|
1699
|
+
return;
|
|
1700
|
+
}
|
|
1961
1701
|
const unique = [];
|
|
1962
1702
|
const seen = new Set();
|
|
1963
1703
|
for (const candidate of candidates) {
|
|
@@ -1970,42 +1710,32 @@ export class FanoutTree extends DirectStream {
|
|
|
1970
1710
|
unique.push(candidate);
|
|
1971
1711
|
}
|
|
1972
1712
|
if (unique.length === 0) {
|
|
1973
|
-
|
|
1713
|
+
localResolve(undefined);
|
|
1714
|
+
return;
|
|
1715
|
+
}
|
|
1716
|
+
if (ch.pendingRouteProxy.size >= ROUTE_PROXY_MAX_PENDING) {
|
|
1717
|
+
ch.metrics.routeProxyRejected += 1;
|
|
1718
|
+
localResolve(undefined);
|
|
1974
1719
|
return;
|
|
1975
1720
|
}
|
|
1976
1721
|
ch.metrics.routeProxyQueries += 1;
|
|
1977
1722
|
ch.metrics.routeProxyFanout += unique.length;
|
|
1978
|
-
|
|
1723
|
+
proxyReqId = this.nextReqId(ch);
|
|
1979
1724
|
const timer = setTimeout(() => {
|
|
1980
1725
|
ch.metrics.routeProxyTimeouts += 1;
|
|
1981
1726
|
this.completeRouteProxy(ch, proxyReqId);
|
|
1982
1727
|
}, timeoutMs);
|
|
1983
|
-
if (options?.signal) {
|
|
1984
|
-
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
1985
|
-
}
|
|
1986
1728
|
ch.pendingRouteProxy.set(proxyReqId, {
|
|
1987
1729
|
requester: this.publicKeyHash,
|
|
1988
1730
|
downstreamReqId: 0,
|
|
1989
1731
|
timer,
|
|
1990
1732
|
expectedReplies: new Set(unique),
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
settled = true;
|
|
1995
|
-
clearTimeout(timer);
|
|
1996
|
-
if (options?.signal) {
|
|
1997
|
-
options.signal.removeEventListener("abort", onAbort);
|
|
1998
|
-
}
|
|
1999
|
-
if (this.isRouteValidForChannel(ch, route)) {
|
|
2000
|
-
this.cacheRoute(ch, route);
|
|
2001
|
-
resolve([...route]);
|
|
2002
|
-
return;
|
|
2003
|
-
}
|
|
2004
|
-
resolve(undefined);
|
|
2005
|
-
},
|
|
1733
|
+
targetHash,
|
|
1734
|
+
direction: "down",
|
|
1735
|
+
localResolve,
|
|
2006
1736
|
});
|
|
2007
|
-
|
|
2008
|
-
|
|
1737
|
+
ch.routeProxyByTarget.set(`down:${targetHash}`, proxyReqId);
|
|
1738
|
+
void this._sendControlMany(unique, this.codec.encodeRouteQuery(ch.id.key, proxyReqId, targetHash)).catch(() => {
|
|
2009
1739
|
this.completeRouteProxy(ch, proxyReqId);
|
|
2010
1740
|
});
|
|
2011
1741
|
});
|
|
@@ -2052,8 +1782,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2052
1782
|
resolve(undefined);
|
|
2053
1783
|
},
|
|
2054
1784
|
});
|
|
2055
|
-
void this._sendControl(ch.parent, encodeRouteQuery(ch.id.key, reqId, targetHash))
|
|
2056
|
-
.catch((error) => {
|
|
1785
|
+
void this._sendControl(ch.parent, this.codec.encodeRouteQuery(ch.id.key, reqId, targetHash)).catch((error) => {
|
|
2057
1786
|
if (settled)
|
|
2058
1787
|
return;
|
|
2059
1788
|
settled = true;
|
|
@@ -2101,7 +1830,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2101
1830
|
throw new Error("Unicast route token target mismatch");
|
|
2102
1831
|
}
|
|
2103
1832
|
if (target === this.publicKeyHash) {
|
|
2104
|
-
const wire = encodeUnicast(ch.id.key, toRoute, payload);
|
|
1833
|
+
const wire = this.codec.encodeUnicast(ch.id.key, toRoute, payload);
|
|
2105
1834
|
const message = await this.createMessage(wire, {
|
|
2106
1835
|
mode: new AnyWhere(),
|
|
2107
1836
|
priority: CONTROL_PRIORITY,
|
|
@@ -2137,7 +1866,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2137
1866
|
if (replyRoute[replyRoute.length - 1] !== selfHash) {
|
|
2138
1867
|
throw new Error("Cannot unicast with ACK: self route token target mismatch");
|
|
2139
1868
|
}
|
|
2140
|
-
const data = encodeUnicast(ch.id.key, toRoute, payload, {
|
|
1869
|
+
const data = this.codec.encodeUnicast(ch.id.key, toRoute, payload, {
|
|
2141
1870
|
ackToken,
|
|
2142
1871
|
replyRoute,
|
|
2143
1872
|
});
|
|
@@ -2231,7 +1960,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2231
1960
|
if (toRoute[0] !== ch.id.root) {
|
|
2232
1961
|
throw new Error("Unicast route token root mismatch");
|
|
2233
1962
|
}
|
|
2234
|
-
const data = encodeUnicast(ch.id.key, toRoute, payload);
|
|
1963
|
+
const data = this.codec.encodeUnicast(ch.id.key, toRoute, payload);
|
|
2235
1964
|
if (ch.isRoot) {
|
|
2236
1965
|
const target = toRoute[toRoute.length - 1];
|
|
2237
1966
|
if (target === this.publicKeyHash) {
|
|
@@ -2314,7 +2043,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2314
2043
|
if (!ch.parent) {
|
|
2315
2044
|
throw new Error(`Cannot proxy publish while not attached to a parent (topic=${topic} root=${root} self=${this.publicKeyHash})`);
|
|
2316
2045
|
}
|
|
2317
|
-
await this._sendControl(ch.parent, encodePublishProxy(ch.id.key, payload));
|
|
2046
|
+
await this._sendControl(ch.parent, this.codec.encodePublishProxy(ch.id.key, payload));
|
|
2318
2047
|
}
|
|
2319
2048
|
async publishToChannelMaybe(topic, root, payload) {
|
|
2320
2049
|
const id = this.getChannelId(topic, root);
|
|
@@ -2412,7 +2141,8 @@ export class FanoutTree extends DirectStream {
|
|
|
2412
2141
|
return;
|
|
2413
2142
|
if (!ch.isRoot)
|
|
2414
2143
|
return;
|
|
2415
|
-
|
|
2144
|
+
ch.endSeqExclusive = Math.max(ch.endSeqExclusive, lastSeqExclusive);
|
|
2145
|
+
await this._sendControlMany([...ch.children.keys()], this.codec.encodeEnd(ch.id.key, lastSeqExclusive));
|
|
2416
2146
|
}
|
|
2417
2147
|
makeDataId(ch, seq) {
|
|
2418
2148
|
const id = new Uint8Array(32);
|
|
@@ -2497,6 +2227,14 @@ export class FanoutTree extends DirectStream {
|
|
|
2497
2227
|
m.trackerFeedbackSent += transmissions;
|
|
2498
2228
|
m.controlBytesSentTracker += sentBytes;
|
|
2499
2229
|
break;
|
|
2230
|
+
case MSG_PARENT_PROBE_REQ:
|
|
2231
|
+
m.parentProbeReqSent += transmissions;
|
|
2232
|
+
m.controlBytesSentJoin += sentBytes;
|
|
2233
|
+
break;
|
|
2234
|
+
case MSG_PARENT_PROBE_REPLY:
|
|
2235
|
+
m.parentProbeReplySent += transmissions;
|
|
2236
|
+
m.controlBytesSentJoin += sentBytes;
|
|
2237
|
+
break;
|
|
2500
2238
|
case MSG_PROVIDER_ANNOUNCE:
|
|
2501
2239
|
case MSG_PROVIDER_QUERY:
|
|
2502
2240
|
case MSG_PROVIDER_REPLY:
|
|
@@ -2562,6 +2300,14 @@ export class FanoutTree extends DirectStream {
|
|
|
2562
2300
|
m.trackerFeedbackReceived += 1;
|
|
2563
2301
|
m.controlBytesReceivedTracker += bytesReceived;
|
|
2564
2302
|
break;
|
|
2303
|
+
case MSG_PARENT_PROBE_REQ:
|
|
2304
|
+
m.parentProbeReqReceived += 1;
|
|
2305
|
+
m.controlBytesReceivedJoin += bytesReceived;
|
|
2306
|
+
break;
|
|
2307
|
+
case MSG_PARENT_PROBE_REPLY:
|
|
2308
|
+
m.parentProbeReplyReceived += 1;
|
|
2309
|
+
m.controlBytesReceivedJoin += bytesReceived;
|
|
2310
|
+
break;
|
|
2565
2311
|
case MSG_PROVIDER_ANNOUNCE:
|
|
2566
2312
|
case MSG_PROVIDER_QUERY:
|
|
2567
2313
|
case MSG_PROVIDER_REPLY:
|
|
@@ -2636,7 +2382,8 @@ export class FanoutTree extends DirectStream {
|
|
|
2636
2382
|
if (elapsedMs <= 0)
|
|
2637
2383
|
return;
|
|
2638
2384
|
ch.neighborRepairLastRefillAt = now;
|
|
2639
|
-
ch.neighborRepairTokens = Math.min(ch.neighborRepairTokenCapacity, ch.neighborRepairTokens +
|
|
2385
|
+
ch.neighborRepairTokens = Math.min(ch.neighborRepairTokenCapacity, ch.neighborRepairTokens +
|
|
2386
|
+
(elapsedMs * ch.neighborRepairBudgetBps) / 1_000);
|
|
2640
2387
|
}
|
|
2641
2388
|
takeIngressBudget(ch, kind, fromHash, costBytes, now = Date.now()) {
|
|
2642
2389
|
if (!fromHash)
|
|
@@ -2673,7 +2420,7 @@ export class FanoutTree extends DirectStream {
|
|
|
2673
2420
|
async _sendData(ch, to, seq, payload) {
|
|
2674
2421
|
this.markCached(ch, seq, payload);
|
|
2675
2422
|
ch.maxSeqSeen = Math.max(ch.maxSeqSeen, seq);
|
|
2676
|
-
const framed = encodeData(payload);
|
|
2423
|
+
const framed = this.codec.encodeData(payload);
|
|
2677
2424
|
const message = await this.createMessage(framed, {
|
|
2678
2425
|
mode: new AnyWhere(),
|
|
2679
2426
|
priority: DATA_PRIORITY,
|
|
@@ -2742,14 +2489,12 @@ export class FanoutTree extends DirectStream {
|
|
|
2742
2489
|
});
|
|
2743
2490
|
const kickCount = Math.min(OVERLOAD_KICK_MAX_PER_EVENT, droppedCandidates.length);
|
|
2744
2491
|
if (kickCount > 0) {
|
|
2745
|
-
const toKick = droppedCandidates
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
ch.dataWriteFailStreakByChild.delete(h);
|
|
2749
|
-
}
|
|
2492
|
+
const toKick = droppedCandidates
|
|
2493
|
+
.slice(0, kickCount)
|
|
2494
|
+
.map((c) => c.hash);
|
|
2750
2495
|
ch.overloadStreak = 0;
|
|
2751
2496
|
ch.lastOverloadKickAt = now;
|
|
2752
|
-
void this.
|
|
2497
|
+
void this.kickChildHashes(ch, toKick).catch(() => { });
|
|
2753
2498
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
2754
2499
|
}
|
|
2755
2500
|
}
|
|
@@ -2778,13 +2523,6 @@ export class FanoutTree extends DirectStream {
|
|
|
2778
2523
|
const stream = c.stream;
|
|
2779
2524
|
if (!stream.isWritable) {
|
|
2780
2525
|
writeDrops += 1;
|
|
2781
|
-
if (ch.children.has(c.hash)) {
|
|
2782
|
-
const streak = (ch.dataWriteFailStreakByChild.get(c.hash) ?? 0) + 1;
|
|
2783
|
-
ch.dataWriteFailStreakByChild.set(c.hash, streak);
|
|
2784
|
-
if (streak >= DATA_WRITE_FAIL_KICK_STREAK_THRESHOLD) {
|
|
2785
|
-
writeFailKickCandidates.push(c.hash);
|
|
2786
|
-
}
|
|
2787
|
-
}
|
|
2788
2526
|
continue;
|
|
2789
2527
|
}
|
|
2790
2528
|
try {
|
|
@@ -2831,12 +2569,8 @@ export class FanoutTree extends DirectStream {
|
|
|
2831
2569
|
const kickCount = Math.min(DATA_WRITE_FAIL_KICK_MAX_PER_EVENT, unique.length);
|
|
2832
2570
|
if (kickCount > 0) {
|
|
2833
2571
|
const toKick = unique.slice(0, kickCount);
|
|
2834
|
-
for (const h of toKick) {
|
|
2835
|
-
ch.children.delete(h);
|
|
2836
|
-
ch.dataWriteFailStreakByChild.delete(h);
|
|
2837
|
-
}
|
|
2838
2572
|
ch.lastDataWriteFailKickAt = now;
|
|
2839
|
-
void this.
|
|
2573
|
+
void this.kickChildHashes(ch, toKick, { resetPeerConnections: true }).catch(() => { });
|
|
2840
2574
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
2841
2575
|
}
|
|
2842
2576
|
}
|
|
@@ -2915,14 +2649,12 @@ export class FanoutTree extends DirectStream {
|
|
|
2915
2649
|
});
|
|
2916
2650
|
const kickCount = Math.min(OVERLOAD_KICK_MAX_PER_EVENT, droppedCandidates.length);
|
|
2917
2651
|
if (kickCount > 0) {
|
|
2918
|
-
const toKick = droppedCandidates
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
ch.dataWriteFailStreakByChild.delete(h);
|
|
2922
|
-
}
|
|
2652
|
+
const toKick = droppedCandidates
|
|
2653
|
+
.slice(0, kickCount)
|
|
2654
|
+
.map((c) => c.hash);
|
|
2923
2655
|
ch.overloadStreak = 0;
|
|
2924
2656
|
ch.lastOverloadKickAt = now;
|
|
2925
|
-
void this.
|
|
2657
|
+
void this.kickChildHashes(ch, toKick).catch(() => { });
|
|
2926
2658
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
2927
2659
|
}
|
|
2928
2660
|
}
|
|
@@ -2940,11 +2672,6 @@ export class FanoutTree extends DirectStream {
|
|
|
2940
2672
|
const stream = c.stream;
|
|
2941
2673
|
if (!stream.isWritable) {
|
|
2942
2674
|
writeDrops += 1;
|
|
2943
|
-
const streak = (ch.dataWriteFailStreakByChild.get(c.hash) ?? 0) + 1;
|
|
2944
|
-
ch.dataWriteFailStreakByChild.set(c.hash, streak);
|
|
2945
|
-
if (streak >= DATA_WRITE_FAIL_KICK_STREAK_THRESHOLD) {
|
|
2946
|
-
writeFailKickCandidates.push(c.hash);
|
|
2947
|
-
}
|
|
2948
2675
|
continue;
|
|
2949
2676
|
}
|
|
2950
2677
|
try {
|
|
@@ -2989,12 +2716,8 @@ export class FanoutTree extends DirectStream {
|
|
|
2989
2716
|
const kickCount = Math.min(DATA_WRITE_FAIL_KICK_MAX_PER_EVENT, unique.length);
|
|
2990
2717
|
if (kickCount > 0) {
|
|
2991
2718
|
const toKick = unique.slice(0, kickCount);
|
|
2992
|
-
for (const h of toKick) {
|
|
2993
|
-
ch.children.delete(h);
|
|
2994
|
-
ch.dataWriteFailStreakByChild.delete(h);
|
|
2995
|
-
}
|
|
2996
2719
|
ch.lastDataWriteFailKickAt = now;
|
|
2997
|
-
void this.
|
|
2720
|
+
void this.kickChildHashes(ch, toKick, { resetPeerConnections: true }).catch(() => { });
|
|
2998
2721
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
2999
2722
|
}
|
|
3000
2723
|
}
|
|
@@ -3002,6 +2725,7 @@ export class FanoutTree extends DirectStream {
|
|
|
3002
2725
|
noteReceivedSeq(ch, fromHash, seq) {
|
|
3003
2726
|
if (!ch.repairEnabled)
|
|
3004
2727
|
return;
|
|
2728
|
+
ch.repairRequestedAtBySeq.delete(seq);
|
|
3005
2729
|
if (!ch.parent || fromHash !== ch.parent) {
|
|
3006
2730
|
// Neighbor-assisted repair may deliver payload from a peer other than the
|
|
3007
2731
|
// current parent; treat it as a "hole fill" only.
|
|
@@ -3025,7 +2749,9 @@ export class FanoutTree extends DirectStream {
|
|
|
3025
2749
|
}
|
|
3026
2750
|
if (seq >= ch.nextExpectedSeq) {
|
|
3027
2751
|
const maxBackfill = Math.max(0, ch.repairMaxBackfillMessages);
|
|
3028
|
-
const start = maxBackfill > 0
|
|
2752
|
+
const start = maxBackfill > 0
|
|
2753
|
+
? Math.max(ch.nextExpectedSeq, Math.max(0, seq - maxBackfill))
|
|
2754
|
+
: ch.nextExpectedSeq;
|
|
3029
2755
|
for (let s = start; s < seq; s++)
|
|
3030
2756
|
ch.missingSeqs.add(s);
|
|
3031
2757
|
ch.nextExpectedSeq = seq + 1;
|
|
@@ -3072,21 +2798,32 @@ export class FanoutTree extends DirectStream {
|
|
|
3072
2798
|
ch.missingSeqs.delete(s);
|
|
3073
2799
|
}
|
|
3074
2800
|
}
|
|
2801
|
+
for (const s of ch.repairRequestedAtBySeq.keys()) {
|
|
2802
|
+
if (!ch.missingSeqs.has(s))
|
|
2803
|
+
ch.repairRequestedAtBySeq.delete(s);
|
|
2804
|
+
}
|
|
3075
2805
|
if (ch.missingSeqs.size === 0)
|
|
3076
2806
|
return false;
|
|
3077
|
-
if (ch.repairIntervalMs > 0 &&
|
|
2807
|
+
if (ch.repairIntervalMs > 0 &&
|
|
2808
|
+
now - ch.lastRepairSentAt < ch.repairIntervalMs) {
|
|
3078
2809
|
return false;
|
|
3079
2810
|
}
|
|
3080
|
-
ch.
|
|
3081
|
-
const missing = [...ch.missingSeqs]
|
|
2811
|
+
const repairRetryMs = Math.max(REPAIR_RETRY_MIN_MS, ch.repairIntervalMs * REPAIR_RETRY_INTERVAL_FACTOR);
|
|
2812
|
+
const missing = [...ch.missingSeqs]
|
|
2813
|
+
.filter((s) => now - (ch.repairRequestedAtBySeq.get(s) ?? 0) >= repairRetryMs)
|
|
2814
|
+
.sort((a, b) => a - b);
|
|
3082
2815
|
const count = Math.min(ch.repairMaxPerReq, missing.length, 255);
|
|
3083
2816
|
if (count <= 0)
|
|
3084
2817
|
return false;
|
|
2818
|
+
ch.lastRepairSentAt = now;
|
|
3085
2819
|
const reqId = (this.random() * 0xffffffff) >>> 0;
|
|
3086
2820
|
const slice = missing.slice(0, count);
|
|
2821
|
+
for (const s of slice)
|
|
2822
|
+
ch.repairRequestedAtBySeq.set(s, now);
|
|
3087
2823
|
let sent = false;
|
|
3088
2824
|
if (ch.parent) {
|
|
3089
|
-
await this._sendControl(ch.parent, encodeRepairReq(ch.id.key, reqId, slice));
|
|
2825
|
+
await this._sendControl(ch.parent, this.codec.encodeRepairReq(ch.id.key, reqId, slice));
|
|
2826
|
+
ch.parentRepairUnansweredStreak += 1;
|
|
3090
2827
|
sent = true;
|
|
3091
2828
|
}
|
|
3092
2829
|
if (ch.neighborRepair && ch.neighborRepairPeers > 0) {
|
|
@@ -3136,35 +2873,25 @@ export class FanoutTree extends DirectStream {
|
|
|
3136
2873
|
updatedAt,
|
|
3137
2874
|
};
|
|
3138
2875
|
});
|
|
3139
|
-
scored.
|
|
2876
|
+
const viable = scored.filter((s) => s.coverage > 0);
|
|
2877
|
+
viable.sort((a, b) => {
|
|
3140
2878
|
if (a.coverage !== b.coverage)
|
|
3141
2879
|
return b.coverage - a.coverage;
|
|
3142
|
-
if (a.
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
return bl - al;
|
|
3151
|
-
}
|
|
3152
|
-
else {
|
|
3153
|
-
const al = ch.lazyPeers.has(a.peerHash) ? 1 : 0;
|
|
3154
|
-
const bl = ch.lazyPeers.has(b.peerHash) ? 1 : 0;
|
|
3155
|
-
if (al !== bl)
|
|
3156
|
-
return bl - al;
|
|
3157
|
-
if (a.successRate !== b.successRate)
|
|
3158
|
-
return b.successRate - a.successRate;
|
|
3159
|
-
if (a.updatedAt !== b.updatedAt)
|
|
3160
|
-
return b.updatedAt - a.updatedAt;
|
|
3161
|
-
}
|
|
2880
|
+
if (a.successRate !== b.successRate)
|
|
2881
|
+
return b.successRate - a.successRate;
|
|
2882
|
+
if (a.updatedAt !== b.updatedAt)
|
|
2883
|
+
return b.updatedAt - a.updatedAt;
|
|
2884
|
+
const al = ch.lazyPeers.has(a.peerHash) ? 1 : 0;
|
|
2885
|
+
const bl = ch.lazyPeers.has(b.peerHash) ? 1 : 0;
|
|
2886
|
+
if (al !== bl)
|
|
2887
|
+
return bl - al;
|
|
3162
2888
|
return a.peerHash < b.peerHash ? -1 : a.peerHash > b.peerHash ? 1 : 0;
|
|
3163
2889
|
});
|
|
3164
|
-
let peers =
|
|
2890
|
+
let peers = viable.slice(0, ch.neighborRepairPeers).map((s) => s.peerHash);
|
|
3165
2891
|
if (peers.length > 0) {
|
|
3166
|
-
const bytes = encodeFetchReq(ch.id.key, reqId, slice);
|
|
3167
|
-
if (ch.neighborRepairBudgetBps > 0 &&
|
|
2892
|
+
const bytes = this.codec.encodeFetchReq(ch.id.key, reqId, slice);
|
|
2893
|
+
if (ch.neighborRepairBudgetBps > 0 &&
|
|
2894
|
+
ch.neighborRepairTokenCapacity > 0) {
|
|
3168
2895
|
this.refillNeighborRepairTokens(ch, now);
|
|
3169
2896
|
const costPerPeer = Math.max(1, bytes.byteLength);
|
|
3170
2897
|
const affordable = Math.floor(ch.neighborRepairTokens / costPerPeer);
|
|
@@ -3269,13 +2996,33 @@ export class FanoutTree extends DirectStream {
|
|
|
3269
2996
|
const bootstraps = this.getBootstrapsForChannel(ch);
|
|
3270
2997
|
if (bootstraps.length === 0)
|
|
3271
2998
|
return;
|
|
3272
|
-
|
|
2999
|
+
// Announce to ALL trackers, not a random one per round: with B bootstraps
|
|
3000
|
+
// and per-round random selection a given directory only refreshes an
|
|
3001
|
+
// entry every ~announceIntervalMs * B on average, which starves
|
|
3002
|
+
// directories (and exceeds the announce TTL entirely once B grows).
|
|
3003
|
+
// Use already-connected tracker peers and only fall back to dialing when
|
|
3004
|
+
// none are connected: serial dial timeouts here would otherwise make
|
|
3005
|
+
// every announce tick glacial.
|
|
3006
|
+
let peers = ch.cachedAnnounceTrackerPeers ?? [];
|
|
3007
|
+
peers = peers.filter((h) => Boolean(this.peers.get(h)));
|
|
3008
|
+
if (peers.length === 0) {
|
|
3009
|
+
// Dial only when no tracker is connected at all: dialing every tick
|
|
3010
|
+
// makes the announce loop glacial and starves everything behind it.
|
|
3011
|
+
peers = await this.ensureBootstrapPeers(bootstraps, ch.bootstrapDialTimeoutMs, signal, 0);
|
|
3012
|
+
}
|
|
3013
|
+
ch.cachedAnnounceTrackerPeers = peers;
|
|
3273
3014
|
if (peers.length === 0)
|
|
3274
3015
|
return;
|
|
3275
3016
|
this.pruneDisconnectedChildren(ch);
|
|
3276
|
-
const
|
|
3017
|
+
const parentUpgradeNoCapacity = ch.isRoot && ch.parentUpgradeTrackerNoCapacityUntil > now;
|
|
3018
|
+
const freeSlots = parentUpgradeNoCapacity
|
|
3019
|
+
? 0
|
|
3020
|
+
: Math.max(0, ch.effectiveMaxChildren - ch.children.size);
|
|
3021
|
+
const announceTtlMs = parentUpgradeNoCapacity
|
|
3022
|
+
? Math.max(1_000, Math.min(ch.announceTtlMs, ch.parentUpgradeTrackerNoCapacityUntil - now))
|
|
3023
|
+
: ch.announceTtlMs;
|
|
3277
3024
|
const addrs = this.getSelfAnnounceAddrs();
|
|
3278
|
-
const bytes = encodeTrackerAnnounce(ch.id.key,
|
|
3025
|
+
const bytes = this.codec.encodeTrackerAnnounce(ch.id.key, announceTtlMs, Number.isFinite(ch.level) ? ch.level : 0xffff, ch.effectiveMaxChildren, freeSlots, ch.bidPerByte, addrs);
|
|
3279
3026
|
await this._sendControlMany(peers, bytes);
|
|
3280
3027
|
}
|
|
3281
3028
|
async _announceLoop(ch) {
|
|
@@ -3289,6 +3036,18 @@ export class FanoutTree extends DirectStream {
|
|
|
3289
3036
|
catch {
|
|
3290
3037
|
// ignore
|
|
3291
3038
|
}
|
|
3039
|
+
try {
|
|
3040
|
+
// Child watermark heartbeat first: it only writes to already
|
|
3041
|
+
// connected peers (children/mesh) and must not starve behind the
|
|
3042
|
+
// dial-bound tracker announce below. Parents (root included - it
|
|
3043
|
+
// runs no repair/mesh loop) advertise their have-range to children
|
|
3044
|
+
// so a child whose data writes were silently lost can detect the
|
|
3045
|
+
// gap and repair.
|
|
3046
|
+
await this.maybeSendIHave(ch, Date.now());
|
|
3047
|
+
}
|
|
3048
|
+
catch {
|
|
3049
|
+
// ignore
|
|
3050
|
+
}
|
|
3292
3051
|
try {
|
|
3293
3052
|
await this.announceToTrackers(ch, signal);
|
|
3294
3053
|
}
|
|
@@ -3408,27 +3167,62 @@ export class FanoutTree extends DirectStream {
|
|
|
3408
3167
|
return { haveFrom, haveToExclusive };
|
|
3409
3168
|
}
|
|
3410
3169
|
async maybeSendIHave(ch, now = Date.now()) {
|
|
3411
|
-
|
|
3170
|
+
const meshEnabled = ch.neighborRepair && ch.neighborMeshPeers > 0;
|
|
3171
|
+
const childWatermarkEnabled = ch.repairEnabled && ch.children.size > 0;
|
|
3172
|
+
if (!meshEnabled && !childWatermarkEnabled) {
|
|
3412
3173
|
return;
|
|
3413
|
-
|
|
3174
|
+
}
|
|
3175
|
+
if (ch.neighborAnnounceIntervalMs <= 0) {
|
|
3414
3176
|
return;
|
|
3415
|
-
|
|
3177
|
+
}
|
|
3178
|
+
if (now - ch.lastIHaveSentAt < ch.neighborAnnounceIntervalMs) {
|
|
3416
3179
|
return;
|
|
3417
|
-
|
|
3180
|
+
}
|
|
3181
|
+
const hasNewData = ch.maxSeqSeen > ch.lastIHaveSentMaxSeq;
|
|
3182
|
+
// Children also need a periodic watermark even without new data: a burst
|
|
3183
|
+
// of data writes can be lost on a not-yet-writable stream and a child
|
|
3184
|
+
// that received nothing has no gap signal to trigger repair. A periodic
|
|
3185
|
+
// IHAVE from the parent converts that silent loss into a repairable gap.
|
|
3186
|
+
const heartbeatDue = childWatermarkEnabled &&
|
|
3187
|
+
(ch.maxSeqSeen >= 0 || ch.endSeqExclusive > 0) &&
|
|
3188
|
+
now - ch.lastIHaveSentAt >=
|
|
3189
|
+
Math.max(ch.neighborAnnounceIntervalMs * 4, 2_000);
|
|
3190
|
+
if (!hasNewData && !heartbeatDue) {
|
|
3418
3191
|
return;
|
|
3419
|
-
|
|
3192
|
+
}
|
|
3193
|
+
const recipients = new Set();
|
|
3194
|
+
if (meshEnabled) {
|
|
3195
|
+
for (const h of ch.lazyPeers) {
|
|
3196
|
+
if (this.peers.get(h))
|
|
3197
|
+
recipients.add(h);
|
|
3198
|
+
}
|
|
3199
|
+
}
|
|
3200
|
+
if (childWatermarkEnabled) {
|
|
3201
|
+
for (const h of ch.children.keys()) {
|
|
3202
|
+
if (this.peers.get(h))
|
|
3203
|
+
recipients.add(h);
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
if (recipients.size === 0) {
|
|
3420
3207
|
return;
|
|
3208
|
+
}
|
|
3209
|
+
let sent = false;
|
|
3421
3210
|
const range = this.getHaveRange(ch);
|
|
3422
|
-
if (
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3211
|
+
if (range && range.haveToExclusive > range.haveFrom) {
|
|
3212
|
+
await this._sendControlMany([...recipients], this.codec.encodeIHave(ch.id.key, range.haveFrom, range.haveToExclusive));
|
|
3213
|
+
sent = true;
|
|
3214
|
+
}
|
|
3215
|
+
if (childWatermarkEnabled && ch.endSeqExclusive > 0) {
|
|
3216
|
+
const children = [...ch.children.keys()].filter((h) => this.peers.get(h));
|
|
3217
|
+
if (children.length > 0) {
|
|
3218
|
+
await this._sendControlMany(children, this.codec.encodeEnd(ch.id.key, ch.endSeqExclusive));
|
|
3219
|
+
sent = true;
|
|
3220
|
+
}
|
|
3221
|
+
}
|
|
3222
|
+
if (sent) {
|
|
3223
|
+
ch.lastIHaveSentAt = now;
|
|
3224
|
+
ch.lastIHaveSentMaxSeq = ch.maxSeqSeen;
|
|
3225
|
+
}
|
|
3432
3226
|
}
|
|
3433
3227
|
async ensureMeshPeers(ch, signal) {
|
|
3434
3228
|
if (ch.neighborMeshPeers <= 0)
|
|
@@ -3471,6 +3265,10 @@ export class FanoutTree extends DirectStream {
|
|
|
3471
3265
|
}
|
|
3472
3266
|
}
|
|
3473
3267
|
async refreshMeshCandidates(ch, signal) {
|
|
3268
|
+
if (ch.neighborMeshPeers <= 0)
|
|
3269
|
+
return;
|
|
3270
|
+
if (ch.lazyPeers.size >= ch.neighborMeshPeers)
|
|
3271
|
+
return;
|
|
3474
3272
|
const bootstraps = this.getBootstrapsForChannel(ch);
|
|
3475
3273
|
if (bootstraps.length === 0)
|
|
3476
3274
|
return;
|
|
@@ -3497,18 +3295,23 @@ export class FanoutTree extends DirectStream {
|
|
|
3497
3295
|
try {
|
|
3498
3296
|
this.pruneLazyPeers(ch);
|
|
3499
3297
|
this.pruneHaveByPeer(ch, now);
|
|
3298
|
+
await this.ensureMeshPeers(ch, signal);
|
|
3500
3299
|
const refreshMs = Math.max(0, ch.neighborMeshRefreshIntervalMs);
|
|
3501
|
-
if (
|
|
3300
|
+
if (ch.lazyPeers.size < ch.neighborMeshPeers &&
|
|
3301
|
+
(refreshMs === 0 || now - lastRefreshAt >= refreshMs)) {
|
|
3502
3302
|
lastRefreshAt = now;
|
|
3503
3303
|
await this.refreshMeshCandidates(ch, signal);
|
|
3304
|
+
await this.ensureMeshPeers(ch, signal);
|
|
3504
3305
|
}
|
|
3505
|
-
await this.ensureMeshPeers(ch, signal);
|
|
3506
3306
|
await this.maybeSendIHave(ch, now);
|
|
3507
3307
|
}
|
|
3508
3308
|
catch {
|
|
3509
3309
|
// ignore
|
|
3510
3310
|
}
|
|
3511
|
-
const intervals = [
|
|
3311
|
+
const intervals = [
|
|
3312
|
+
ch.neighborAnnounceIntervalMs,
|
|
3313
|
+
ch.neighborMeshRefreshIntervalMs,
|
|
3314
|
+
].filter((v) => v > 0);
|
|
3512
3315
|
const sleepMs = intervals.length > 0 ? Math.min(...intervals) : 500;
|
|
3513
3316
|
await delay(Math.max(50, sleepMs));
|
|
3514
3317
|
}
|
|
@@ -3522,7 +3325,7 @@ export class FanoutTree extends DirectStream {
|
|
|
3522
3325
|
const p = new Promise((resolve) => {
|
|
3523
3326
|
ch.pendingTrackerQuery.set(reqId, { resolve });
|
|
3524
3327
|
});
|
|
3525
|
-
void this._sendControl(trackerHash, encodeTrackerQuery(ch.id.key, reqId, want));
|
|
3328
|
+
void this._sendControl(trackerHash, this.codec.encodeTrackerQuery(ch.id.key, reqId, want));
|
|
3526
3329
|
const res = await Promise.race([
|
|
3527
3330
|
p,
|
|
3528
3331
|
delay(perTrackerTimeout, { signal }).then(() => null),
|
|
@@ -3552,7 +3355,11 @@ export class FanoutTree extends DirectStream {
|
|
|
3552
3355
|
return false;
|
|
3553
3356
|
try {
|
|
3554
3357
|
await this.components.connectionManager.openConnection(a);
|
|
3555
|
-
await this.waitFor(hash, {
|
|
3358
|
+
await this.waitFor(hash, {
|
|
3359
|
+
seek: "present",
|
|
3360
|
+
timeout: timeoutMs,
|
|
3361
|
+
signal,
|
|
3362
|
+
});
|
|
3556
3363
|
return true;
|
|
3557
3364
|
}
|
|
3558
3365
|
catch {
|
|
@@ -3567,7 +3374,9 @@ export class FanoutTree extends DirectStream {
|
|
|
3567
3374
|
return [];
|
|
3568
3375
|
try {
|
|
3569
3376
|
const peer = await this.components.peerStore.get(stream.peerId);
|
|
3570
|
-
const addresses = Array.isArray(peer?.addresses)
|
|
3377
|
+
const addresses = Array.isArray(peer?.addresses)
|
|
3378
|
+
? peer.addresses
|
|
3379
|
+
: [];
|
|
3571
3380
|
const out = [];
|
|
3572
3381
|
for (const a of addresses) {
|
|
3573
3382
|
const ma = a?.multiaddr ?? a;
|
|
@@ -3634,7 +3443,10 @@ export class FanoutTree extends DirectStream {
|
|
|
3634
3443
|
if (out.length >= max)
|
|
3635
3444
|
break;
|
|
3636
3445
|
seen.add(e.hash);
|
|
3637
|
-
out.push({
|
|
3446
|
+
out.push({
|
|
3447
|
+
hash: e.hash,
|
|
3448
|
+
addrs: e.addrs.slice(0, JOIN_REJECT_REDIRECT_ADDR_MAX),
|
|
3449
|
+
});
|
|
3638
3450
|
}
|
|
3639
3451
|
}
|
|
3640
3452
|
if (out.length >= max)
|
|
@@ -3665,12 +3477,166 @@ export class FanoutTree extends DirectStream {
|
|
|
3665
3477
|
}
|
|
3666
3478
|
async sendJoinReject(ch, toHash, reqId, reason) {
|
|
3667
3479
|
const redirects = await this.pickJoinRejectRedirects(ch, toHash, JOIN_REJECT_REDIRECT_MAX);
|
|
3668
|
-
await this._sendControl(toHash, encodeJoinReject(ch.id.key, reqId, reason, redirects));
|
|
3480
|
+
await this._sendControl(toHash, this.codec.encodeJoinReject(ch.id.key, reqId, reason, redirects));
|
|
3669
3481
|
}
|
|
3670
3482
|
async sendTrackerFeedback(ch, trackerPeers, candidateHash, event, reason = 0) {
|
|
3671
3483
|
if (trackerPeers.length === 0)
|
|
3672
3484
|
return;
|
|
3673
|
-
await this._sendControlMany(trackerPeers, encodeTrackerFeedback(ch.id.key, candidateHash, event, reason));
|
|
3485
|
+
await this._sendControlMany(trackerPeers, this.codec.encodeTrackerFeedback(ch.id.key, candidateHash, event, reason));
|
|
3486
|
+
}
|
|
3487
|
+
pruneParentUpgradeReservations(ch, now = Date.now()) {
|
|
3488
|
+
for (const [hash, reservation] of ch.parentUpgradeReservationsByHash) {
|
|
3489
|
+
if (reservation.expiresAt <= now) {
|
|
3490
|
+
ch.parentUpgradeReservationsByHash.delete(hash);
|
|
3491
|
+
ch.metrics.parentUpgradeRootReservationExpired += 1;
|
|
3492
|
+
continue;
|
|
3493
|
+
}
|
|
3494
|
+
if (ch.children.has(hash)) {
|
|
3495
|
+
ch.parentUpgradeReservationsByHash.delete(hash);
|
|
3496
|
+
}
|
|
3497
|
+
}
|
|
3498
|
+
}
|
|
3499
|
+
parentUpgradeReservationCount(ch, excludeHash, now = Date.now()) {
|
|
3500
|
+
this.pruneParentUpgradeReservations(ch, now);
|
|
3501
|
+
let count = 0;
|
|
3502
|
+
for (const hash of ch.parentUpgradeReservationsByHash.keys()) {
|
|
3503
|
+
if (hash === excludeHash)
|
|
3504
|
+
continue;
|
|
3505
|
+
if (ch.children.has(hash))
|
|
3506
|
+
continue;
|
|
3507
|
+
count += 1;
|
|
3508
|
+
}
|
|
3509
|
+
return count;
|
|
3510
|
+
}
|
|
3511
|
+
createParentUpgradeReservation(ch, toHash, minFreeSlots = 0, now = Date.now()) {
|
|
3512
|
+
if (!ch.isRoot || ch.children.has(toHash))
|
|
3513
|
+
return 0;
|
|
3514
|
+
const maxChildren = Math.max(0, Math.floor(ch.effectiveMaxChildren));
|
|
3515
|
+
if (maxChildren <= 0)
|
|
3516
|
+
return 0;
|
|
3517
|
+
const reserved = this.parentUpgradeReservationCount(ch, toHash, now);
|
|
3518
|
+
const requestedMinFreeSlots = Math.max(0, Math.floor(minFreeSlots));
|
|
3519
|
+
const freeSlots = maxChildren - ch.children.size - reserved;
|
|
3520
|
+
if (freeSlots <= 0 ||
|
|
3521
|
+
(requestedMinFreeSlots > 0 && freeSlots < requestedMinFreeSlots)) {
|
|
3522
|
+
ch.metrics.parentUpgradeRootReservationBlocked += 1;
|
|
3523
|
+
return 0;
|
|
3524
|
+
}
|
|
3525
|
+
const token = (this.random() * 0xffffffff) >>> 0 || 1;
|
|
3526
|
+
ch.parentUpgradeReservationsByHash.set(toHash, {
|
|
3527
|
+
token,
|
|
3528
|
+
expiresAt: now + PARENT_UPGRADE_ROOT_RESERVATION_TTL_MS,
|
|
3529
|
+
minFreeSlots: requestedMinFreeSlots,
|
|
3530
|
+
});
|
|
3531
|
+
ch.metrics.parentUpgradeRootReservationCreated += 1;
|
|
3532
|
+
return token;
|
|
3533
|
+
}
|
|
3534
|
+
consumeParentUpgradeReservation(ch, fromHash, token, now = Date.now()) {
|
|
3535
|
+
if (!ch.isRoot || token <= 0)
|
|
3536
|
+
return false;
|
|
3537
|
+
this.pruneParentUpgradeReservations(ch, now);
|
|
3538
|
+
const reservation = ch.parentUpgradeReservationsByHash.get(fromHash);
|
|
3539
|
+
if (!reservation)
|
|
3540
|
+
return false;
|
|
3541
|
+
if (reservation.expiresAt <= now) {
|
|
3542
|
+
ch.parentUpgradeReservationsByHash.delete(fromHash);
|
|
3543
|
+
ch.metrics.parentUpgradeRootReservationExpired += 1;
|
|
3544
|
+
return false;
|
|
3545
|
+
}
|
|
3546
|
+
if (reservation.token !== token)
|
|
3547
|
+
return false;
|
|
3548
|
+
const requestedMinFreeSlots = Math.max(0, Math.floor(reservation.minFreeSlots ?? 0));
|
|
3549
|
+
if (requestedMinFreeSlots > 0) {
|
|
3550
|
+
const maxChildren = Math.max(0, Math.floor(ch.effectiveMaxChildren));
|
|
3551
|
+
const freeSlots = Math.max(0, maxChildren - ch.children.size);
|
|
3552
|
+
if (freeSlots < requestedMinFreeSlots) {
|
|
3553
|
+
ch.parentUpgradeReservationsByHash.delete(fromHash);
|
|
3554
|
+
ch.metrics.parentUpgradeRootReservationMarginRejected += 1;
|
|
3555
|
+
return false;
|
|
3556
|
+
}
|
|
3557
|
+
}
|
|
3558
|
+
ch.parentUpgradeReservationsByHash.delete(fromHash);
|
|
3559
|
+
ch.metrics.parentUpgradeRootReservationConsumed += 1;
|
|
3560
|
+
return true;
|
|
3561
|
+
}
|
|
3562
|
+
isRootedForParentProbe(ch) {
|
|
3563
|
+
if (ch.isRoot)
|
|
3564
|
+
return true;
|
|
3565
|
+
const route = ch.routeFromRoot;
|
|
3566
|
+
return Boolean(ch.parent &&
|
|
3567
|
+
Array.isArray(route) &&
|
|
3568
|
+
route.length >= 2 &&
|
|
3569
|
+
route[0] === ch.id.root &&
|
|
3570
|
+
route[route.length - 1] === this.publicKeyHash);
|
|
3571
|
+
}
|
|
3572
|
+
encodeParentProbeReplyForChannel(ch, reqId, toHash, minFreeSlots = 0, reserveRootCapacity = true) {
|
|
3573
|
+
this.pruneDisconnectedChildren(ch);
|
|
3574
|
+
const rooted = this.isRootedForParentProbe(ch);
|
|
3575
|
+
const requestedMinFreeSlots = Math.max(0, Math.floor(minFreeSlots));
|
|
3576
|
+
const maxChildren = Math.max(0, Math.floor(ch.effectiveMaxChildren));
|
|
3577
|
+
const reserved = ch.isRoot
|
|
3578
|
+
? this.parentUpgradeReservationCount(ch, toHash)
|
|
3579
|
+
: 0;
|
|
3580
|
+
const freeSlots = rooted
|
|
3581
|
+
? Math.max(0, maxChildren - ch.children.size - reserved)
|
|
3582
|
+
: 0;
|
|
3583
|
+
const requesterHasEnoughSlots = freeSlots > 0 &&
|
|
3584
|
+
(requestedMinFreeSlots <= 0 || freeSlots >= requestedMinFreeSlots);
|
|
3585
|
+
if (ch.isRoot &&
|
|
3586
|
+
rooted &&
|
|
3587
|
+
maxChildren > 0 &&
|
|
3588
|
+
!ch.children.has(toHash) &&
|
|
3589
|
+
!requesterHasEnoughSlots) {
|
|
3590
|
+
ch.metrics.parentUpgradeRootReservationBlocked += 1;
|
|
3591
|
+
}
|
|
3592
|
+
const reservationToken = ch.isRoot && rooted && requesterHasEnoughSlots && reserveRootCapacity
|
|
3593
|
+
? this.createParentUpgradeReservation(ch, toHash, requestedMinFreeSlots)
|
|
3594
|
+
: 0;
|
|
3595
|
+
const repairing = ch.missingSeqs.size > 0;
|
|
3596
|
+
const overloaded = ch.overloadStreak > 0;
|
|
3597
|
+
let flags = 0;
|
|
3598
|
+
if (rooted)
|
|
3599
|
+
flags |= PARENT_PROBE_FLAG_ROOTED;
|
|
3600
|
+
if (rooted &&
|
|
3601
|
+
(reservationToken > 0 ||
|
|
3602
|
+
(!ch.isRoot && requesterHasEnoughSlots) ||
|
|
3603
|
+
(ch.isRoot && requesterHasEnoughSlots && !reserveRootCapacity))) {
|
|
3604
|
+
flags |= PARENT_PROBE_FLAG_ACCEPTING;
|
|
3605
|
+
}
|
|
3606
|
+
if (repairing)
|
|
3607
|
+
flags |= PARENT_PROBE_FLAG_REPAIRING;
|
|
3608
|
+
if (overloaded)
|
|
3609
|
+
flags |= PARENT_PROBE_FLAG_OVERLOADED;
|
|
3610
|
+
return this.codec.encodeParentProbeReply(ch.id.key, reqId, {
|
|
3611
|
+
flags,
|
|
3612
|
+
level: Number.isFinite(ch.level) ? ch.level : 0xffff,
|
|
3613
|
+
maxChildren,
|
|
3614
|
+
freeSlots,
|
|
3615
|
+
// For roots, expose reserved slots as child pressure so concurrent
|
|
3616
|
+
// upgrade probes cannot all pass the load-ratio check from stale state.
|
|
3617
|
+
children: ch.isRoot ? ch.children.size + reserved : ch.children.size,
|
|
3618
|
+
haveToExclusive: ch.maxSeqSeen >= 0 ? ch.maxSeqSeen + 1 : 0,
|
|
3619
|
+
missingSeqs: ch.missingSeqs.size,
|
|
3620
|
+
dataWriteDrops: ch.metrics.dataWriteDrops,
|
|
3621
|
+
droppedForwards: ch.droppedForwards,
|
|
3622
|
+
reservationToken,
|
|
3623
|
+
});
|
|
3624
|
+
}
|
|
3625
|
+
async probeParentCandidate(ch, parentHash, timeoutMs, signal, minFreeSlots = 0, reserveRootCapacity = true) {
|
|
3626
|
+
if (!this.peers.get(parentHash))
|
|
3627
|
+
return;
|
|
3628
|
+
const reqId = (this.random() * 0xffffffff) >>> 0;
|
|
3629
|
+
const p = new Promise((resolve) => {
|
|
3630
|
+
ch.pendingParentProbe.set(reqId, { resolve });
|
|
3631
|
+
});
|
|
3632
|
+
await this._sendControl(parentHash, this.codec.encodeParentProbeReq(ch.id.key, reqId, minFreeSlots, reserveRootCapacity));
|
|
3633
|
+
const res = await Promise.race([
|
|
3634
|
+
p,
|
|
3635
|
+
delay(Math.max(1, timeoutMs), { signal }).then(() => undefined),
|
|
3636
|
+
]);
|
|
3637
|
+
if (!res)
|
|
3638
|
+
ch.pendingParentProbe.delete(reqId);
|
|
3639
|
+
return res;
|
|
3674
3640
|
}
|
|
3675
3641
|
async _joinLoop(ch, joinOpts) {
|
|
3676
3642
|
const retryMs = Math.max(1, Math.floor(joinOpts.retryMs ?? 200));
|
|
@@ -3686,6 +3652,9 @@ export class FanoutTree extends DirectStream {
|
|
|
3686
3652
|
const trackerQueryIntervalMs = Math.max(0, Math.floor(joinOpts.trackerQueryIntervalMs ?? ch.trackerQueryIntervalMs));
|
|
3687
3653
|
const joinAttemptsPerRound = Math.max(1, Math.floor(joinOpts.joinAttemptsPerRound ?? 8));
|
|
3688
3654
|
const candidateCooldownMs = Math.max(0, Math.floor(joinOpts.candidateCooldownMs ?? 2_000));
|
|
3655
|
+
const parentUpgrade = this.nativeFanout
|
|
3656
|
+
? this.nativeFanout.normalizeParentUpgradePolicy(joinOpts)
|
|
3657
|
+
: normalizeParentUpgradePolicy(joinOpts);
|
|
3689
3658
|
const candidateScoringModeRaw = joinOpts.candidateScoringMode ?? "ranked-shuffle";
|
|
3690
3659
|
const candidateScoringMode = candidateScoringModeRaw === "ranked-strict" ||
|
|
3691
3660
|
candidateScoringModeRaw === "weighted" ||
|
|
@@ -3705,6 +3674,56 @@ export class FanoutTree extends DirectStream {
|
|
|
3705
3674
|
? anySignal([this.closeController.signal, joinOpts.signal])
|
|
3706
3675
|
: this.closeController.signal;
|
|
3707
3676
|
const signal = combinedSignal;
|
|
3677
|
+
let nextParentUpgradeCheckAt = 0;
|
|
3678
|
+
let parentUpgradeCheckSeq = 0;
|
|
3679
|
+
let parentUpgradeActiveGuardBackoffMs = 0;
|
|
3680
|
+
const scheduleNextParentUpgradeCheck = (now, first = false, minDelayMs = parentUpgrade.intervalMs) => {
|
|
3681
|
+
if (parentUpgrade.intervalMs <= 0) {
|
|
3682
|
+
nextParentUpgradeCheckAt = 0;
|
|
3683
|
+
return;
|
|
3684
|
+
}
|
|
3685
|
+
const factor = first
|
|
3686
|
+
? stableUnitInterval(`${ch.id.suffixKey}:${this.publicKeyHash}:parent-upgrade-phase`)
|
|
3687
|
+
: 0.75 +
|
|
3688
|
+
stableUnitInterval(`${ch.id.suffixKey}:${this.publicKeyHash}:parent-upgrade-interval:${parentUpgradeCheckSeq++}`) *
|
|
3689
|
+
0.5;
|
|
3690
|
+
nextParentUpgradeCheckAt =
|
|
3691
|
+
now +
|
|
3692
|
+
Math.max(1, Math.floor(parentUpgrade.intervalMs * factor), Math.floor(minDelayMs));
|
|
3693
|
+
};
|
|
3694
|
+
const resetParentUpgradeActiveGuardBackoff = () => {
|
|
3695
|
+
parentUpgradeActiveGuardBackoffMs = 0;
|
|
3696
|
+
};
|
|
3697
|
+
const parentUpgradeGuardDelayMs = (reason, now) => {
|
|
3698
|
+
const baseDelayMs = (() => {
|
|
3699
|
+
switch (reason) {
|
|
3700
|
+
case "data":
|
|
3701
|
+
return Math.max(parentUpgrade.quietMs, parentUpgrade.repairQuietMs);
|
|
3702
|
+
case "repair":
|
|
3703
|
+
return parentUpgrade.repairQuietMs;
|
|
3704
|
+
case "quiet":
|
|
3705
|
+
return ch.lastParentUpgradeActivityAt > 0
|
|
3706
|
+
? Math.max(0, parentUpgrade.quietMs -
|
|
3707
|
+
(now - ch.lastParentUpgradeActivityAt))
|
|
3708
|
+
: parentUpgrade.quietMs;
|
|
3709
|
+
case "cooldown":
|
|
3710
|
+
return Math.max(0, ch.parentUpgradeBackoffUntil - now, ch.parentUpgradeLastAt > 0
|
|
3711
|
+
? parentUpgrade.cooldownMs - (now - ch.parentUpgradeLastAt)
|
|
3712
|
+
: 0);
|
|
3713
|
+
default:
|
|
3714
|
+
return parentUpgrade.intervalMs;
|
|
3715
|
+
}
|
|
3716
|
+
})();
|
|
3717
|
+
if (reason !== "data" && reason !== "repair") {
|
|
3718
|
+
resetParentUpgradeActiveGuardBackoff();
|
|
3719
|
+
return baseDelayMs;
|
|
3720
|
+
}
|
|
3721
|
+
parentUpgradeActiveGuardBackoffMs =
|
|
3722
|
+
parentUpgradeActiveGuardBackoffMs > 0
|
|
3723
|
+
? Math.min(Math.max(parentUpgrade.failedBackoff.maxMs, baseDelayMs), Math.max(baseDelayMs, parentUpgradeActiveGuardBackoffMs * 2))
|
|
3724
|
+
: baseDelayMs;
|
|
3725
|
+
return parentUpgradeActiveGuardBackoffMs;
|
|
3726
|
+
};
|
|
3708
3727
|
try {
|
|
3709
3728
|
for (;;) {
|
|
3710
3729
|
if (ch.closed) {
|
|
@@ -3730,6 +3749,8 @@ export class FanoutTree extends DirectStream {
|
|
|
3730
3749
|
ch.metrics.reparentDisconnect += 1;
|
|
3731
3750
|
const hadChildren = ch.children.size > 0;
|
|
3732
3751
|
this.detachFromParent(ch);
|
|
3752
|
+
nextParentUpgradeCheckAt = 0;
|
|
3753
|
+
resetParentUpgradeActiveGuardBackoff();
|
|
3733
3754
|
// If we lose our parent, we are no longer on the rooted tree; detach children so
|
|
3734
3755
|
// they can rejoin as well (prevents stable disconnected components).
|
|
3735
3756
|
void this.kickChildren(ch).catch(() => { });
|
|
@@ -3742,43 +3763,123 @@ export class FanoutTree extends DirectStream {
|
|
|
3742
3763
|
const endedAndComplete = ch.endSeqExclusive > 0 &&
|
|
3743
3764
|
ch.missingSeqs.size === 0 &&
|
|
3744
3765
|
ch.nextExpectedSeq >= ch.endSeqExclusive;
|
|
3745
|
-
const expectingData = ch.missingSeqs.size > 0 ||
|
|
3746
|
-
|
|
3747
|
-
ch.
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3766
|
+
const expectingData = ch.missingSeqs.size > 0 ||
|
|
3767
|
+
(ch.repairEnabled && ch.maxSeqSeen >= 0 && ch.endSeqExclusive < 0) ||
|
|
3768
|
+
(ch.maxDataAgeMs > 0 && !endedAndComplete);
|
|
3769
|
+
// Repair requests travel child->parent on a direction that works
|
|
3770
|
+
// even when parent->child data writes silently fail. A repair that
|
|
3771
|
+
// does not fill a gap is not sufficient evidence by itself though:
|
|
3772
|
+
// the repair payload can be dropped or the parent cache can miss.
|
|
3773
|
+
// Re-parent only after both repeated unanswered repairs and a
|
|
3774
|
+
// quiet parent liveness window.
|
|
3775
|
+
const parentRepairDeadLivenessMs = Math.max(PARENT_REPAIR_DEAD_MIN_LIVENESS_MS, ch.repairIntervalMs * PARENT_REPAIR_DEAD_STREAK_THRESHOLD, ch.neighborAnnounceIntervalMs * 8);
|
|
3776
|
+
const parentRepairNow = Date.now();
|
|
3777
|
+
const parentLivenessQuietMs = ch.lastParentDataAt > 0 ? parentRepairNow - ch.lastParentDataAt : 0;
|
|
3778
|
+
const parentRepairDead = ch.repairEnabled &&
|
|
3779
|
+
ch.parentRepairUnansweredStreak >=
|
|
3780
|
+
PARENT_REPAIR_DEAD_STREAK_THRESHOLD &&
|
|
3781
|
+
parentLivenessQuietMs >= parentRepairDeadLivenessMs;
|
|
3782
|
+
if (parentRepairDead ||
|
|
3783
|
+
(staleAfterMs > 0 &&
|
|
3784
|
+
ch.receivedAnyParentData &&
|
|
3785
|
+
ch.lastParentDataAt > 0 &&
|
|
3786
|
+
Date.now() - ch.lastParentDataAt > staleAfterMs &&
|
|
3787
|
+
expectingData)) {
|
|
3751
3788
|
// Parent is "alive" at the stream layer but we're not receiving
|
|
3752
3789
|
// data at the expected rate; detach and try to re-parent.
|
|
3753
3790
|
ch.metrics.reparentStale += 1;
|
|
3754
3791
|
ch.parent = undefined;
|
|
3792
|
+
ch.parentRepairUnansweredStreak = 0;
|
|
3755
3793
|
ch.level = Number.POSITIVE_INFINITY;
|
|
3756
3794
|
ch.routeFromRoot = undefined;
|
|
3757
3795
|
ch.routeByPeer.clear();
|
|
3758
3796
|
ch.lastParentDataAt = 0;
|
|
3797
|
+
ch.lastParentUpgradeActivityAt = 0;
|
|
3759
3798
|
ch.receivedAnyParentData = false;
|
|
3760
3799
|
ch.pendingJoin.clear();
|
|
3761
|
-
ch.
|
|
3762
|
-
|
|
3763
|
-
|
|
3800
|
+
ch.pendingParentProbe.clear();
|
|
3801
|
+
ch.parentShadow = undefined;
|
|
3802
|
+
void Promise.all(this.clearParentUpgradeGrace(ch, true, true)).catch(() => { });
|
|
3803
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
3804
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
3764
3805
|
}
|
|
3765
|
-
ch.
|
|
3806
|
+
ch.parentUpgradeRetryAfterSeq = -1;
|
|
3807
|
+
ch.parentProbeRejectUntilByHash.clear();
|
|
3808
|
+
ch.parentProbeRejectBackoffMsByHash.clear();
|
|
3809
|
+
nextParentUpgradeCheckAt = 0;
|
|
3810
|
+
resetParentUpgradeActiveGuardBackoff();
|
|
3811
|
+
ch.pendingRouteQuery.clear();
|
|
3812
|
+
this.clearRouteProxies(ch);
|
|
3766
3813
|
void this.kickChildren(ch).catch(() => { });
|
|
3767
|
-
await delay(retryMs);
|
|
3814
|
+
await delay(retryMs, { signal });
|
|
3768
3815
|
continue;
|
|
3769
3816
|
}
|
|
3817
|
+
if (parentUpgrade.intervalMs > 0 && ch.level > 1) {
|
|
3818
|
+
const now = Date.now();
|
|
3819
|
+
if (nextParentUpgradeCheckAt === 0) {
|
|
3820
|
+
scheduleNextParentUpgradeCheck(now, true, parentUpgrade.dataGuard || parentUpgrade.repairGuard
|
|
3821
|
+
? Math.max(parentUpgrade.quietMs, parentUpgrade.repairQuietMs)
|
|
3822
|
+
: parentUpgrade.intervalMs);
|
|
3823
|
+
}
|
|
3824
|
+
const due = now >= nextParentUpgradeCheckAt;
|
|
3825
|
+
if (due) {
|
|
3826
|
+
const gateOptions = {
|
|
3827
|
+
leafOnly: parentUpgrade.leafOnly,
|
|
3828
|
+
repairGuard: parentUpgrade.repairGuard,
|
|
3829
|
+
dataGuard: parentUpgrade.dataGuard,
|
|
3830
|
+
endedAndComplete,
|
|
3831
|
+
maxPerPeer: parentUpgrade.maxPerPeer,
|
|
3832
|
+
cooldownMs: parentUpgrade.cooldownMs,
|
|
3833
|
+
quietMs: parentUpgrade.quietMs,
|
|
3834
|
+
repairQuietMs: parentUpgrade.repairQuietMs,
|
|
3835
|
+
now,
|
|
3836
|
+
};
|
|
3837
|
+
const gate = this.nativeFanout
|
|
3838
|
+
? this.nativeFanout.evaluateParentUpgradeGate(ch, gateOptions)
|
|
3839
|
+
: evaluateParentUpgradeGate(ch, gateOptions);
|
|
3840
|
+
if ("reason" in gate) {
|
|
3841
|
+
recordParentUpgradeSkip(ch.metrics, gate.reason);
|
|
3842
|
+
scheduleNextParentUpgradeCheck(now, false, parentUpgradeGuardDelayMs(gate.reason, now));
|
|
3843
|
+
}
|
|
3844
|
+
else {
|
|
3845
|
+
resetParentUpgradeActiveGuardBackoff();
|
|
3846
|
+
const improved = await this.maybeImproveParent(ch, {
|
|
3847
|
+
signal,
|
|
3848
|
+
candidateShuffleTopK,
|
|
3849
|
+
candidateScoringMode,
|
|
3850
|
+
candidateScoringWeights,
|
|
3851
|
+
joinAttemptsPerRound,
|
|
3852
|
+
joinReqTimeoutMs,
|
|
3853
|
+
parentUpgrade,
|
|
3854
|
+
trackerPeers: ch.cachedBootstrapPeers.filter((h) => Boolean(this.peers.get(h))),
|
|
3855
|
+
});
|
|
3856
|
+
scheduleNextParentUpgradeCheck(Date.now());
|
|
3857
|
+
if (improved) {
|
|
3858
|
+
ch.metrics.reparentUpgrade += 1;
|
|
3859
|
+
ch.parentUpgradeCount += 1;
|
|
3860
|
+
ch.parentUpgradeLastAt = Date.now();
|
|
3861
|
+
ch.parentUpgradeBackoffMs = 0;
|
|
3862
|
+
ch.parentUpgradeBackoffUntil = 0;
|
|
3863
|
+
ch.parentUpgradeRetryAfterSeq = -1;
|
|
3864
|
+
ch.parentUpgradeStaleRootProbeRound = 0;
|
|
3865
|
+
}
|
|
3866
|
+
}
|
|
3867
|
+
}
|
|
3868
|
+
}
|
|
3770
3869
|
if (!ch.joinedOnce)
|
|
3771
3870
|
ch.joinedOnce = createDeferred();
|
|
3772
3871
|
ch.joinedOnce.resolve();
|
|
3773
3872
|
ch.joinedAtLeastOnce = true;
|
|
3774
3873
|
// Once attached, we don't need a fast retry cadence; keep polling coarse to
|
|
3775
3874
|
// avoid excessive timers when simulating many nodes in one process.
|
|
3776
|
-
await delay(Math.max(retryMs, 1_000));
|
|
3875
|
+
await delay(Math.max(retryMs, 1_000), { signal });
|
|
3777
3876
|
continue;
|
|
3778
3877
|
}
|
|
3779
3878
|
// `timeoutMs` is meant to bound the initial `joinChannel()` await, not to
|
|
3780
3879
|
// stop re-parenting attempts for long-running nodes.
|
|
3781
|
-
if (!ch.joinedAtLeastOnce &&
|
|
3880
|
+
if (!ch.joinedAtLeastOnce &&
|
|
3881
|
+
timeoutMs > 0 &&
|
|
3882
|
+
Date.now() - start > timeoutMs) {
|
|
3782
3883
|
const bootstrapsCount = this.getBootstrapsForChannel(ch).length;
|
|
3783
3884
|
const rootPeer = this.peers.get(ch.id.root);
|
|
3784
3885
|
const rootNeighbor = Boolean(rootPeer && rootPeer.isReadable && rootPeer.isWritable);
|
|
@@ -3881,12 +3982,19 @@ export class FanoutTree extends DirectStream {
|
|
|
3881
3982
|
source: 1,
|
|
3882
3983
|
});
|
|
3883
3984
|
}
|
|
3884
|
-
// Fallback: try a bounded set of already-connected peers.
|
|
3985
|
+
// Fallback: try a bounded set of already-connected peers. Bootstrap
|
|
3986
|
+
// peers are often just trackers, but they can also be the only
|
|
3987
|
+
// connected channel host and may not list themselves in tracker
|
|
3988
|
+
// replies. Keep them as low-priority candidates unless tracker state
|
|
3989
|
+
// already represented them above.
|
|
3990
|
+
const bootstrapPeerSet = new Set(bootstrapPeers);
|
|
3885
3991
|
let connectedFallbackAdded = 0;
|
|
3886
3992
|
const connectedFallbackMax = 64;
|
|
3887
3993
|
for (const h of this.peers.keys()) {
|
|
3888
3994
|
if (h === this.publicKeyHash)
|
|
3889
3995
|
continue;
|
|
3996
|
+
if (bootstrapPeerSet.has(h) && candidatesByHash.has(h))
|
|
3997
|
+
continue;
|
|
3890
3998
|
upsertCandidate({
|
|
3891
3999
|
hash: h,
|
|
3892
4000
|
addrs: [],
|
|
@@ -3926,12 +4034,14 @@ export class FanoutTree extends DirectStream {
|
|
|
3926
4034
|
if (until != null && until > now)
|
|
3927
4035
|
nextAt = Math.min(nextAt, until);
|
|
3928
4036
|
}
|
|
3929
|
-
const waitMs = nextAt !== Number.POSITIVE_INFINITY
|
|
4037
|
+
const waitMs = nextAt !== Number.POSITIVE_INFINITY
|
|
4038
|
+
? Math.max(1, nextAt - now)
|
|
4039
|
+
: retryMs;
|
|
3930
4040
|
const capMs = Math.max(retryMs, trackerQueryIntervalMs > 0 ? trackerQueryIntervalMs : retryMs);
|
|
3931
|
-
await delay(Math.max(1, Math.min(waitMs, capMs)));
|
|
4041
|
+
await delay(Math.max(1, Math.min(waitMs, capMs)), { signal });
|
|
3932
4042
|
continue;
|
|
3933
4043
|
}
|
|
3934
|
-
await delay(retryMs);
|
|
4044
|
+
await delay(retryMs, { signal });
|
|
3935
4045
|
continue;
|
|
3936
4046
|
}
|
|
3937
4047
|
let ordered = [...candidates];
|
|
@@ -4137,26 +4247,922 @@ export class FanoutTree extends DirectStream {
|
|
|
4137
4247
|
const stream = this.peers.get(h);
|
|
4138
4248
|
if (!stream)
|
|
4139
4249
|
continue;
|
|
4140
|
-
void this.components.connectionManager
|
|
4250
|
+
void this.components.connectionManager
|
|
4251
|
+
.closeConnections(stream.peerId)
|
|
4252
|
+
.catch(() => { });
|
|
4141
4253
|
}
|
|
4142
4254
|
continue;
|
|
4143
4255
|
}
|
|
4144
|
-
await delay(retryMs);
|
|
4256
|
+
await delay(retryMs, { signal });
|
|
4145
4257
|
}
|
|
4146
4258
|
}
|
|
4147
4259
|
finally {
|
|
4148
4260
|
signal.clear?.();
|
|
4149
4261
|
}
|
|
4150
4262
|
}
|
|
4151
|
-
async
|
|
4152
|
-
|
|
4263
|
+
async maybeImproveParent(ch, options) {
|
|
4264
|
+
const parentUpgrade = options.parentUpgrade;
|
|
4265
|
+
const currentParent = ch.parent;
|
|
4266
|
+
if (!currentParent || ch.closed || ch.isRoot || ch.level <= 1)
|
|
4267
|
+
return false;
|
|
4268
|
+
if (parentUpgrade.mode === "shadow" &&
|
|
4269
|
+
this.parentUpgradeShadowInFlightSuffixKey != null &&
|
|
4270
|
+
this.parentUpgradeShadowInFlightSuffixKey !== ch.id.suffixKey) {
|
|
4271
|
+
ch.metrics.reparentUpgradeSkipBudget += 1;
|
|
4272
|
+
return false;
|
|
4273
|
+
}
|
|
4274
|
+
const staleRootProbeRound = Math.max(0, Math.floor(ch.parentUpgradeStaleRootProbeRound || 0));
|
|
4275
|
+
ch.parentUpgradeStaleRootProbeRound = Math.min(0x7fffffff, staleRootProbeRound + 1);
|
|
4276
|
+
const upgradeRandom = (scope) => stableUnitInterval(`${ch.id.suffixKey}:${this.publicKeyHash}:${currentParent}:${scope}`);
|
|
4277
|
+
const clearFailedBackoff = () => {
|
|
4278
|
+
ch.parentUpgradeBackoffMs = 0;
|
|
4279
|
+
ch.parentUpgradeBackoffUntil = 0;
|
|
4280
|
+
};
|
|
4281
|
+
const clearShadowInFlight = () => {
|
|
4282
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
4283
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
4284
|
+
}
|
|
4285
|
+
};
|
|
4286
|
+
const resetShadow = () => {
|
|
4287
|
+
if (!ch.parentShadow)
|
|
4288
|
+
return;
|
|
4289
|
+
ch.parentShadow = undefined;
|
|
4290
|
+
clearShadowInFlight();
|
|
4291
|
+
ch.metrics.parentShadowReset += 1;
|
|
4292
|
+
};
|
|
4293
|
+
const deferParentUpgradeRetryUntilFreshData = (multiplier = 1) => {
|
|
4294
|
+
const minFreshMessages = Math.max(1, parentUpgrade.shadow.dualPathMinMessages *
|
|
4295
|
+
Math.max(1, Math.floor(multiplier)));
|
|
4296
|
+
ch.parentUpgradeRetryAfterSeq = Math.max(ch.parentUpgradeRetryAfterSeq, ch.maxSeqSeen + minFreshMessages - 1);
|
|
4297
|
+
};
|
|
4298
|
+
const applyFailedBackoff = () => {
|
|
4299
|
+
const minMs = parentUpgrade.failedBackoff.minMs;
|
|
4300
|
+
const maxMs = parentUpgrade.failedBackoff.maxMs;
|
|
4301
|
+
if (minMs <= 0 || maxMs <= 0) {
|
|
4302
|
+
ch.parentUpgradeBackoffMs = 0;
|
|
4303
|
+
ch.parentUpgradeBackoffUntil = 0;
|
|
4304
|
+
return;
|
|
4305
|
+
}
|
|
4306
|
+
const nextMs = ch.parentUpgradeBackoffMs > 0
|
|
4307
|
+
? Math.min(maxMs, Math.max(minMs, ch.parentUpgradeBackoffMs * 2))
|
|
4308
|
+
: minMs;
|
|
4309
|
+
ch.parentUpgradeBackoffMs = nextMs;
|
|
4310
|
+
const jitter = 0.75 + upgradeRandom(`failed-backoff:${nextMs}`) * 0.5;
|
|
4311
|
+
ch.parentUpgradeBackoffUntil =
|
|
4312
|
+
Date.now() + Math.max(1, Math.floor(nextMs * jitter));
|
|
4313
|
+
};
|
|
4314
|
+
const isDirectNeighbor = (hash) => {
|
|
4315
|
+
const peer = this.peers.get(hash);
|
|
4316
|
+
if (!peer)
|
|
4317
|
+
return false;
|
|
4318
|
+
try {
|
|
4319
|
+
const conns = this.components.connectionManager.getConnections(peer.peerId);
|
|
4320
|
+
return (conns?.length ?? 0) > 0;
|
|
4321
|
+
}
|
|
4322
|
+
catch {
|
|
4323
|
+
return peer.isReadable || peer.isWritable;
|
|
4324
|
+
}
|
|
4325
|
+
};
|
|
4326
|
+
const candidatesByHash = new Map();
|
|
4327
|
+
const upsertCandidate = (c) => {
|
|
4328
|
+
const prev = candidatesByHash.get(c.hash);
|
|
4329
|
+
if (!prev) {
|
|
4330
|
+
candidatesByHash.set(c.hash, { ...c });
|
|
4331
|
+
return;
|
|
4332
|
+
}
|
|
4333
|
+
if (prev.addrs.length === 0 && c.addrs.length > 0)
|
|
4334
|
+
prev.addrs = c.addrs;
|
|
4335
|
+
prev.level = Math.min(prev.level, c.level);
|
|
4336
|
+
prev.freeSlots = Math.max(prev.freeSlots, c.freeSlots);
|
|
4337
|
+
prev.bidPerByte = Math.max(prev.bidPerByte, c.bidPerByte);
|
|
4338
|
+
prev.source = Math.min(prev.source, c.source);
|
|
4339
|
+
};
|
|
4340
|
+
const nonRootMinLevelGain = Math.max(parentUpgrade.minLevelGain, parentUpgrade.nonRootMinLevelGain);
|
|
4341
|
+
const rootMinLevelGain = Math.max(parentUpgrade.minLevelGain, parentUpgrade.rootMinLevelGain);
|
|
4342
|
+
const rootMinSubtreeGain = Math.max(parentUpgrade.minLevelGain, parentUpgrade.rootMinSubtreeGain);
|
|
4343
|
+
const levelGainFor = (parentLevel) => ch.level - (parentLevel + 1);
|
|
4344
|
+
const impactedPeersForRoot = () => Math.max(1, 1 + ch.children.size);
|
|
4345
|
+
const rootSubtreeGainFor = (parentLevel) => levelGainFor(parentLevel) * impactedPeersForRoot();
|
|
4346
|
+
const isEnoughLevelGain = (hash, parentLevel) => {
|
|
4347
|
+
const levelGain = levelGainFor(parentLevel);
|
|
4348
|
+
if (hash !== ch.id.root)
|
|
4349
|
+
return levelGain >= nonRootMinLevelGain;
|
|
4350
|
+
if (levelGain >= rootMinLevelGain)
|
|
4351
|
+
return true;
|
|
4352
|
+
return rootSubtreeGainFor(parentLevel) >= rootMinSubtreeGain;
|
|
4353
|
+
};
|
|
4354
|
+
let localChannelCount = 1;
|
|
4355
|
+
if (this.channelsBySuffixKey instanceof Map) {
|
|
4356
|
+
localChannelCount = 0;
|
|
4357
|
+
for (const localCh of this.channelsBySuffixKey.values()) {
|
|
4358
|
+
if (localCh.closed)
|
|
4359
|
+
continue;
|
|
4360
|
+
localChannelCount += 1;
|
|
4361
|
+
if (localChannelCount > 1)
|
|
4362
|
+
break;
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
const singleChannelBranch = ch.children.size > 0 && localChannelCount <= 1;
|
|
4366
|
+
const multiChannelBranch = ch.children.size > 0 && localChannelCount > 1;
|
|
4367
|
+
const endedAndComplete = ch.endSeqExclusive > 0 &&
|
|
4368
|
+
ch.missingSeqs.size === 0 &&
|
|
4369
|
+
ch.nextExpectedSeq >= ch.endSeqExclusive;
|
|
4370
|
+
const singleChannelSettledLeaf = ch.children.size === 0 && localChannelCount <= 1 && endedAndComplete;
|
|
4371
|
+
const multiChannelSettledLeaf = ch.children.size === 0 && localChannelCount > 1 && endedAndComplete;
|
|
4372
|
+
const knownChannelPeerCount = Math.max(1, ch.channelPeers?.size ?? 1);
|
|
4373
|
+
const defaultStaleRootProbeProbability = 0.015625;
|
|
4374
|
+
const configuredStaleRootProbeProbabilityRaw = Number(parentUpgrade.staleRootProbeProbability);
|
|
4375
|
+
const staleRootProbeBaseProbability = Number.isFinite(configuredStaleRootProbeProbabilityRaw)
|
|
4376
|
+
? Math.max(0, Math.min(1, configuredStaleRootProbeProbabilityRaw))
|
|
4377
|
+
: defaultStaleRootProbeProbability;
|
|
4378
|
+
const defaultMultiChannelSettledLeafRootCandidate = parentUpgrade.leafOnly &&
|
|
4379
|
+
multiChannelSettledLeaf &&
|
|
4380
|
+
staleRootProbeBaseProbability <= defaultStaleRootProbeProbability;
|
|
4381
|
+
const hasDefaultMultiChannelLeafRootSignal = () => {
|
|
4382
|
+
if (!defaultMultiChannelSettledLeafRootCandidate)
|
|
4383
|
+
return true;
|
|
4384
|
+
if (!isEnoughLevelGain(ch.id.root, 0))
|
|
4385
|
+
return false;
|
|
4386
|
+
if ((ch.parentDataLatencySamples ?? 0) < 8)
|
|
4387
|
+
return false;
|
|
4388
|
+
if ((ch.parentDataLatencyEwmaMs ?? 0) < 64 &&
|
|
4389
|
+
(ch.parentDataLatencyMaxMs ?? 0) < 256) {
|
|
4390
|
+
return false;
|
|
4391
|
+
}
|
|
4392
|
+
if (staleRootProbeBaseProbability <= 0)
|
|
4393
|
+
return false;
|
|
4394
|
+
const sampleProbability = Math.min(1, staleRootProbeBaseProbability);
|
|
4395
|
+
return (stableUnitInterval(`${ch.id.suffixKey}:${this.publicKeyHash}:${ch.id.root}:multi-channel-leaf-root-signal:${ch.maxSeqSeen}`) < sampleProbability);
|
|
4396
|
+
};
|
|
4397
|
+
const defaultMultiChannelLeafRootSignal = hasDefaultMultiChannelLeafRootSignal();
|
|
4398
|
+
if (defaultMultiChannelSettledLeafRootCandidate &&
|
|
4399
|
+
defaultMultiChannelLeafRootSignal &&
|
|
4400
|
+
ch.id.root !== this.publicKeyHash &&
|
|
4401
|
+
isDirectNeighbor(ch.id.root) &&
|
|
4402
|
+
(options.trackerPeers?.length ?? 0) > 0) {
|
|
4403
|
+
const rootMinFreeSlots = parentUpgrade.rootMinFreeSlots;
|
|
4404
|
+
const cachedRoot = ch.cachedTrackerCandidates.find((c) => c.hash === ch.id.root);
|
|
4405
|
+
if (!cachedRoot || cachedRoot.freeSlots < rootMinFreeSlots) {
|
|
4406
|
+
const refreshed = await this.queryTrackers(ch, options.trackerPeers ?? [], Math.max(16, ch.cachedTrackerCandidates.length), parentUpgrade.probe.timeoutMs, options.signal).catch(() => []);
|
|
4407
|
+
if (refreshed.length > 0) {
|
|
4408
|
+
ch.cachedTrackerCandidates = refreshed;
|
|
4409
|
+
}
|
|
4410
|
+
}
|
|
4411
|
+
}
|
|
4412
|
+
const configuredRejectCooldownMs = Math.max(0, parentUpgrade.probe.rejectCooldownMs);
|
|
4413
|
+
const rejectCooldownMs = localChannelCount > 1
|
|
4414
|
+
? Math.max(configuredRejectCooldownMs, 20_000)
|
|
4415
|
+
: configuredRejectCooldownMs;
|
|
4416
|
+
const rejectCooldownMaxMs = Math.max(rejectCooldownMs, parentUpgrade.probe.rejectCooldownMaxMs);
|
|
4417
|
+
const clearProbeRejectBackoff = (candidateHash) => {
|
|
4418
|
+
ch.parentProbeRejectUntilByHash.delete(candidateHash);
|
|
4419
|
+
ch.parentProbeRejectBackoffMsByHash.delete(candidateHash);
|
|
4420
|
+
};
|
|
4421
|
+
const rejectProbeCandidate = (candidateHash) => {
|
|
4422
|
+
if (rejectCooldownMs <= 0)
|
|
4423
|
+
return;
|
|
4424
|
+
const nextBackoffMs = (ch.parentProbeRejectBackoffMsByHash.get(candidateHash) ?? 0) > 0
|
|
4425
|
+
? Math.min(rejectCooldownMaxMs, Math.max(rejectCooldownMs, (ch.parentProbeRejectBackoffMsByHash.get(candidateHash) ?? 0) *
|
|
4426
|
+
2))
|
|
4427
|
+
: rejectCooldownMs;
|
|
4428
|
+
ch.parentProbeRejectBackoffMsByHash.set(candidateHash, nextBackoffMs);
|
|
4429
|
+
const jitter = 0.75 +
|
|
4430
|
+
upgradeRandom(`probe-reject:${candidateHash}:${nextBackoffMs}`) * 0.5;
|
|
4431
|
+
ch.parentProbeRejectUntilByHash.set(candidateHash, Date.now() + Math.max(1, Math.floor(nextBackoffMs * jitter)));
|
|
4432
|
+
while (ch.parentProbeRejectBackoffMsByHash.size >
|
|
4433
|
+
KNOWN_CANDIDATE_ADDRS_MAX_ENTRIES) {
|
|
4434
|
+
const oldest = ch.parentProbeRejectBackoffMsByHash.keys().next()
|
|
4435
|
+
.value;
|
|
4436
|
+
if (!oldest)
|
|
4437
|
+
break;
|
|
4438
|
+
ch.parentProbeRejectBackoffMsByHash.delete(oldest);
|
|
4439
|
+
ch.parentProbeRejectUntilByHash.delete(oldest);
|
|
4440
|
+
}
|
|
4441
|
+
while (ch.parentProbeRejectUntilByHash.size > KNOWN_CANDIDATE_ADDRS_MAX_ENTRIES) {
|
|
4442
|
+
const oldest = ch.parentProbeRejectUntilByHash.keys().next().value;
|
|
4443
|
+
if (!oldest)
|
|
4444
|
+
break;
|
|
4445
|
+
ch.parentProbeRejectUntilByHash.delete(oldest);
|
|
4446
|
+
ch.parentProbeRejectBackoffMsByHash.delete(oldest);
|
|
4447
|
+
}
|
|
4448
|
+
};
|
|
4449
|
+
const minFreeSlotsFor = (hash) => hash === ch.id.root
|
|
4450
|
+
? parentUpgrade.rootMinFreeSlots
|
|
4451
|
+
: parentUpgrade.minFreeSlots;
|
|
4452
|
+
const refreshCachedTrackerCandidate = (hash, level, freeSlots) => {
|
|
4453
|
+
const cached = ch.cachedTrackerCandidates.find((c) => c.hash === hash);
|
|
4454
|
+
if (!cached)
|
|
4455
|
+
return;
|
|
4456
|
+
if (Number.isFinite(level))
|
|
4457
|
+
cached.level = Math.max(0, Math.floor(level));
|
|
4458
|
+
if (Number.isFinite(freeSlots)) {
|
|
4459
|
+
cached.freeSlots = Math.max(0, Math.floor(freeSlots));
|
|
4460
|
+
}
|
|
4461
|
+
};
|
|
4462
|
+
const sendProbeTrackerFeedback = (candidateHash, reason) => {
|
|
4463
|
+
const trackerPeers = options.trackerPeers ?? [];
|
|
4464
|
+
if (trackerPeers.length === 0)
|
|
4465
|
+
return;
|
|
4466
|
+
void this.sendTrackerFeedback(ch, trackerPeers, candidateHash, TRACKER_FEEDBACK_JOIN_REJECT, reason).catch(() => { });
|
|
4467
|
+
};
|
|
4468
|
+
for (const c of ch.cachedTrackerCandidates) {
|
|
4469
|
+
if (c.hash === this.publicKeyHash)
|
|
4470
|
+
continue;
|
|
4471
|
+
if (!isDirectNeighbor(c.hash))
|
|
4472
|
+
continue;
|
|
4473
|
+
upsertCandidate({
|
|
4474
|
+
hash: c.hash,
|
|
4475
|
+
addrs: c.addrs,
|
|
4476
|
+
level: c.level,
|
|
4477
|
+
freeSlots: Number.isFinite(c.freeSlots)
|
|
4478
|
+
? Math.max(0, Math.floor(c.freeSlots))
|
|
4479
|
+
: 0,
|
|
4480
|
+
bidPerByte: c.bidPerByte,
|
|
4481
|
+
source: 0,
|
|
4482
|
+
});
|
|
4483
|
+
}
|
|
4484
|
+
if (ch.id.root !== this.publicKeyHash &&
|
|
4485
|
+
isDirectNeighbor(ch.id.root) &&
|
|
4486
|
+
!candidatesByHash.has(ch.id.root)) {
|
|
4487
|
+
upsertCandidate({
|
|
4488
|
+
hash: ch.id.root,
|
|
4489
|
+
addrs: [],
|
|
4490
|
+
level: 0,
|
|
4491
|
+
freeSlots: 1,
|
|
4492
|
+
bidPerByte: 0,
|
|
4493
|
+
source: -1,
|
|
4494
|
+
});
|
|
4495
|
+
}
|
|
4496
|
+
let connectedFallbackAdded = 0;
|
|
4497
|
+
const connectedFallbackMax = 64;
|
|
4498
|
+
for (const h of this.peers.keys()) {
|
|
4499
|
+
if (h === this.publicKeyHash)
|
|
4500
|
+
continue;
|
|
4501
|
+
if (!isDirectNeighbor(h))
|
|
4502
|
+
continue;
|
|
4503
|
+
upsertCandidate({
|
|
4504
|
+
hash: h,
|
|
4505
|
+
addrs: [],
|
|
4506
|
+
level: 0xffff,
|
|
4507
|
+
freeSlots: 0,
|
|
4508
|
+
bidPerByte: 0,
|
|
4509
|
+
source: 2,
|
|
4510
|
+
});
|
|
4511
|
+
connectedFallbackAdded += 1;
|
|
4512
|
+
if (connectedFallbackAdded >= connectedFallbackMax)
|
|
4513
|
+
break;
|
|
4514
|
+
}
|
|
4515
|
+
const orderedCandidates = [];
|
|
4516
|
+
for (const c of candidatesByHash.values()) {
|
|
4517
|
+
if (c.hash === this.publicKeyHash || c.hash === currentParent)
|
|
4518
|
+
continue;
|
|
4519
|
+
const canVerifyRootCapacityLive = parentUpgrade.verifyStaleRootCapacity === true &&
|
|
4520
|
+
parentUpgrade.mode === "shadow" &&
|
|
4521
|
+
c.hash === ch.id.root;
|
|
4522
|
+
const requiredMinFreeSlots = minFreeSlotsFor(c.hash);
|
|
4523
|
+
if (c.hash === ch.id.root &&
|
|
4524
|
+
defaultMultiChannelSettledLeafRootCandidate &&
|
|
4525
|
+
!defaultMultiChannelLeafRootSignal) {
|
|
4526
|
+
ch.metrics.reparentUpgradeSkipCandidateSlots += 1;
|
|
4527
|
+
continue;
|
|
4528
|
+
}
|
|
4529
|
+
if (canVerifyRootCapacityLive && c.freeSlots < requiredMinFreeSlots) {
|
|
4530
|
+
const staleRootProbeProbability = (() => {
|
|
4531
|
+
if (singleChannelBranch && staleRootProbeBaseProbability > 0) {
|
|
4532
|
+
const highValueBranch = rootSubtreeGainFor(c.level) >=
|
|
4533
|
+
Math.max(rootMinSubtreeGain + 1, 4);
|
|
4534
|
+
const branchCap = highValueBranch ? 0.25 : 0.2;
|
|
4535
|
+
return Math.max(staleRootProbeBaseProbability, Math.min(branchCap, staleRootProbeBaseProbability * 8));
|
|
4536
|
+
}
|
|
4537
|
+
if (multiChannelBranch && staleRootProbeBaseProbability > 0) {
|
|
4538
|
+
return Math.max(staleRootProbeBaseProbability, Math.min(0.03125, staleRootProbeBaseProbability * 2));
|
|
4539
|
+
}
|
|
4540
|
+
if (singleChannelSettledLeaf && staleRootProbeBaseProbability > 0) {
|
|
4541
|
+
return 0.5;
|
|
4542
|
+
}
|
|
4543
|
+
if (multiChannelSettledLeaf && parentUpgrade.leafOnly) {
|
|
4544
|
+
return staleRootProbeBaseProbability >
|
|
4545
|
+
defaultStaleRootProbeProbability
|
|
4546
|
+
? Math.max(staleRootProbeBaseProbability, Math.min(0.125, 3 / knownChannelPeerCount))
|
|
4547
|
+
: 0;
|
|
4548
|
+
}
|
|
4549
|
+
return Math.min(1, staleRootProbeBaseProbability *
|
|
4550
|
+
Math.max(1, Math.min(4, staleRootProbeRound + 1)));
|
|
4551
|
+
})();
|
|
4552
|
+
const staleRootProbeScore = stableUnitInterval(`${ch.id.suffixKey}:${this.publicKeyHash}:${c.hash}:stale-root-probe:${staleRootProbeRound}`);
|
|
4553
|
+
if (staleRootProbeScore >= staleRootProbeProbability) {
|
|
4554
|
+
ch.metrics.reparentUpgradeSkipCandidateSlots += 1;
|
|
4555
|
+
continue;
|
|
4556
|
+
}
|
|
4557
|
+
}
|
|
4558
|
+
if (!canVerifyRootCapacityLive && c.freeSlots < requiredMinFreeSlots) {
|
|
4559
|
+
ch.metrics.reparentUpgradeSkipCandidateSlots += 1;
|
|
4560
|
+
continue;
|
|
4561
|
+
}
|
|
4562
|
+
if (!isEnoughLevelGain(c.hash, c.level)) {
|
|
4563
|
+
ch.metrics.reparentUpgradeSkipCandidateLevel += 1;
|
|
4564
|
+
continue;
|
|
4565
|
+
}
|
|
4566
|
+
orderedCandidates.push(c);
|
|
4567
|
+
}
|
|
4568
|
+
let ordered = orderedCandidates.sort((a, b) => {
|
|
4569
|
+
if (a.level !== b.level)
|
|
4570
|
+
return a.level - b.level;
|
|
4571
|
+
if (a.freeSlots !== b.freeSlots)
|
|
4572
|
+
return b.freeSlots - a.freeSlots;
|
|
4573
|
+
if (a.bidPerByte !== b.bidPerByte)
|
|
4574
|
+
return b.bidPerByte - a.bidPerByte;
|
|
4575
|
+
if (a.source !== b.source)
|
|
4576
|
+
return a.source - b.source;
|
|
4577
|
+
return a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0;
|
|
4578
|
+
});
|
|
4579
|
+
if (ordered.length === 0) {
|
|
4580
|
+
if (parentUpgrade.mode === "probe" || parentUpgrade.mode === "shadow") {
|
|
4581
|
+
applyFailedBackoff();
|
|
4582
|
+
}
|
|
4583
|
+
return false;
|
|
4584
|
+
}
|
|
4585
|
+
if (options.candidateScoringMode === "ranked-shuffle") {
|
|
4586
|
+
if (options.candidateShuffleTopK > 0 && ordered.length > 1) {
|
|
4587
|
+
const k = Math.min(options.candidateShuffleTopK, ordered.length);
|
|
4588
|
+
const shuffleScope = ordered.map((c) => c.hash).join(",");
|
|
4589
|
+
for (let i = k - 1; i > 0; i--) {
|
|
4590
|
+
const j = Math.floor(upgradeRandom(`shuffle:${shuffleScope}:${i}`) * (i + 1));
|
|
4591
|
+
const tmp = ordered[i];
|
|
4592
|
+
ordered[i] = ordered[j];
|
|
4593
|
+
ordered[j] = tmp;
|
|
4594
|
+
}
|
|
4595
|
+
}
|
|
4596
|
+
}
|
|
4597
|
+
else if (options.candidateScoringMode === "weighted") {
|
|
4598
|
+
const wLevel = Number.isFinite(options.candidateScoringWeights.level)
|
|
4599
|
+
? Math.max(0, options.candidateScoringWeights.level)
|
|
4600
|
+
: 0;
|
|
4601
|
+
const wSlots = Number.isFinite(options.candidateScoringWeights.freeSlots)
|
|
4602
|
+
? Math.max(0, options.candidateScoringWeights.freeSlots)
|
|
4603
|
+
: 0;
|
|
4604
|
+
const wConnected = Number.isFinite(options.candidateScoringWeights.connected)
|
|
4605
|
+
? Math.max(0, options.candidateScoringWeights.connected)
|
|
4606
|
+
: 0;
|
|
4607
|
+
const wBid = Number.isFinite(options.candidateScoringWeights.bidPerByte)
|
|
4608
|
+
? Math.max(0, options.candidateScoringWeights.bidPerByte)
|
|
4609
|
+
: 0;
|
|
4610
|
+
const wSource = Number.isFinite(options.candidateScoringWeights.source)
|
|
4611
|
+
? Math.max(0, options.candidateScoringWeights.source)
|
|
4612
|
+
: 0;
|
|
4613
|
+
const k = options.candidateShuffleTopK > 0
|
|
4614
|
+
? Math.min(options.candidateShuffleTopK, ordered.length)
|
|
4615
|
+
: 0;
|
|
4616
|
+
const hasWeightedSignal = wLevel > 0 || wSlots > 0 || wConnected > 0 || wBid > 0 || wSource > 0;
|
|
4617
|
+
if (k > 1 && hasWeightedSignal) {
|
|
4618
|
+
const head = ordered.slice(0, k);
|
|
4619
|
+
const tail = ordered.slice(k);
|
|
4620
|
+
const weightOf = (c) => {
|
|
4621
|
+
const level = Math.max(0, Math.floor(c.level));
|
|
4622
|
+
const freeSlots = Math.max(0, Math.floor(c.freeSlots));
|
|
4623
|
+
const bidPerByte = Math.max(0, Math.floor(c.bidPerByte));
|
|
4624
|
+
const source = Math.max(0, Math.floor(c.source));
|
|
4625
|
+
const connected = Boolean(this.peers.get(c.hash));
|
|
4626
|
+
let weight = 1;
|
|
4627
|
+
if (wLevel > 0)
|
|
4628
|
+
weight *= 1 / (1 + wLevel * level);
|
|
4629
|
+
if (wSlots > 0)
|
|
4630
|
+
weight *= 1 + wSlots * freeSlots;
|
|
4631
|
+
if (wConnected > 0 && connected)
|
|
4632
|
+
weight *= 1 + wConnected;
|
|
4633
|
+
if (wBid > 0)
|
|
4634
|
+
weight *= 1 + wBid * bidPerByte;
|
|
4635
|
+
if (wSource > 0)
|
|
4636
|
+
weight *= 1 / (1 + wSource * source);
|
|
4637
|
+
return weight;
|
|
4638
|
+
};
|
|
4639
|
+
const out = [];
|
|
4640
|
+
const remaining = [...head];
|
|
4641
|
+
while (remaining.length > 0) {
|
|
4642
|
+
let sum = 0;
|
|
4643
|
+
const weights = new Array(remaining.length);
|
|
4644
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
4645
|
+
const w = weightOf(remaining[i]);
|
|
4646
|
+
const v = Number.isFinite(w) ? Math.max(0, w) : 0;
|
|
4647
|
+
weights[i] = v;
|
|
4648
|
+
sum += v;
|
|
4649
|
+
}
|
|
4650
|
+
if (sum <= 0) {
|
|
4651
|
+
out.push(...remaining);
|
|
4652
|
+
break;
|
|
4653
|
+
}
|
|
4654
|
+
let r = upgradeRandom(`weighted:${out.length}:${remaining
|
|
4655
|
+
.map((candidate) => candidate.hash)
|
|
4656
|
+
.join(",")}`) * sum;
|
|
4657
|
+
let pick = 0;
|
|
4658
|
+
for (; pick < weights.length; pick++) {
|
|
4659
|
+
r -= weights[pick];
|
|
4660
|
+
if (r <= 0)
|
|
4661
|
+
break;
|
|
4662
|
+
}
|
|
4663
|
+
if (pick >= remaining.length)
|
|
4664
|
+
pick = remaining.length - 1;
|
|
4665
|
+
out.push(remaining[pick]);
|
|
4666
|
+
remaining.splice(pick, 1);
|
|
4667
|
+
}
|
|
4668
|
+
ordered = out.concat(tail);
|
|
4669
|
+
}
|
|
4670
|
+
}
|
|
4671
|
+
if (parentUpgrade.mode === "probe" || parentUpgrade.mode === "shadow") {
|
|
4672
|
+
const mode = parentUpgrade.mode;
|
|
4673
|
+
const probeLimit = parentUpgrade.mode === "shadow"
|
|
4674
|
+
? 1
|
|
4675
|
+
: parentUpgrade.probe.maxPerRound;
|
|
4676
|
+
const shadowObserveMsForGate = parentUpgrade.shadow.observeMs;
|
|
4677
|
+
const shadowMinObservationsForGate = parentUpgrade.shadow.minObservations;
|
|
4678
|
+
const probeTimeoutMs = parentUpgrade.probe.timeoutMs;
|
|
4679
|
+
const maxLag = parentUpgrade.probe.maxLagMessages;
|
|
4680
|
+
const now = Date.now();
|
|
4681
|
+
const candidatesToProbe = [];
|
|
4682
|
+
const shadowCandidate = mode === "shadow" && ch.parentShadow
|
|
4683
|
+
? ordered.find((candidate) => candidate.hash === ch.parentShadow?.hash)
|
|
4684
|
+
: undefined;
|
|
4685
|
+
if (mode === "shadow" &&
|
|
4686
|
+
shadowCandidate &&
|
|
4687
|
+
ch.parentShadow &&
|
|
4688
|
+
shadowObserveMsForGate > 0 &&
|
|
4689
|
+
now - ch.parentShadow.startedAt < shadowObserveMsForGate) {
|
|
4690
|
+
return false;
|
|
4691
|
+
}
|
|
4692
|
+
const probeOrder = shadowCandidate
|
|
4693
|
+
? [
|
|
4694
|
+
shadowCandidate,
|
|
4695
|
+
...ordered.filter((candidate) => candidate.hash !== shadowCandidate.hash),
|
|
4696
|
+
]
|
|
4697
|
+
: ordered;
|
|
4698
|
+
for (const candidate of probeOrder) {
|
|
4699
|
+
const rejectUntil = ch.parentProbeRejectUntilByHash.get(candidate.hash);
|
|
4700
|
+
if (rejectUntil != null && rejectUntil > now) {
|
|
4701
|
+
ch.metrics.reparentUpgradeSkipProbeCooldown += 1;
|
|
4702
|
+
continue;
|
|
4703
|
+
}
|
|
4704
|
+
if (rejectUntil != null)
|
|
4705
|
+
ch.parentProbeRejectUntilByHash.delete(candidate.hash);
|
|
4706
|
+
candidatesToProbe.push(candidate);
|
|
4707
|
+
if (candidatesToProbe.length >= probeLimit)
|
|
4708
|
+
break;
|
|
4709
|
+
}
|
|
4710
|
+
if (candidatesToProbe.length === 0) {
|
|
4711
|
+
applyFailedBackoff();
|
|
4712
|
+
return false;
|
|
4713
|
+
}
|
|
4714
|
+
const shouldReserveRootCapacityForProbe = (candidate) => {
|
|
4715
|
+
if (candidate.hash !== ch.id.root)
|
|
4716
|
+
return true;
|
|
4717
|
+
if (mode !== "shadow")
|
|
4718
|
+
return true;
|
|
4719
|
+
if (endedAndComplete && candidate.level === 0)
|
|
4720
|
+
return true;
|
|
4721
|
+
if (!ch.parentShadow || ch.parentShadow.hash !== candidate.hash) {
|
|
4722
|
+
return (shadowObserveMsForGate <= 0 && shadowMinObservationsForGate <= 1);
|
|
4723
|
+
}
|
|
4724
|
+
return (now - ch.parentShadow.startedAt >= shadowObserveMsForGate &&
|
|
4725
|
+
ch.parentShadow.observations + 1 >= shadowMinObservationsForGate);
|
|
4726
|
+
};
|
|
4727
|
+
const rejectShadowCandidate = (candidateHash, reason) => {
|
|
4728
|
+
if (mode !== "shadow")
|
|
4729
|
+
return;
|
|
4730
|
+
switch (reason) {
|
|
4731
|
+
case "noReply":
|
|
4732
|
+
ch.metrics.parentShadowRejectNoReply += 1;
|
|
4733
|
+
break;
|
|
4734
|
+
case "notRooted":
|
|
4735
|
+
ch.metrics.parentShadowRejectNotRooted += 1;
|
|
4736
|
+
break;
|
|
4737
|
+
case "capacity":
|
|
4738
|
+
ch.metrics.parentShadowRejectCapacity += 1;
|
|
4739
|
+
break;
|
|
4740
|
+
case "repair":
|
|
4741
|
+
ch.metrics.parentShadowRejectRepair += 1;
|
|
4742
|
+
break;
|
|
4743
|
+
case "lag":
|
|
4744
|
+
ch.metrics.parentShadowRejectLag += 1;
|
|
4745
|
+
break;
|
|
4746
|
+
case "overloaded":
|
|
4747
|
+
ch.metrics.parentShadowRejectOverloaded += 1;
|
|
4748
|
+
break;
|
|
4749
|
+
case "level":
|
|
4750
|
+
ch.metrics.parentShadowRejectLevel += 1;
|
|
4751
|
+
break;
|
|
4752
|
+
}
|
|
4753
|
+
if (ch.parentShadow?.hash === candidateHash)
|
|
4754
|
+
resetShadow();
|
|
4755
|
+
deferParentUpgradeRetryUntilFreshData();
|
|
4756
|
+
};
|
|
4757
|
+
const probed = await Promise.all(candidatesToProbe.map(async (candidate) => ({
|
|
4758
|
+
candidate,
|
|
4759
|
+
reply: await this.probeParentCandidate(ch, candidate.hash, probeTimeoutMs, options.signal, minFreeSlotsFor(candidate.hash), shouldReserveRootCapacityForProbe(candidate)),
|
|
4760
|
+
})));
|
|
4761
|
+
const liveCandidates = [];
|
|
4762
|
+
for (const { candidate, reply } of probed) {
|
|
4763
|
+
if (!reply) {
|
|
4764
|
+
ch.metrics.reparentUpgradeSkipProbeNoReply += 1;
|
|
4765
|
+
rejectShadowCandidate(candidate.hash, "noReply");
|
|
4766
|
+
rejectProbeCandidate(candidate.hash);
|
|
4767
|
+
continue;
|
|
4768
|
+
}
|
|
4769
|
+
if (!reply.rooted) {
|
|
4770
|
+
ch.metrics.reparentUpgradeSkipProbeNotRooted += 1;
|
|
4771
|
+
refreshCachedTrackerCandidate(candidate.hash, reply.level, 0);
|
|
4772
|
+
sendProbeTrackerFeedback(candidate.hash, JOIN_REJECT_NOT_ATTACHED);
|
|
4773
|
+
rejectShadowCandidate(candidate.hash, "notRooted");
|
|
4774
|
+
rejectProbeCandidate(candidate.hash);
|
|
4775
|
+
continue;
|
|
4776
|
+
}
|
|
4777
|
+
const liveLevel = Number.isFinite(reply.level)
|
|
4778
|
+
? Math.max(0, Math.floor(reply.level))
|
|
4779
|
+
: candidate.level;
|
|
4780
|
+
const liveFreeSlots = Number.isFinite(reply.freeSlots)
|
|
4781
|
+
? Math.max(0, Math.floor(reply.freeSlots))
|
|
4782
|
+
: 0;
|
|
4783
|
+
refreshCachedTrackerCandidate(candidate.hash, liveLevel, liveFreeSlots);
|
|
4784
|
+
const requiredMinFreeSlots = minFreeSlotsFor(candidate.hash);
|
|
4785
|
+
if (!reply.accepting || liveFreeSlots < requiredMinFreeSlots) {
|
|
4786
|
+
ch.metrics.reparentUpgradeSkipCandidateSlots += 1;
|
|
4787
|
+
if (!reply.accepting || liveFreeSlots <= 0) {
|
|
4788
|
+
sendProbeTrackerFeedback(candidate.hash, JOIN_REJECT_NO_CAPACITY);
|
|
4789
|
+
}
|
|
4790
|
+
rejectShadowCandidate(candidate.hash, "capacity");
|
|
4791
|
+
rejectProbeCandidate(candidate.hash);
|
|
4792
|
+
continue;
|
|
4793
|
+
}
|
|
4794
|
+
const configuredMaxChildLoadRatioRaw = candidate.hash === ch.id.root
|
|
4795
|
+
? parentUpgrade.rootMaxChildLoadRatio
|
|
4796
|
+
: parentUpgrade.maxChildLoadRatio;
|
|
4797
|
+
const configuredMaxChildLoadRatio = candidate.hash === ch.id.root &&
|
|
4798
|
+
localChannelCount > 1 &&
|
|
4799
|
+
Number(configuredMaxChildLoadRatioRaw) > 0
|
|
4800
|
+
? Math.min(0.2, Number(configuredMaxChildLoadRatioRaw))
|
|
4801
|
+
: configuredMaxChildLoadRatioRaw;
|
|
4802
|
+
const maxChildLoadRatio = Math.max(0, Number(configuredMaxChildLoadRatio));
|
|
4803
|
+
if (maxChildLoadRatio > 0 &&
|
|
4804
|
+
reply.maxChildren > 0 &&
|
|
4805
|
+
(reply.children + 1) / reply.maxChildren > maxChildLoadRatio) {
|
|
4806
|
+
ch.metrics.reparentUpgradeSkipCandidateSlots += 1;
|
|
4807
|
+
ch.metrics.reparentUpgradeSkipCandidatePressure += 1;
|
|
4808
|
+
if (candidate.hash === ch.id.root) {
|
|
4809
|
+
ch.metrics.reparentUpgradeSkipRootPressure += 1;
|
|
4810
|
+
refreshCachedTrackerCandidate(candidate.hash, liveLevel, 0);
|
|
4811
|
+
sendProbeTrackerFeedback(candidate.hash, JOIN_REJECT_NO_CAPACITY);
|
|
4812
|
+
}
|
|
4813
|
+
rejectShadowCandidate(candidate.hash, "capacity");
|
|
4814
|
+
rejectProbeCandidate(candidate.hash);
|
|
4815
|
+
continue;
|
|
4816
|
+
}
|
|
4817
|
+
if (reply.repairing || reply.missingSeqs > 0) {
|
|
4818
|
+
ch.metrics.reparentUpgradeSkipProbeRepair += 1;
|
|
4819
|
+
rejectShadowCandidate(candidate.hash, "repair");
|
|
4820
|
+
rejectProbeCandidate(candidate.hash);
|
|
4821
|
+
continue;
|
|
4822
|
+
}
|
|
4823
|
+
if (reply.overloaded) {
|
|
4824
|
+
ch.metrics.reparentUpgradeSkipProbeOverloaded += 1;
|
|
4825
|
+
rejectShadowCandidate(candidate.hash, "overloaded");
|
|
4826
|
+
rejectProbeCandidate(candidate.hash);
|
|
4827
|
+
continue;
|
|
4828
|
+
}
|
|
4829
|
+
if (ch.maxSeqSeen >= 0 &&
|
|
4830
|
+
reply.haveToExclusive + maxLag < ch.maxSeqSeen + 1) {
|
|
4831
|
+
ch.metrics.reparentUpgradeSkipProbeLag += 1;
|
|
4832
|
+
rejectShadowCandidate(candidate.hash, "lag");
|
|
4833
|
+
rejectProbeCandidate(candidate.hash);
|
|
4834
|
+
continue;
|
|
4835
|
+
}
|
|
4836
|
+
if (!isEnoughLevelGain(candidate.hash, liveLevel)) {
|
|
4837
|
+
ch.metrics.reparentUpgradeSkipCandidateLevel += 1;
|
|
4838
|
+
rejectShadowCandidate(candidate.hash, "level");
|
|
4839
|
+
rejectProbeCandidate(candidate.hash);
|
|
4840
|
+
continue;
|
|
4841
|
+
}
|
|
4842
|
+
clearProbeRejectBackoff(candidate.hash);
|
|
4843
|
+
liveCandidates.push({
|
|
4844
|
+
...candidate,
|
|
4845
|
+
level: liveLevel,
|
|
4846
|
+
freeSlots: liveFreeSlots,
|
|
4847
|
+
haveToExclusive: reply.haveToExclusive,
|
|
4848
|
+
reservationToken: reply.reservationToken,
|
|
4849
|
+
});
|
|
4850
|
+
}
|
|
4851
|
+
ordered = liveCandidates.sort((a, b) => {
|
|
4852
|
+
if (a.level !== b.level)
|
|
4853
|
+
return a.level - b.level;
|
|
4854
|
+
if (a.freeSlots !== b.freeSlots)
|
|
4855
|
+
return b.freeSlots - a.freeSlots;
|
|
4856
|
+
if (a.bidPerByte !== b.bidPerByte)
|
|
4857
|
+
return b.bidPerByte - a.bidPerByte;
|
|
4858
|
+
if (a.source !== b.source)
|
|
4859
|
+
return a.source - b.source;
|
|
4860
|
+
return a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0;
|
|
4861
|
+
});
|
|
4862
|
+
if (ordered.length === 0) {
|
|
4863
|
+
applyFailedBackoff();
|
|
4864
|
+
return false;
|
|
4865
|
+
}
|
|
4866
|
+
clearFailedBackoff();
|
|
4867
|
+
if (mode === "shadow") {
|
|
4868
|
+
const nowAfterProbe = Date.now();
|
|
4869
|
+
const observeMs = parentUpgrade.shadow.observeMs;
|
|
4870
|
+
const minObservations = shadowMinObservationsForGate;
|
|
4871
|
+
const requireDataForShadowPromotion = parentUpgrade.shadow.dualPathMs > 0;
|
|
4872
|
+
const selected = ch.parentShadow != null
|
|
4873
|
+
? (ordered.find((candidate) => candidate.hash === ch.parentShadow?.hash) ?? ordered[0])
|
|
4874
|
+
: ordered[0];
|
|
4875
|
+
if (!selected) {
|
|
4876
|
+
resetShadow();
|
|
4877
|
+
return false;
|
|
4878
|
+
}
|
|
4879
|
+
if (endedAndComplete &&
|
|
4880
|
+
selected.hash === ch.id.root &&
|
|
4881
|
+
selected.level === 0 &&
|
|
4882
|
+
!requireDataForShadowPromotion) {
|
|
4883
|
+
if (!ch.parentShadow || ch.parentShadow.hash !== selected.hash) {
|
|
4884
|
+
ch.parentShadow = {
|
|
4885
|
+
hash: selected.hash,
|
|
4886
|
+
startedAt: nowAfterProbe - observeMs,
|
|
4887
|
+
observations: minObservations,
|
|
4888
|
+
level: selected.level,
|
|
4889
|
+
freeSlots: selected.freeSlots,
|
|
4890
|
+
haveToExclusive: "haveToExclusive" in selected &&
|
|
4891
|
+
typeof selected.haveToExclusive === "number"
|
|
4892
|
+
? selected.haveToExclusive
|
|
4893
|
+
: 0,
|
|
4894
|
+
liveDataMessages: 0,
|
|
4895
|
+
liveFirstDataMessages: 0,
|
|
4896
|
+
liveParentFirstDataMessages: 0,
|
|
4897
|
+
liveCandidateLeadSamples: 0,
|
|
4898
|
+
liveCandidateLeadMsTotal: 0,
|
|
4899
|
+
liveCandidateFirstAtBySeq: new Map(),
|
|
4900
|
+
liveParentFirstAtBySeq: new Map(),
|
|
4901
|
+
liveComparedSeqs: new Set(),
|
|
4902
|
+
liveMaxSeqSeen: -1,
|
|
4903
|
+
liveLastDataAt: 0,
|
|
4904
|
+
};
|
|
4905
|
+
this.parentUpgradeShadowInFlightSuffixKey = ch.id.suffixKey;
|
|
4906
|
+
ch.metrics.parentShadowStart += 1;
|
|
4907
|
+
}
|
|
4908
|
+
else {
|
|
4909
|
+
ch.parentShadow.observations += 1;
|
|
4910
|
+
ch.parentShadow.level = selected.level;
|
|
4911
|
+
ch.parentShadow.freeSlots = selected.freeSlots;
|
|
4912
|
+
ch.parentShadow.haveToExclusive =
|
|
4913
|
+
"haveToExclusive" in selected &&
|
|
4914
|
+
typeof selected.haveToExclusive === "number"
|
|
4915
|
+
? selected.haveToExclusive
|
|
4916
|
+
: 0;
|
|
4917
|
+
ch.metrics.parentShadowObserve += 1;
|
|
4918
|
+
}
|
|
4919
|
+
ordered = [selected];
|
|
4920
|
+
}
|
|
4921
|
+
else {
|
|
4922
|
+
if (!ch.parentShadow || ch.parentShadow.hash !== selected.hash) {
|
|
4923
|
+
ch.parentShadow = {
|
|
4924
|
+
hash: selected.hash,
|
|
4925
|
+
startedAt: nowAfterProbe,
|
|
4926
|
+
observations: 1,
|
|
4927
|
+
level: selected.level,
|
|
4928
|
+
freeSlots: selected.freeSlots,
|
|
4929
|
+
haveToExclusive: "haveToExclusive" in selected &&
|
|
4930
|
+
typeof selected.haveToExclusive === "number"
|
|
4931
|
+
? selected.haveToExclusive
|
|
4932
|
+
: 0,
|
|
4933
|
+
liveDataMessages: 0,
|
|
4934
|
+
liveFirstDataMessages: 0,
|
|
4935
|
+
liveParentFirstDataMessages: 0,
|
|
4936
|
+
liveCandidateLeadSamples: 0,
|
|
4937
|
+
liveCandidateLeadMsTotal: 0,
|
|
4938
|
+
liveCandidateFirstAtBySeq: new Map(),
|
|
4939
|
+
liveParentFirstAtBySeq: new Map(),
|
|
4940
|
+
liveComparedSeqs: new Set(),
|
|
4941
|
+
liveMaxSeqSeen: -1,
|
|
4942
|
+
liveLastDataAt: 0,
|
|
4943
|
+
};
|
|
4944
|
+
this.parentUpgradeShadowInFlightSuffixKey = ch.id.suffixKey;
|
|
4945
|
+
ch.metrics.parentShadowStart += 1;
|
|
4946
|
+
return false;
|
|
4947
|
+
}
|
|
4948
|
+
ch.parentShadow.observations += 1;
|
|
4949
|
+
ch.parentShadow.level = selected.level;
|
|
4950
|
+
ch.parentShadow.freeSlots = selected.freeSlots;
|
|
4951
|
+
ch.parentShadow.haveToExclusive =
|
|
4952
|
+
"haveToExclusive" in selected &&
|
|
4953
|
+
typeof selected.haveToExclusive === "number"
|
|
4954
|
+
? selected.haveToExclusive
|
|
4955
|
+
: 0;
|
|
4956
|
+
ch.metrics.parentShadowObserve += 1;
|
|
4957
|
+
if (ch.parentShadow.observations < minObservations ||
|
|
4958
|
+
nowAfterProbe - ch.parentShadow.startedAt < observeMs) {
|
|
4959
|
+
return false;
|
|
4960
|
+
}
|
|
4961
|
+
}
|
|
4962
|
+
ordered = [selected];
|
|
4963
|
+
}
|
|
4964
|
+
}
|
|
4965
|
+
let attempts = 0;
|
|
4966
|
+
for (const candidate of ordered) {
|
|
4967
|
+
if (options.signal.aborted)
|
|
4968
|
+
break;
|
|
4969
|
+
if (attempts >= options.joinAttemptsPerRound)
|
|
4970
|
+
break;
|
|
4971
|
+
attempts += 1;
|
|
4972
|
+
if (!isDirectNeighbor(candidate.hash))
|
|
4973
|
+
continue;
|
|
4974
|
+
const previousParent = ch.parent;
|
|
4975
|
+
const previousLevel = ch.level;
|
|
4976
|
+
const previousRouteFromRoot = ch.routeFromRoot
|
|
4977
|
+
? [...ch.routeFromRoot]
|
|
4978
|
+
: undefined;
|
|
4979
|
+
const previousLastParentDataAt = ch.lastParentDataAt;
|
|
4980
|
+
const previousLastParentUpgradeActivityAt = ch.lastParentUpgradeActivityAt;
|
|
4981
|
+
const previousReceivedAnyParentData = ch.receivedAnyParentData;
|
|
4982
|
+
const shadowDualPathMs = parentUpgrade.shadow.dualPathMs;
|
|
4983
|
+
const shadowDualPathMinMessages = parentUpgrade.shadow.dualPathMinMessages;
|
|
4984
|
+
const shadowDualPathMinAdvantageMessages = Math.max(1, Math.min(16, Math.ceil(shadowDualPathMinMessages / 2)));
|
|
4985
|
+
const shadowDualPathMinLeadSamples = Math.max(1, Math.min(32, shadowDualPathMinMessages));
|
|
4986
|
+
const shadowDualPathMaxParentFirstMessages = Math.max(0, Math.floor(shadowDualPathMinMessages / 8));
|
|
4987
|
+
const shadowDualPathMinLeadMsAvg = shadowDualPathMinMessages <= 1 ? 0 : 128;
|
|
4988
|
+
const useShadowDualPath = parentUpgrade.mode === "shadow" &&
|
|
4989
|
+
shadowDualPathMs > 0 &&
|
|
4990
|
+
previousParent != null &&
|
|
4991
|
+
previousParent !== candidate.hash;
|
|
4992
|
+
const seqAtShadowAttach = ch.maxSeqSeen;
|
|
4993
|
+
const reqId = (this.random() * 0xffffffff) >>> 0;
|
|
4994
|
+
const res = await this.tryJoinOnce(ch, candidate.hash, reqId, options.joinReqTimeoutMs, options.signal, {
|
|
4995
|
+
allowReplace: true,
|
|
4996
|
+
parentUpgradeReservationToken: candidate.hash === ch.id.root &&
|
|
4997
|
+
"reservationToken" in candidate &&
|
|
4998
|
+
typeof candidate.reservationToken === "number"
|
|
4999
|
+
? candidate.reservationToken
|
|
5000
|
+
: 0,
|
|
5001
|
+
shadowAttach: useShadowDualPath,
|
|
5002
|
+
});
|
|
5003
|
+
if (res.ok) {
|
|
5004
|
+
if (useShadowDualPath) {
|
|
5005
|
+
const acceptedParentLevel = Number.isFinite(res.parentLevel)
|
|
5006
|
+
? Math.max(0, Math.floor(res.parentLevel))
|
|
5007
|
+
: candidate.level;
|
|
5008
|
+
if (!isEnoughLevelGain(candidate.hash, acceptedParentLevel)) {
|
|
5009
|
+
void this._sendControl(candidate.hash, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
5010
|
+
resetShadow();
|
|
5011
|
+
rejectProbeCandidate(candidate.hash);
|
|
5012
|
+
deferParentUpgradeRetryUntilFreshData(2);
|
|
5013
|
+
applyFailedBackoff();
|
|
5014
|
+
return false;
|
|
5015
|
+
}
|
|
5016
|
+
const shadow = ch.parentShadow;
|
|
5017
|
+
if (shadow?.hash === candidate.hash) {
|
|
5018
|
+
shadow.liveDataMessages = 0;
|
|
5019
|
+
shadow.liveFirstDataMessages = 0;
|
|
5020
|
+
shadow.liveParentFirstDataMessages = 0;
|
|
5021
|
+
shadow.liveCandidateLeadSamples = 0;
|
|
5022
|
+
shadow.liveCandidateLeadMsTotal = 0;
|
|
5023
|
+
shadow.liveCandidateFirstAtBySeq.clear();
|
|
5024
|
+
shadow.liveParentFirstAtBySeq.clear();
|
|
5025
|
+
shadow.liveComparedSeqs.clear();
|
|
5026
|
+
shadow.liveMaxSeqSeen = seqAtShadowAttach;
|
|
5027
|
+
shadow.liveLastDataAt = 0;
|
|
5028
|
+
}
|
|
5029
|
+
const deadline = Date.now() + shadowDualPathMs;
|
|
5030
|
+
for (;;) {
|
|
5031
|
+
if (options.signal.aborted)
|
|
5032
|
+
break;
|
|
5033
|
+
const currentShadow = ch.parentShadow;
|
|
5034
|
+
const observedFreshCandidateData = currentShadow?.hash === candidate.hash &&
|
|
5035
|
+
currentShadow.liveFirstDataMessages >=
|
|
5036
|
+
shadowDualPathMinMessages &&
|
|
5037
|
+
currentShadow.liveParentFirstDataMessages <=
|
|
5038
|
+
shadowDualPathMaxParentFirstMessages &&
|
|
5039
|
+
currentShadow.liveFirstDataMessages >=
|
|
5040
|
+
currentShadow.liveParentFirstDataMessages +
|
|
5041
|
+
shadowDualPathMinAdvantageMessages &&
|
|
5042
|
+
currentShadow.liveCandidateLeadSamples >=
|
|
5043
|
+
shadowDualPathMinLeadSamples &&
|
|
5044
|
+
currentShadow.liveCandidateLeadMsTotal >=
|
|
5045
|
+
currentShadow.liveCandidateLeadSamples *
|
|
5046
|
+
shadowDualPathMinLeadMsAvg &&
|
|
5047
|
+
currentShadow.liveMaxSeqSeen > seqAtShadowAttach;
|
|
5048
|
+
if (observedFreshCandidateData) {
|
|
5049
|
+
const parentRoute = res.parentRouteFromRoot;
|
|
5050
|
+
if (!parentRoute || parentRoute.length === 0)
|
|
5051
|
+
break;
|
|
5052
|
+
ch.parent = candidate.hash;
|
|
5053
|
+
ch.level = Math.max(1, Math.floor(res.parentLevel ?? candidate.level) + 1);
|
|
5054
|
+
ch.routeFromRoot = [...parentRoute, this.publicKeyHash];
|
|
5055
|
+
ch.joinedAtLeastOnce = true;
|
|
5056
|
+
ch.lastParentDataAt =
|
|
5057
|
+
currentShadow.liveLastDataAt > 0
|
|
5058
|
+
? currentShadow.liveLastDataAt
|
|
5059
|
+
: Date.now();
|
|
5060
|
+
ch.lastParentUpgradeActivityAt = ch.lastParentDataAt;
|
|
5061
|
+
ch.receivedAnyParentData = true;
|
|
5062
|
+
this.resetParentDataLatency(ch);
|
|
5063
|
+
this.touchPeerHint(ch, candidate.hash);
|
|
5064
|
+
if (previousParent) {
|
|
5065
|
+
this.startParentUpgradeGrace(ch, {
|
|
5066
|
+
previousParent,
|
|
5067
|
+
candidateParent: candidate.hash,
|
|
5068
|
+
previousLevel,
|
|
5069
|
+
previousRouteFromRoot,
|
|
5070
|
+
previousLastParentDataAt,
|
|
5071
|
+
previousLastParentUpgradeActivityAt,
|
|
5072
|
+
previousReceivedAnyParentData,
|
|
5073
|
+
candidateFirstMessages: 0,
|
|
5074
|
+
previousFirstMessages: 0,
|
|
5075
|
+
minCandidateFirstMessages: shadowDualPathMinMessages,
|
|
5076
|
+
minCandidateAdvantageMessages: shadowDualPathMinAdvantageMessages,
|
|
5077
|
+
maxPreviousFirstMessages: shadowDualPathMaxParentFirstMessages,
|
|
5078
|
+
timeoutMs: shadowDualPathMs,
|
|
5079
|
+
});
|
|
5080
|
+
}
|
|
5081
|
+
ch.metrics.parentShadowPromote += 1;
|
|
5082
|
+
ch.parentShadow = undefined;
|
|
5083
|
+
clearShadowInFlight();
|
|
5084
|
+
clearFailedBackoff();
|
|
5085
|
+
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
5086
|
+
return true;
|
|
5087
|
+
}
|
|
5088
|
+
const remainingMs = deadline - Date.now();
|
|
5089
|
+
if (remainingMs <= 0)
|
|
5090
|
+
break;
|
|
5091
|
+
await delay(Math.min(50, remainingMs), {
|
|
5092
|
+
signal: options.signal,
|
|
5093
|
+
});
|
|
5094
|
+
}
|
|
5095
|
+
if (ch.parent === previousParent) {
|
|
5096
|
+
ch.level = previousLevel;
|
|
5097
|
+
ch.routeFromRoot = previousRouteFromRoot;
|
|
5098
|
+
ch.lastParentDataAt = previousLastParentDataAt;
|
|
5099
|
+
ch.lastParentUpgradeActivityAt =
|
|
5100
|
+
previousLastParentUpgradeActivityAt;
|
|
5101
|
+
ch.receivedAnyParentData = previousReceivedAnyParentData;
|
|
5102
|
+
}
|
|
5103
|
+
void this._sendControl(candidate.hash, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
5104
|
+
if (ch.parentShadow) {
|
|
5105
|
+
ch.parentShadow = undefined;
|
|
5106
|
+
clearShadowInFlight();
|
|
5107
|
+
ch.metrics.parentShadowReset += 1;
|
|
5108
|
+
}
|
|
5109
|
+
// A shadow cutover can only be proven by fresh data. If no
|
|
5110
|
+
// candidate-first data arrived, wait for enough newer data before
|
|
5111
|
+
// spending more probes on the same channel. Also cool down this
|
|
5112
|
+
// candidate; accepting JOIN without proving live data is a weak
|
|
5113
|
+
// parent signal, not a reason to probe it repeatedly.
|
|
5114
|
+
rejectProbeCandidate(candidate.hash);
|
|
5115
|
+
deferParentUpgradeRetryUntilFreshData(2);
|
|
5116
|
+
applyFailedBackoff();
|
|
5117
|
+
return false;
|
|
5118
|
+
}
|
|
5119
|
+
const newParent = ch.parent;
|
|
5120
|
+
if (previousParent && newParent && newParent !== previousParent) {
|
|
5121
|
+
void this._sendControl(previousParent, this.codec.encodeLeave(ch.id.key)).catch(() => { });
|
|
5122
|
+
if (parentUpgrade.mode === "shadow") {
|
|
5123
|
+
ch.metrics.parentShadowPromote += 1;
|
|
5124
|
+
ch.parentShadow = undefined;
|
|
5125
|
+
clearShadowInFlight();
|
|
5126
|
+
clearFailedBackoff();
|
|
5127
|
+
}
|
|
5128
|
+
return true;
|
|
5129
|
+
}
|
|
5130
|
+
return false;
|
|
5131
|
+
}
|
|
5132
|
+
if (useShadowDualPath) {
|
|
5133
|
+
if (ch.parent === previousParent) {
|
|
5134
|
+
ch.level = previousLevel;
|
|
5135
|
+
ch.routeFromRoot = previousRouteFromRoot;
|
|
5136
|
+
ch.lastParentDataAt = previousLastParentDataAt;
|
|
5137
|
+
ch.lastParentUpgradeActivityAt =
|
|
5138
|
+
previousLastParentUpgradeActivityAt;
|
|
5139
|
+
ch.receivedAnyParentData = previousReceivedAnyParentData;
|
|
5140
|
+
}
|
|
5141
|
+
if (ch.parentShadow?.hash === candidate.hash) {
|
|
5142
|
+
ch.parentShadow = undefined;
|
|
5143
|
+
clearShadowInFlight();
|
|
5144
|
+
ch.metrics.parentShadowReset += 1;
|
|
5145
|
+
}
|
|
5146
|
+
rejectProbeCandidate(candidate.hash);
|
|
5147
|
+
deferParentUpgradeRetryUntilFreshData(2);
|
|
5148
|
+
applyFailedBackoff();
|
|
5149
|
+
return false;
|
|
5150
|
+
}
|
|
5151
|
+
}
|
|
5152
|
+
return false;
|
|
5153
|
+
}
|
|
5154
|
+
async tryJoinOnce(ch, parentHash, reqId, timeoutMs, signal, options) {
|
|
5155
|
+
if (ch.parent && options?.allowReplace !== true)
|
|
4153
5156
|
return { ok: true };
|
|
4154
5157
|
if (!this.peers.get(parentHash))
|
|
4155
5158
|
return { ok: false, timedOut: true };
|
|
4156
5159
|
const p = new Promise((resolve) => {
|
|
4157
|
-
ch.pendingJoin.set(reqId, {
|
|
5160
|
+
ch.pendingJoin.set(reqId, {
|
|
5161
|
+
resolve,
|
|
5162
|
+
shadowAttach: options?.shadowAttach === true,
|
|
5163
|
+
});
|
|
4158
5164
|
});
|
|
4159
|
-
await this._sendControl(parentHash, encodeJoinReq(ch.id.key, reqId, ch.bidPerByte));
|
|
5165
|
+
await this._sendControl(parentHash, this.codec.encodeJoinReq(ch.id.key, reqId, ch.bidPerByte, options?.parentUpgradeReservationToken));
|
|
4160
5166
|
const res = await Promise.race([
|
|
4161
5167
|
p,
|
|
4162
5168
|
delay(Math.max(1, timeoutMs), { signal }).then(() => ({
|
|
@@ -4168,13 +5174,40 @@ export class FanoutTree extends DirectStream {
|
|
|
4168
5174
|
ch.pendingJoin.delete(reqId);
|
|
4169
5175
|
return res;
|
|
4170
5176
|
}
|
|
4171
|
-
async
|
|
4172
|
-
const
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
5177
|
+
async kickChildHashes(ch, children, options) {
|
|
5178
|
+
const unique = [...new Set(children)].filter((h) => ch.children.has(h));
|
|
5179
|
+
for (const h of unique) {
|
|
5180
|
+
ch.children.delete(h);
|
|
5181
|
+
ch.dataWriteFailStreakByChild.delete(h);
|
|
5182
|
+
}
|
|
5183
|
+
if (unique.length === 0)
|
|
4176
5184
|
return;
|
|
4177
|
-
|
|
5185
|
+
const resetPeerConnections = options?.resetPeerConnections === true;
|
|
5186
|
+
let kickFailed = false;
|
|
5187
|
+
try {
|
|
5188
|
+
await this._sendControlMany(unique, this.codec.encodeKick(ch.id.key));
|
|
5189
|
+
}
|
|
5190
|
+
catch (error) {
|
|
5191
|
+
kickFailed = true;
|
|
5192
|
+
throw error;
|
|
5193
|
+
}
|
|
5194
|
+
finally {
|
|
5195
|
+
if (resetPeerConnections && kickFailed) {
|
|
5196
|
+
// If KICK cannot be written on the existing stream, close the peer
|
|
5197
|
+
// connection so the topology change becomes observable.
|
|
5198
|
+
for (const h of unique) {
|
|
5199
|
+
const stream = this.peers.get(h);
|
|
5200
|
+
if (!stream)
|
|
5201
|
+
continue;
|
|
5202
|
+
void this.components.connectionManager
|
|
5203
|
+
.closeConnections(stream.peerId)
|
|
5204
|
+
.catch(() => { });
|
|
5205
|
+
}
|
|
5206
|
+
}
|
|
5207
|
+
}
|
|
5208
|
+
}
|
|
5209
|
+
async kickChildren(ch) {
|
|
5210
|
+
await this.kickChildHashes(ch, [...ch.children.keys()]);
|
|
4178
5211
|
}
|
|
4179
5212
|
async onDataMessage(from, peerStream, message, seenBefore) {
|
|
4180
5213
|
const ignore = this.shouldIgnore(message, seenBefore);
|
|
@@ -4203,6 +5236,8 @@ export class FanoutTree extends DirectStream {
|
|
|
4203
5236
|
kind === MSG_TRACKER_QUERY ||
|
|
4204
5237
|
kind === MSG_TRACKER_REPLY ||
|
|
4205
5238
|
kind === MSG_TRACKER_FEEDBACK ||
|
|
5239
|
+
kind === MSG_PARENT_PROBE_REQ ||
|
|
5240
|
+
kind === MSG_PARENT_PROBE_REPLY ||
|
|
4206
5241
|
kind === MSG_PROVIDER_ANNOUNCE ||
|
|
4207
5242
|
kind === MSG_PROVIDER_QUERY ||
|
|
4208
5243
|
kind === MSG_PROVIDER_REPLY ||
|
|
@@ -4224,33 +5259,26 @@ export class FanoutTree extends DirectStream {
|
|
|
4224
5259
|
fromHash === ch.parent)) {
|
|
4225
5260
|
return false;
|
|
4226
5261
|
}
|
|
5262
|
+
if (!ch && kind === MSG_JOIN_REQ && data.length >= 1 + 32 + 4) {
|
|
5263
|
+
// A joiner probed us for a channel we do not host (connected-peer
|
|
5264
|
+
// fallbacks routinely hit bootstraps and non-subscribers). Failing
|
|
5265
|
+
// fast lets the joiner spend its probe budget elsewhere instead of
|
|
5266
|
+
// burning a full join-request timeout and a long cooldown on us.
|
|
5267
|
+
const reqId = readU32BE(data, 33);
|
|
5268
|
+
void this._sendControl(fromHash, this.codec.encodeJoinReject(channelKey, reqId, JOIN_REJECT_NOT_ATTACHED)).catch(() => { });
|
|
5269
|
+
return true;
|
|
5270
|
+
}
|
|
4227
5271
|
if (kind === MSG_TRACKER_ANNOUNCE) {
|
|
4228
|
-
|
|
5272
|
+
const decoded = this.codec.decodeTrackerAnnounce(data);
|
|
5273
|
+
if (!decoded)
|
|
4229
5274
|
return false;
|
|
4230
|
-
const ttlMs
|
|
4231
|
-
const level = readU16BE(data, 37);
|
|
4232
|
-
// maxChildren is currently unused, but kept in the wire format.
|
|
4233
|
-
// const maxChildren = readU16BE(data, 39);
|
|
4234
|
-
const freeSlots = readU16BE(data, 41);
|
|
4235
|
-
const bidPerByte = readU32BE(data, 43);
|
|
4236
|
-
const addrCount = data[47];
|
|
4237
|
-
let offset = 48;
|
|
4238
|
-
const addrs = [];
|
|
4239
|
-
const max = Math.min(addrCount, 16);
|
|
4240
|
-
for (let i = 0; i < max; i++) {
|
|
4241
|
-
if (offset + 2 > data.length)
|
|
4242
|
-
break;
|
|
4243
|
-
const len = readU16BE(data, offset);
|
|
4244
|
-
offset += 2;
|
|
4245
|
-
if (offset + len > data.length)
|
|
4246
|
-
break;
|
|
4247
|
-
addrs.push(data.subarray(offset, offset + len));
|
|
4248
|
-
offset += len;
|
|
4249
|
-
}
|
|
5275
|
+
const { ttlMs, level, freeSlots, bidPerByte, addrs } = decoded;
|
|
4250
5276
|
if (addrs.length === 0) {
|
|
4251
5277
|
try {
|
|
4252
5278
|
const peer = await this.components.peerStore.get(peerStream.peerId);
|
|
4253
|
-
const addresses = Array.isArray(peer?.addresses)
|
|
5279
|
+
const addresses = Array.isArray(peer?.addresses)
|
|
5280
|
+
? peer.addresses
|
|
5281
|
+
: [];
|
|
4254
5282
|
for (const a of addresses) {
|
|
4255
5283
|
const ma = a?.multiaddr ?? a;
|
|
4256
5284
|
const bytes = normalizeUint8Array(ma?.bytes);
|
|
@@ -4296,10 +5324,10 @@ export class FanoutTree extends DirectStream {
|
|
|
4296
5324
|
return true;
|
|
4297
5325
|
}
|
|
4298
5326
|
if (kind === MSG_TRACKER_QUERY) {
|
|
4299
|
-
|
|
5327
|
+
const decoded = this.codec.decodeTrackerQuery(data);
|
|
5328
|
+
if (!decoded)
|
|
4300
5329
|
return false;
|
|
4301
|
-
const reqId =
|
|
4302
|
-
const want = readU16BE(data, 37);
|
|
5330
|
+
const { reqId, want } = decoded;
|
|
4303
5331
|
const now = Date.now();
|
|
4304
5332
|
this.touchTrackerNamespace(suffixKey, now);
|
|
4305
5333
|
const byPeer = this.trackerBySuffixKey.get(suffixKey);
|
|
@@ -4312,8 +5340,9 @@ export class FanoutTree extends DirectStream {
|
|
|
4312
5340
|
}
|
|
4313
5341
|
if (hash === fromHash)
|
|
4314
5342
|
continue;
|
|
4315
|
-
if (e.freeSlots <= 0)
|
|
5343
|
+
if (e.freeSlots <= 0) {
|
|
4316
5344
|
continue;
|
|
5345
|
+
}
|
|
4317
5346
|
entries.push(e);
|
|
4318
5347
|
}
|
|
4319
5348
|
this.pruneTrackerNamespaceIfEmpty(suffixKey);
|
|
@@ -4327,21 +5356,28 @@ export class FanoutTree extends DirectStream {
|
|
|
4327
5356
|
return b.bidPerByte - a.bidPerByte;
|
|
4328
5357
|
return a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0;
|
|
4329
5358
|
});
|
|
4330
|
-
|
|
4331
|
-
|
|
5359
|
+
// Returning the deterministic head of the directory sends every
|
|
5360
|
+
// concurrent joiner to the same few candidates while deeper capacity
|
|
5361
|
+
// goes unprobed. Sample the reply from a wider best-first pool so
|
|
5362
|
+
// load spreads across all advertised capacity.
|
|
5363
|
+
const wantCount = clampU16(want);
|
|
5364
|
+
const poolSize = Math.min(entries.length, Math.max(wantCount * 4, wantCount));
|
|
5365
|
+
const pool = entries.slice(0, poolSize);
|
|
5366
|
+
for (let i = 0; i < Math.min(wantCount, pool.length); i++) {
|
|
5367
|
+
const j = i + Math.floor(this.random() * (pool.length - i));
|
|
5368
|
+
const tmp = pool[i];
|
|
5369
|
+
pool[i] = pool[j];
|
|
5370
|
+
pool[j] = tmp;
|
|
5371
|
+
}
|
|
5372
|
+
const picked = pool.slice(0, wantCount);
|
|
5373
|
+
void this._sendControl(fromHash, this.codec.encodeTrackerReply(channelKey, reqId, picked));
|
|
4332
5374
|
return true;
|
|
4333
5375
|
}
|
|
4334
5376
|
if (kind === MSG_TRACKER_FEEDBACK) {
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
const hashLen = data[33];
|
|
4338
|
-
let offset = 34;
|
|
4339
|
-
if (offset + hashLen + 2 > data.length)
|
|
5377
|
+
const decoded = this.codec.decodeTrackerFeedback(data);
|
|
5378
|
+
if (!decoded)
|
|
4340
5379
|
return false;
|
|
4341
|
-
const candidateHash
|
|
4342
|
-
offset += hashLen;
|
|
4343
|
-
const event = data[offset++] & 0xff;
|
|
4344
|
-
const reason = data[offset++] & 0xff;
|
|
5380
|
+
const { candidateHash, event, reason } = decoded;
|
|
4345
5381
|
const byPeer = this.trackerBySuffixKey.get(suffixKey);
|
|
4346
5382
|
if (!byPeer)
|
|
4347
5383
|
return true;
|
|
@@ -4359,13 +5395,15 @@ export class FanoutTree extends DirectStream {
|
|
|
4359
5395
|
entry.freeSlots = clampU16(Math.max(0, entry.freeSlots - 1));
|
|
4360
5396
|
return true;
|
|
4361
5397
|
}
|
|
4362
|
-
if (event === TRACKER_FEEDBACK_DIAL_FAILED ||
|
|
5398
|
+
if (event === TRACKER_FEEDBACK_DIAL_FAILED ||
|
|
5399
|
+
event === TRACKER_FEEDBACK_JOIN_TIMEOUT) {
|
|
4363
5400
|
byPeer.delete(candidateHash);
|
|
4364
5401
|
this.pruneTrackerNamespaceIfEmpty(suffixKey);
|
|
4365
5402
|
return true;
|
|
4366
5403
|
}
|
|
4367
5404
|
if (event === TRACKER_FEEDBACK_JOIN_REJECT) {
|
|
4368
|
-
if (reason === JOIN_REJECT_NO_CAPACITY ||
|
|
5405
|
+
if (reason === JOIN_REJECT_NO_CAPACITY ||
|
|
5406
|
+
reason === JOIN_REJECT_NOT_ATTACHED) {
|
|
4369
5407
|
entry.freeSlots = 0;
|
|
4370
5408
|
// Expire earlier to reduce time-to-recovery if the entry is stale.
|
|
4371
5409
|
entry.expiresAt = Math.min(entry.expiresAt, now + 2_000);
|
|
@@ -4375,24 +5413,11 @@ export class FanoutTree extends DirectStream {
|
|
|
4375
5413
|
return true;
|
|
4376
5414
|
}
|
|
4377
5415
|
if (kind === MSG_PROVIDER_ANNOUNCE) {
|
|
4378
|
-
|
|
5416
|
+
const decoded = this.codec.decodeProviderAnnounce(data);
|
|
5417
|
+
if (!decoded)
|
|
4379
5418
|
return false;
|
|
4380
5419
|
const providerId = this.getProviderNamespaceIdFromKey(channelKey, suffixKey);
|
|
4381
|
-
const ttlMs =
|
|
4382
|
-
const addrCount = data[37];
|
|
4383
|
-
let offset = 38;
|
|
4384
|
-
const addrs = [];
|
|
4385
|
-
const max = Math.min(addrCount, 16);
|
|
4386
|
-
for (let i = 0; i < max; i++) {
|
|
4387
|
-
if (offset + 2 > data.length)
|
|
4388
|
-
break;
|
|
4389
|
-
const len = readU16BE(data, offset);
|
|
4390
|
-
offset += 2;
|
|
4391
|
-
if (offset + len > data.length)
|
|
4392
|
-
break;
|
|
4393
|
-
addrs.push(data.subarray(offset, offset + len));
|
|
4394
|
-
offset += len;
|
|
4395
|
-
}
|
|
5420
|
+
const { ttlMs, addrs } = decoded;
|
|
4396
5421
|
if (addrs.length === 0) {
|
|
4397
5422
|
try {
|
|
4398
5423
|
const peer = await this.components.peerStore.get(peerStream.peerId);
|
|
@@ -4446,11 +5471,10 @@ export class FanoutTree extends DirectStream {
|
|
|
4446
5471
|
return true;
|
|
4447
5472
|
}
|
|
4448
5473
|
if (kind === MSG_PROVIDER_QUERY) {
|
|
4449
|
-
|
|
5474
|
+
const decoded = this.codec.decodeProviderQuery(data);
|
|
5475
|
+
if (!decoded)
|
|
4450
5476
|
return false;
|
|
4451
|
-
const reqId =
|
|
4452
|
-
const want = readU16BE(data, 37);
|
|
4453
|
-
const seed = readU32BE(data, 39);
|
|
5477
|
+
const { reqId, want, seed } = decoded;
|
|
4454
5478
|
const now = Date.now();
|
|
4455
5479
|
this.touchProviderNamespace(suffixKey, now);
|
|
4456
5480
|
const byPeer = this.providerBySuffixKey.get(suffixKey);
|
|
@@ -4491,14 +5515,15 @@ export class FanoutTree extends DirectStream {
|
|
|
4491
5515
|
}
|
|
4492
5516
|
}
|
|
4493
5517
|
const picked = entries.slice(0, clampU16(want));
|
|
4494
|
-
void this._sendControl(fromHash, encodeProviderReply(channelKey, reqId, picked));
|
|
5518
|
+
void this._sendControl(fromHash, this.codec.encodeProviderReply(channelKey, reqId, picked));
|
|
4495
5519
|
return true;
|
|
4496
5520
|
}
|
|
4497
5521
|
if (kind === MSG_PROVIDER_SUBSCRIBE) {
|
|
4498
|
-
|
|
5522
|
+
const decoded = this.codec.decodeProviderSubscribe(data);
|
|
5523
|
+
if (!decoded)
|
|
4499
5524
|
return false;
|
|
4500
|
-
const want = Math.max(1,
|
|
4501
|
-
const ttlMs = Math.min(120_000, Math.max(1_000,
|
|
5525
|
+
const want = Math.max(1, decoded.want);
|
|
5526
|
+
const ttlMs = Math.min(120_000, Math.max(1_000, decoded.ttlMs));
|
|
4502
5527
|
const now = Date.now();
|
|
4503
5528
|
let watchers = this.providerWatchersBySuffixKey.get(suffixKey);
|
|
4504
5529
|
if (!watchers) {
|
|
@@ -4520,18 +5545,18 @@ export class FanoutTree extends DirectStream {
|
|
|
4520
5545
|
return true;
|
|
4521
5546
|
}
|
|
4522
5547
|
if (kind === MSG_PROVIDER_REPLY) {
|
|
4523
|
-
|
|
5548
|
+
const decoded = this.codec.decodeProviderReply(data);
|
|
5549
|
+
if (!decoded)
|
|
4524
5550
|
return false;
|
|
4525
|
-
const reqId =
|
|
5551
|
+
const reqId = decoded.reqId;
|
|
4526
5552
|
const pendingByReq = this.pendingProviderQueryBySuffixKey.get(suffixKey);
|
|
4527
5553
|
const pending = pendingByReq?.get(reqId);
|
|
4528
5554
|
if (!pending)
|
|
4529
5555
|
return true;
|
|
4530
5556
|
pendingByReq.delete(reqId);
|
|
4531
|
-
const count = data[37];
|
|
4532
5557
|
const now = Date.now();
|
|
4533
5558
|
this.touchProviderNamespace(suffixKey, now);
|
|
4534
|
-
const
|
|
5559
|
+
const candidates = this.toProviderCandidates(decoded.entries);
|
|
4535
5560
|
this.rememberProviderCandidates(this.getProviderNamespaceIdFromKey(channelKey, suffixKey), candidates, now + 60_000);
|
|
4536
5561
|
this.pruneProviderNamespaceIfEmpty(suffixKey);
|
|
4537
5562
|
if (candidates.length > 0) {
|
|
@@ -4541,11 +5566,11 @@ export class FanoutTree extends DirectStream {
|
|
|
4541
5566
|
return true;
|
|
4542
5567
|
}
|
|
4543
5568
|
if (kind === MSG_PROVIDER_NOTIFY) {
|
|
4544
|
-
|
|
5569
|
+
const decoded = this.codec.decodeProviderNotify(data);
|
|
5570
|
+
if (!decoded)
|
|
4545
5571
|
return false;
|
|
4546
|
-
const count = data[33];
|
|
4547
5572
|
const now = Date.now();
|
|
4548
|
-
const
|
|
5573
|
+
const providers = this.toProviderCandidates(decoded.entries);
|
|
4549
5574
|
this.rememberProviderCandidates(this.getProviderNamespaceIdFromKey(channelKey, suffixKey), providers, now + 60_000);
|
|
4550
5575
|
if (providers.length > 0) {
|
|
4551
5576
|
this.emitProviderUpdate(this.getProviderNamespaceIdFromKey(channelKey, suffixKey), providers);
|
|
@@ -4555,45 +5580,18 @@ export class FanoutTree extends DirectStream {
|
|
|
4555
5580
|
if (kind === MSG_TRACKER_REPLY) {
|
|
4556
5581
|
if (!ch)
|
|
4557
5582
|
return true;
|
|
4558
|
-
|
|
5583
|
+
const decoded = this.codec.decodeTrackerReply(data);
|
|
5584
|
+
if (!decoded)
|
|
4559
5585
|
return false;
|
|
4560
|
-
const reqId =
|
|
5586
|
+
const reqId = decoded.reqId;
|
|
4561
5587
|
const pending = ch.pendingTrackerQuery.get(reqId);
|
|
4562
|
-
if (
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
const count = data[37];
|
|
4566
|
-
let offset = 38;
|
|
5588
|
+
if (pending) {
|
|
5589
|
+
ch.pendingTrackerQuery.delete(reqId);
|
|
5590
|
+
}
|
|
4567
5591
|
const candidates = [];
|
|
4568
|
-
const
|
|
4569
|
-
for (let i = 0; i < max; i++) {
|
|
4570
|
-
if (offset + 1 > data.length)
|
|
4571
|
-
break;
|
|
4572
|
-
const hashLen = data[offset++];
|
|
4573
|
-
if (offset + hashLen > data.length)
|
|
4574
|
-
break;
|
|
4575
|
-
const hash = textDecoder.decode(data.subarray(offset, offset + hashLen));
|
|
4576
|
-
offset += hashLen;
|
|
4577
|
-
if (offset + 2 + 2 + 4 + 1 > data.length)
|
|
4578
|
-
break;
|
|
4579
|
-
const level = readU16BE(data, offset);
|
|
4580
|
-
offset += 2;
|
|
4581
|
-
const freeSlots = readU16BE(data, offset);
|
|
4582
|
-
offset += 2;
|
|
4583
|
-
const bidPerByte = readU32BE(data, offset);
|
|
4584
|
-
offset += 4;
|
|
4585
|
-
const addrCount = data[offset++];
|
|
5592
|
+
for (const entry of decoded.entries) {
|
|
4586
5593
|
const addrs = [];
|
|
4587
|
-
const
|
|
4588
|
-
for (let j = 0; j < addrMax; j++) {
|
|
4589
|
-
if (offset + 2 > data.length)
|
|
4590
|
-
break;
|
|
4591
|
-
const len = readU16BE(data, offset);
|
|
4592
|
-
offset += 2;
|
|
4593
|
-
if (offset + len > data.length)
|
|
4594
|
-
break;
|
|
4595
|
-
const bytes = data.subarray(offset, offset + len);
|
|
4596
|
-
offset += len;
|
|
5594
|
+
for (const bytes of entry.addrs) {
|
|
4597
5595
|
try {
|
|
4598
5596
|
addrs.push(multiaddr(bytes));
|
|
4599
5597
|
}
|
|
@@ -4601,10 +5599,56 @@ export class FanoutTree extends DirectStream {
|
|
|
4601
5599
|
// ignore invalid multiaddrs
|
|
4602
5600
|
}
|
|
4603
5601
|
}
|
|
4604
|
-
candidates.push({
|
|
4605
|
-
|
|
5602
|
+
candidates.push({
|
|
5603
|
+
hash: entry.hash,
|
|
5604
|
+
level: entry.level,
|
|
5605
|
+
freeSlots: entry.freeSlots,
|
|
5606
|
+
bidPerByte: entry.bidPerByte,
|
|
5607
|
+
addrs,
|
|
5608
|
+
});
|
|
5609
|
+
this.cacheKnownCandidateAddrs(ch, entry.hash, addrs);
|
|
4606
5610
|
}
|
|
4607
|
-
pending
|
|
5611
|
+
if (pending) {
|
|
5612
|
+
pending.resolve(candidates);
|
|
5613
|
+
}
|
|
5614
|
+
else if (candidates.length > 0) {
|
|
5615
|
+
// Late reply: the query already timed out, but the candidate
|
|
5616
|
+
// list is still fresh directory state. Under load the 1s query
|
|
5617
|
+
// timeout is routinely missed while trackers hold plenty of
|
|
5618
|
+
// capacity - dropping these replies starves joiners of
|
|
5619
|
+
// candidates they already paid a round trip for.
|
|
5620
|
+
const merged = new Map();
|
|
5621
|
+
for (const c of ch.cachedTrackerCandidates)
|
|
5622
|
+
merged.set(c.hash, c);
|
|
5623
|
+
for (const c of candidates)
|
|
5624
|
+
merged.set(c.hash, c);
|
|
5625
|
+
ch.cachedTrackerCandidates = [...merged.values()].slice(-256);
|
|
5626
|
+
}
|
|
5627
|
+
return true;
|
|
5628
|
+
}
|
|
5629
|
+
if (kind === MSG_PARENT_PROBE_REQ) {
|
|
5630
|
+
if (!ch || ch.closed)
|
|
5631
|
+
return true;
|
|
5632
|
+
const decoded = this.codec.decodeParentProbeReq(data);
|
|
5633
|
+
if (!decoded)
|
|
5634
|
+
return false;
|
|
5635
|
+
this.touchPeerHint(ch, fromHash);
|
|
5636
|
+
void this._sendControl(fromHash, this.encodeParentProbeReplyForChannel(ch, decoded.reqId, fromHash, decoded.minFreeSlots, decoded.reserveRootCapacity)).catch(() => { });
|
|
5637
|
+
return true;
|
|
5638
|
+
}
|
|
5639
|
+
if (kind === MSG_PARENT_PROBE_REPLY) {
|
|
5640
|
+
if (!ch || ch.closed)
|
|
5641
|
+
return true;
|
|
5642
|
+
const decoded = this.codec.decodeParentProbeReply(data, fromHash);
|
|
5643
|
+
if (!decoded)
|
|
5644
|
+
return false;
|
|
5645
|
+
const { reqId, ...reply } = decoded;
|
|
5646
|
+
const pending = ch.pendingParentProbe.get(reqId);
|
|
5647
|
+
if (!pending)
|
|
5648
|
+
return true;
|
|
5649
|
+
ch.pendingParentProbe.delete(reqId);
|
|
5650
|
+
pending.resolve(reply);
|
|
5651
|
+
this.touchPeerHint(ch, fromHash);
|
|
4608
5652
|
return true;
|
|
4609
5653
|
}
|
|
4610
5654
|
if (!ch)
|
|
@@ -4623,20 +5667,20 @@ export class FanoutTree extends DirectStream {
|
|
|
4623
5667
|
return true;
|
|
4624
5668
|
}
|
|
4625
5669
|
if (kind === MSG_ROUTE_QUERY) {
|
|
4626
|
-
|
|
5670
|
+
const decoded = this.codec.decodeRouteQuery(data);
|
|
5671
|
+
if (!decoded)
|
|
4627
5672
|
return false;
|
|
4628
|
-
const reqId =
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
void this._sendControl(fromHash, encodeRouteReply(ch.id.key, reqId)).catch(() => { });
|
|
5673
|
+
const reqId = decoded.reqId;
|
|
5674
|
+
if (decoded.targetHash == null) {
|
|
5675
|
+
void this._sendControl(fromHash, this.codec.encodeRouteReply(ch.id.key, reqId)).catch(() => { });
|
|
4632
5676
|
return true;
|
|
4633
5677
|
}
|
|
4634
|
-
const targetHash =
|
|
5678
|
+
const targetHash = decoded.targetHash;
|
|
4635
5679
|
const localRoute = targetHash === this.publicKeyHash && ch.routeFromRoot
|
|
4636
5680
|
? ch.routeFromRoot
|
|
4637
5681
|
: this.getCachedRoute(ch, targetHash);
|
|
4638
5682
|
if (this.isRouteValidForChannel(ch, localRoute)) {
|
|
4639
|
-
void this._sendControl(fromHash, encodeRouteReply(ch.id.key, reqId, localRoute)).catch(() => { });
|
|
5683
|
+
void this._sendControl(fromHash, this.codec.encodeRouteReply(ch.id.key, reqId, localRoute)).catch(() => { });
|
|
4640
5684
|
return true;
|
|
4641
5685
|
}
|
|
4642
5686
|
if (ch.isRoot) {
|
|
@@ -4646,7 +5690,7 @@ export class FanoutTree extends DirectStream {
|
|
|
4646
5690
|
if (rootRoute)
|
|
4647
5691
|
this.cacheRoute(ch, rootRoute);
|
|
4648
5692
|
if (rootRoute) {
|
|
4649
|
-
void this._sendControl(fromHash, encodeRouteReply(ch.id.key, reqId, rootRoute)).catch(() => { });
|
|
5693
|
+
void this._sendControl(fromHash, this.codec.encodeRouteReply(ch.id.key, reqId, rootRoute)).catch(() => { });
|
|
4650
5694
|
return true;
|
|
4651
5695
|
}
|
|
4652
5696
|
}
|
|
@@ -4659,16 +5703,19 @@ export class FanoutTree extends DirectStream {
|
|
|
4659
5703
|
}
|
|
4660
5704
|
// Cache miss on a parent->child (or root->child) query:
|
|
4661
5705
|
// recursively search subtree branches and reply with the first valid route.
|
|
4662
|
-
this.proxyRouteQuery(ch, fromHash, reqId, targetHash, [
|
|
5706
|
+
this.proxyRouteQuery(ch, fromHash, reqId, targetHash, [
|
|
5707
|
+
...ch.children.keys(),
|
|
5708
|
+
]);
|
|
4663
5709
|
return true;
|
|
4664
5710
|
}
|
|
4665
5711
|
if (kind === MSG_ROUTE_REPLY) {
|
|
4666
|
-
|
|
5712
|
+
const decoded = this.codec.decodeRouteReply(data);
|
|
5713
|
+
if (!decoded)
|
|
4667
5714
|
return false;
|
|
4668
|
-
const reqId =
|
|
4669
|
-
const
|
|
4670
|
-
|
|
4671
|
-
|
|
5715
|
+
const { reqId, route } = decoded;
|
|
5716
|
+
const parsedRoute = this.isRouteValidForChannel(ch, route)
|
|
5717
|
+
? route
|
|
5718
|
+
: undefined;
|
|
4672
5719
|
if (parsedRoute) {
|
|
4673
5720
|
this.cacheRoute(ch, parsedRoute);
|
|
4674
5721
|
}
|
|
@@ -4694,11 +5741,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4694
5741
|
return true;
|
|
4695
5742
|
}
|
|
4696
5743
|
if (kind === MSG_IHAVE) {
|
|
4697
|
-
|
|
5744
|
+
const decoded = this.codec.decodeIHave(data);
|
|
5745
|
+
if (!decoded)
|
|
4698
5746
|
return false;
|
|
4699
|
-
const haveFrom =
|
|
4700
|
-
const haveToExclusive = readU32BE(data, 37);
|
|
5747
|
+
const { haveFrom, haveToExclusive } = decoded;
|
|
4701
5748
|
const now = Date.now();
|
|
5749
|
+
if (ch.parent && fromHash === ch.parent) {
|
|
5750
|
+
ch.lastParentDataAt = now;
|
|
5751
|
+
ch.receivedAnyParentData = true;
|
|
5752
|
+
ch.parentRepairUnansweredStreak = 0;
|
|
5753
|
+
}
|
|
4702
5754
|
const prev = ch.haveByPeer.get(fromHash);
|
|
4703
5755
|
if (prev) {
|
|
4704
5756
|
prev.haveFrom = haveFrom;
|
|
@@ -4715,6 +5767,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4715
5767
|
});
|
|
4716
5768
|
}
|
|
4717
5769
|
this.touchPeerHint(ch, fromHash, now);
|
|
5770
|
+
if (ch.parent && fromHash === ch.parent && ch.maxDataAgeMs === 0) {
|
|
5771
|
+
// A watermark from our parent doubles as gap detection: if data
|
|
5772
|
+
// writes to us were lost (e.g. burst into a not-yet-writable
|
|
5773
|
+
// stream) we may have received nothing at all and would
|
|
5774
|
+
// otherwise never know data exists. Mark the advertised range
|
|
5775
|
+
// missing so the repair loop pulls it. Deadline-oriented live
|
|
5776
|
+
// channels (maxDataAgeMs > 0) skip this: catching up on stale
|
|
5777
|
+
// history would trade deadline latency for completeness.
|
|
5778
|
+
this.noteEnd(ch, fromHash, haveToExclusive);
|
|
5779
|
+
}
|
|
4718
5780
|
// Opportunistic reciprocity: if we still have room in our lazy mesh, add
|
|
4719
5781
|
// peers that are actively exchanging IHAVE summaries with us.
|
|
4720
5782
|
if (ch.neighborRepair &&
|
|
@@ -4728,11 +5790,21 @@ export class FanoutTree extends DirectStream {
|
|
|
4728
5790
|
return true;
|
|
4729
5791
|
}
|
|
4730
5792
|
if (kind === MSG_JOIN_REQ) {
|
|
4731
|
-
|
|
5793
|
+
const decoded = this.codec.decodeJoinReq(data);
|
|
5794
|
+
if (!decoded)
|
|
4732
5795
|
return false;
|
|
4733
|
-
const reqId =
|
|
4734
|
-
const bidPerByte = readU32BE(data, 37);
|
|
5796
|
+
const { reqId, bidPerByte, parentUpgradeReservationToken } = decoded;
|
|
4735
5797
|
this.pruneDisconnectedChildren(ch);
|
|
5798
|
+
const hasParentUpgradeReservation = ch.isRoot && parentUpgradeReservationToken > 0;
|
|
5799
|
+
const consumedParentUpgradeReservation = hasParentUpgradeReservation
|
|
5800
|
+
? this.consumeParentUpgradeReservation(ch, fromHash, parentUpgradeReservationToken)
|
|
5801
|
+
: false;
|
|
5802
|
+
if (hasParentUpgradeReservation && !consumedParentUpgradeReservation) {
|
|
5803
|
+
ch.metrics.parentUpgradeRootReservationRejected += 1;
|
|
5804
|
+
void this.sendJoinReject(ch, fromHash, reqId, JOIN_REJECT_NO_CAPACITY).catch(() => { });
|
|
5805
|
+
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
5806
|
+
return true;
|
|
5807
|
+
}
|
|
4736
5808
|
// Only accept if we're already attached.
|
|
4737
5809
|
if (!ch.isRoot && !ch.parent) {
|
|
4738
5810
|
void this.sendJoinReject(ch, fromHash, reqId, JOIN_REJECT_NOT_ATTACHED).catch(() => { });
|
|
@@ -4755,8 +5827,16 @@ export class FanoutTree extends DirectStream {
|
|
|
4755
5827
|
void this.sendJoinReject(ch, fromHash, reqId, JOIN_REJECT_NO_CAPACITY).catch(() => { });
|
|
4756
5828
|
return true;
|
|
4757
5829
|
}
|
|
4758
|
-
if (!ch.children.has(fromHash) &&
|
|
4759
|
-
|
|
5830
|
+
if (!ch.children.has(fromHash) &&
|
|
5831
|
+
(consumedParentUpgradeReservation
|
|
5832
|
+
? ch.children.size >= ch.effectiveMaxChildren
|
|
5833
|
+
: ch.children.size +
|
|
5834
|
+
this.parentUpgradeReservationCount(ch, fromHash) >=
|
|
5835
|
+
ch.effectiveMaxChildren)) {
|
|
5836
|
+
const blockedByReservedRootCapacity = ch.isRoot &&
|
|
5837
|
+
!consumedParentUpgradeReservation &&
|
|
5838
|
+
ch.children.size < ch.effectiveMaxChildren;
|
|
5839
|
+
if (!ch.allowKick || blockedByReservedRootCapacity) {
|
|
4760
5840
|
void this.sendJoinReject(ch, fromHash, reqId, JOIN_REJECT_NO_CAPACITY).catch(() => { });
|
|
4761
5841
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
4762
5842
|
return true;
|
|
@@ -4774,56 +5854,70 @@ export class FanoutTree extends DirectStream {
|
|
|
4774
5854
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
4775
5855
|
return true;
|
|
4776
5856
|
}
|
|
4777
|
-
|
|
4778
|
-
ch.dataWriteFailStreakByChild.delete(worstChild);
|
|
4779
|
-
void this._sendControl(worstChild, encodeKick(ch.id.key));
|
|
5857
|
+
void this.kickChildHashes(ch, [worstChild]).catch(() => { });
|
|
4780
5858
|
}
|
|
4781
5859
|
ch.children.set(fromHash, { bidPerByte });
|
|
5860
|
+
if (ch.isRoot && consumedParentUpgradeReservation) {
|
|
5861
|
+
ch.parentUpgradeTrackerNoCapacityUntil = Math.max(ch.parentUpgradeTrackerNoCapacityUntil, Date.now() + 2_000);
|
|
5862
|
+
}
|
|
4782
5863
|
this.touchPeerHint(ch, fromHash);
|
|
4783
|
-
|
|
5864
|
+
const joinAccept = this.codec.encodeJoinAccept(ch.id.key, reqId, ch.level, ch.routeFromRoot, this.getHaveRange(ch));
|
|
5865
|
+
void (async () => {
|
|
5866
|
+
await this._sendControl(fromHash, joinAccept);
|
|
5867
|
+
if (ch.endSeqExclusive > 0) {
|
|
5868
|
+
await this._sendControl(fromHash, this.codec.encodeEnd(ch.id.key, ch.endSeqExclusive));
|
|
5869
|
+
}
|
|
5870
|
+
})().catch(() => { });
|
|
4784
5871
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
4785
5872
|
return true;
|
|
4786
5873
|
}
|
|
4787
5874
|
if (kind === MSG_JOIN_ACCEPT || kind === MSG_JOIN_REJECT) {
|
|
4788
|
-
|
|
5875
|
+
const reqId = this.codec.decodeJoinResponseReqId(data);
|
|
5876
|
+
if (reqId == null)
|
|
4789
5877
|
return false;
|
|
4790
|
-
const reqId = readU32BE(data, 33);
|
|
4791
5878
|
const pending = ch.pendingJoin.get(reqId);
|
|
4792
5879
|
if (!pending)
|
|
4793
5880
|
return true;
|
|
4794
5881
|
ch.pendingJoin.delete(reqId);
|
|
4795
5882
|
if (kind === MSG_JOIN_ACCEPT) {
|
|
4796
|
-
|
|
5883
|
+
const decoded = this.codec.decodeJoinAccept(data);
|
|
5884
|
+
if (!decoded)
|
|
4797
5885
|
return false;
|
|
4798
|
-
const parentLevel =
|
|
4799
|
-
const routeCount = Math.min(255, data[39]);
|
|
4800
|
-
let offset = 40;
|
|
4801
|
-
const parentRouteFromRoot = [];
|
|
4802
|
-
const max = Math.min(routeCount, MAX_ROUTE_HOPS);
|
|
4803
|
-
for (let i = 0; i < max; i++) {
|
|
4804
|
-
if (offset + 1 > data.length)
|
|
4805
|
-
break;
|
|
4806
|
-
const len = data[offset++];
|
|
4807
|
-
if (len === 0)
|
|
4808
|
-
break;
|
|
4809
|
-
if (offset + len > data.length)
|
|
4810
|
-
break;
|
|
4811
|
-
parentRouteFromRoot.push(textDecoder.decode(data.subarray(offset, offset + len)));
|
|
4812
|
-
offset += len;
|
|
4813
|
-
}
|
|
5886
|
+
const { parentLevel, parentRouteFromRoot } = decoded;
|
|
4814
5887
|
const hasValidParentRoute = parentRouteFromRoot.length > 0 &&
|
|
4815
5888
|
parentRouteFromRoot[0] === ch.id.root &&
|
|
4816
5889
|
parentRouteFromRoot[parentRouteFromRoot.length - 1] === fromHash;
|
|
4817
5890
|
// Defensive: a JOIN_ACCEPT without a rooted route token (unless the parent is the
|
|
4818
5891
|
// actual root) can create stable disconnected components. Treat it as a reject.
|
|
4819
5892
|
if (fromHash !== ch.id.root && !hasValidParentRoute) {
|
|
4820
|
-
pending.resolve({
|
|
5893
|
+
pending.resolve({
|
|
5894
|
+
ok: false,
|
|
5895
|
+
rejectReason: JOIN_REJECT_NOT_ATTACHED,
|
|
5896
|
+
});
|
|
5897
|
+
return true;
|
|
5898
|
+
}
|
|
5899
|
+
const acceptedParentRoute = hasValidParentRoute || fromHash === ch.id.root
|
|
5900
|
+
? fromHash === ch.id.root && !hasValidParentRoute
|
|
5901
|
+
? [ch.id.root]
|
|
5902
|
+
: parentRouteFromRoot
|
|
5903
|
+
: undefined;
|
|
5904
|
+
if (pending.shadowAttach) {
|
|
5905
|
+
this.touchPeerHint(ch, fromHash);
|
|
5906
|
+
pending.resolve({
|
|
5907
|
+
ok: true,
|
|
5908
|
+
parentLevel,
|
|
5909
|
+
parentRouteFromRoot: acceptedParentRoute,
|
|
5910
|
+
});
|
|
4821
5911
|
return true;
|
|
4822
5912
|
}
|
|
5913
|
+
const attachedAt = Date.now();
|
|
4823
5914
|
ch.parent = fromHash;
|
|
4824
5915
|
ch.level = parentLevel + 1;
|
|
4825
5916
|
ch.joinedAtLeastOnce = true;
|
|
4826
|
-
ch.lastParentDataAt =
|
|
5917
|
+
ch.lastParentDataAt = attachedAt;
|
|
5918
|
+
ch.lastParentUpgradeActivityAt = attachedAt;
|
|
5919
|
+
ch.parentRepairUnansweredStreak = 0;
|
|
5920
|
+
this.resetParentDataLatency(ch);
|
|
4827
5921
|
// Treat JOIN_ACCEPT as parent liveness for stale re-parenting:
|
|
4828
5922
|
// if callers enable `staleAfterMs`, we should be able to detach even
|
|
4829
5923
|
// before the first data message arrives (for example, during churn
|
|
@@ -4837,58 +5931,56 @@ export class FanoutTree extends DirectStream {
|
|
|
4837
5931
|
// Minimal fallback: parent is the root.
|
|
4838
5932
|
ch.routeFromRoot = [ch.id.root, this.publicKeyHash];
|
|
4839
5933
|
}
|
|
5934
|
+
if (ch.repairEnabled &&
|
|
5935
|
+
ch.maxDataAgeMs === 0 &&
|
|
5936
|
+
decoded.haveRange != null) {
|
|
5937
|
+
// Optional appendix: the parent's current have-range. Lets a
|
|
5938
|
+
// freshly attached child learn immediately which sequences
|
|
5939
|
+
// already exist, so lost deliveries surface as missing seqs
|
|
5940
|
+
// (and repair) instead of silent absence.
|
|
5941
|
+
const { haveFrom, haveToExclusive } = decoded.haveRange;
|
|
5942
|
+
if (haveToExclusive > haveFrom) {
|
|
5943
|
+
// Mark only; the repair loop's own cadence picks the gaps
|
|
5944
|
+
// up, giving in-flight live deliveries time to land first
|
|
5945
|
+
// (an immediate pull here races them into duplicates).
|
|
5946
|
+
this.noteEnd(ch, fromHash, haveToExclusive);
|
|
5947
|
+
}
|
|
5948
|
+
}
|
|
4840
5949
|
this.touchPeerHint(ch, fromHash);
|
|
4841
|
-
pending.resolve({
|
|
5950
|
+
pending.resolve({
|
|
5951
|
+
ok: true,
|
|
5952
|
+
parentLevel,
|
|
5953
|
+
parentRouteFromRoot: acceptedParentRoute,
|
|
5954
|
+
});
|
|
4842
5955
|
this.dispatchEvent(new CustomEvent("fanout:joined", {
|
|
4843
|
-
detail: {
|
|
5956
|
+
detail: {
|
|
5957
|
+
topic: ch.id.topic,
|
|
5958
|
+
root: ch.id.root,
|
|
5959
|
+
parent: fromHash,
|
|
5960
|
+
},
|
|
4844
5961
|
}));
|
|
4845
5962
|
void this.announceToTrackers(ch, this.closeController.signal).catch(() => { });
|
|
4846
5963
|
}
|
|
4847
5964
|
else {
|
|
4848
|
-
|
|
5965
|
+
const decoded = this.codec.decodeJoinReject(data);
|
|
5966
|
+
if (!decoded)
|
|
4849
5967
|
return false;
|
|
4850
|
-
const reason =
|
|
5968
|
+
const reason = decoded.reason;
|
|
4851
5969
|
const redirects = [];
|
|
4852
|
-
|
|
4853
|
-
const
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
if (offset + 1 > data.length)
|
|
4858
|
-
break;
|
|
4859
|
-
const hashLen = data[offset++];
|
|
4860
|
-
if (hashLen === 0)
|
|
4861
|
-
break;
|
|
4862
|
-
if (offset + hashLen > data.length)
|
|
4863
|
-
break;
|
|
4864
|
-
const hash = textDecoder.decode(data.subarray(offset, offset + hashLen));
|
|
4865
|
-
offset += hashLen;
|
|
4866
|
-
if (offset + 1 > data.length)
|
|
4867
|
-
break;
|
|
4868
|
-
const addrCount = Math.min(255, data[offset++]);
|
|
4869
|
-
const addrs = [];
|
|
4870
|
-
const addrMax = Math.min(addrCount, JOIN_REJECT_REDIRECT_ADDR_MAX);
|
|
4871
|
-
for (let j = 0; j < addrMax; j++) {
|
|
4872
|
-
if (offset + 2 > data.length)
|
|
4873
|
-
break;
|
|
4874
|
-
const len = readU16BE(data, offset);
|
|
4875
|
-
offset += 2;
|
|
4876
|
-
if (offset + len > data.length)
|
|
4877
|
-
break;
|
|
4878
|
-
const bytes = data.subarray(offset, offset + len);
|
|
4879
|
-
offset += len;
|
|
4880
|
-
try {
|
|
4881
|
-
addrs.push(multiaddr(bytes));
|
|
4882
|
-
}
|
|
4883
|
-
catch {
|
|
4884
|
-
// ignore invalid multiaddrs
|
|
4885
|
-
}
|
|
5970
|
+
for (const redirect of decoded.redirects) {
|
|
5971
|
+
const addrs = [];
|
|
5972
|
+
for (const bytes of redirect.addrs) {
|
|
5973
|
+
try {
|
|
5974
|
+
addrs.push(multiaddr(bytes));
|
|
4886
5975
|
}
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
this.cacheKnownCandidateAddrs(ch, hash, addrs);
|
|
5976
|
+
catch {
|
|
5977
|
+
// ignore invalid multiaddrs
|
|
4890
5978
|
}
|
|
4891
5979
|
}
|
|
5980
|
+
if (redirect.hash && addrs.length > 0) {
|
|
5981
|
+
redirects.push({ hash: redirect.hash, addrs });
|
|
5982
|
+
this.cacheKnownCandidateAddrs(ch, redirect.hash, addrs);
|
|
5983
|
+
}
|
|
4892
5984
|
}
|
|
4893
5985
|
pending.resolve({ ok: false, rejectReason: reason, redirects });
|
|
4894
5986
|
}
|
|
@@ -4954,11 +6046,10 @@ export class FanoutTree extends DirectStream {
|
|
|
4954
6046
|
const isFromChild = ch.children.has(fromHash);
|
|
4955
6047
|
if (!isFromParent && !isFromChild)
|
|
4956
6048
|
return true;
|
|
4957
|
-
|
|
6049
|
+
const decoded = this.codec.decodeUnicastAck(data);
|
|
6050
|
+
if (!decoded)
|
|
4958
6051
|
return false;
|
|
4959
|
-
const ackToken =
|
|
4960
|
-
const routeCount = Math.min(255, data[41]);
|
|
4961
|
-
const decoded = decodeRoute(data, 42, routeCount);
|
|
6052
|
+
const ackToken = decoded.ackToken;
|
|
4962
6053
|
const route = decoded.route;
|
|
4963
6054
|
const target = route.length > 0 ? route[route.length - 1] : "";
|
|
4964
6055
|
const origin = message.header.signatures?.publicKeys?.[0]?.hashcode?.() ?? fromHash;
|
|
@@ -5038,33 +6129,13 @@ export class FanoutTree extends DirectStream {
|
|
|
5038
6129
|
return true;
|
|
5039
6130
|
}
|
|
5040
6131
|
}
|
|
5041
|
-
|
|
6132
|
+
const decoded = this.codec.decodeUnicast(data);
|
|
6133
|
+
if (!decoded)
|
|
5042
6134
|
return false;
|
|
5043
|
-
const
|
|
5044
|
-
let offset = 34;
|
|
5045
|
-
let ackToken;
|
|
5046
|
-
if (flags & UNICAST_FLAG_ACK) {
|
|
5047
|
-
if (data.length < offset + 8 + 1)
|
|
5048
|
-
return false;
|
|
5049
|
-
ackToken = readU64BE(data, offset);
|
|
5050
|
-
offset += 8;
|
|
5051
|
-
}
|
|
5052
|
-
const routeCount = Math.min(255, data[offset]);
|
|
5053
|
-
offset += 1;
|
|
5054
|
-
const decoded = decodeRoute(data, offset, routeCount);
|
|
6135
|
+
const ackToken = decoded.ackToken;
|
|
5055
6136
|
const route = decoded.route;
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
if (ackToken != null) {
|
|
5059
|
-
if (data.length < offset + 1)
|
|
5060
|
-
return false;
|
|
5061
|
-
const replyCount = Math.min(255, data[offset]);
|
|
5062
|
-
offset += 1;
|
|
5063
|
-
const decodedReply = decodeRoute(data, offset, replyCount);
|
|
5064
|
-
replyRoute = decodedReply.route;
|
|
5065
|
-
offset = decodedReply.offset;
|
|
5066
|
-
}
|
|
5067
|
-
const payload = data.subarray(offset);
|
|
6137
|
+
const replyRoute = decoded.replyRoute;
|
|
6138
|
+
const payload = data.subarray(decoded.payloadOffset);
|
|
5068
6139
|
const target = route.length > 0 ? route[route.length - 1] : "";
|
|
5069
6140
|
const origin = message.header.signatures?.publicKeys?.[0]?.hashcode?.() ?? fromHash;
|
|
5070
6141
|
this.touchPeerHint(ch, fromHash);
|
|
@@ -5094,7 +6165,7 @@ export class FanoutTree extends DirectStream {
|
|
|
5094
6165
|
if (canAck) {
|
|
5095
6166
|
const nextHop = replyRoute[1];
|
|
5096
6167
|
if (nextHop && ch.children.has(nextHop)) {
|
|
5097
|
-
void this._sendControl(nextHop, encodeUnicastAck(ch.id.key, ackToken, replyRoute)).catch(() => { });
|
|
6168
|
+
void this._sendControl(nextHop, this.codec.encodeUnicastAck(ch.id.key, ackToken, replyRoute)).catch(() => { });
|
|
5098
6169
|
}
|
|
5099
6170
|
}
|
|
5100
6171
|
}
|
|
@@ -5106,6 +6177,11 @@ export class FanoutTree extends DirectStream {
|
|
|
5106
6177
|
const stream = this.peers.get(nextHop);
|
|
5107
6178
|
if (!stream)
|
|
5108
6179
|
return true;
|
|
6180
|
+
// A validated source route is passing through: learn it (and the
|
|
6181
|
+
// reply route) so subsequent unicasts skip route discovery.
|
|
6182
|
+
this.cacheRoute(ch, route);
|
|
6183
|
+
if (replyRoute)
|
|
6184
|
+
this.cacheRoute(ch, replyRoute);
|
|
5109
6185
|
void stream
|
|
5110
6186
|
.waitForWrite(message.bytes(), Number(message.header.priority ?? CONTROL_PRIORITY), this.closeController.signal)
|
|
5111
6187
|
.catch(() => { });
|
|
@@ -5137,7 +6213,7 @@ export class FanoutTree extends DirectStream {
|
|
|
5137
6213
|
replyRoute[0] === ch.id.root &&
|
|
5138
6214
|
replyRoute[replyRoute.length - 1] === origin;
|
|
5139
6215
|
if (canAck) {
|
|
5140
|
-
void this._sendControl(ch.parent, encodeUnicastAck(ch.id.key, ackToken, replyRoute)).catch(() => { });
|
|
6216
|
+
void this._sendControl(ch.parent, this.codec.encodeUnicastAck(ch.id.key, ackToken, replyRoute)).catch(() => { });
|
|
5141
6217
|
}
|
|
5142
6218
|
}
|
|
5143
6219
|
return true;
|
|
@@ -5148,6 +6224,9 @@ export class FanoutTree extends DirectStream {
|
|
|
5148
6224
|
const stream = this.peers.get(nextHop);
|
|
5149
6225
|
if (!stream)
|
|
5150
6226
|
return true;
|
|
6227
|
+
this.cacheRoute(ch, route);
|
|
6228
|
+
if (replyRoute)
|
|
6229
|
+
this.cacheRoute(ch, replyRoute);
|
|
5151
6230
|
void stream
|
|
5152
6231
|
.waitForWrite(message.bytes(), Number(message.header.priority ?? CONTROL_PRIORITY), this.closeController.signal)
|
|
5153
6232
|
.catch(() => { });
|
|
@@ -5171,14 +6250,18 @@ export class FanoutTree extends DirectStream {
|
|
|
5171
6250
|
ch.routeFromRoot = undefined;
|
|
5172
6251
|
ch.routeByPeer.clear();
|
|
5173
6252
|
ch.lastParentDataAt = 0;
|
|
6253
|
+
ch.lastParentUpgradeActivityAt = 0;
|
|
5174
6254
|
ch.receivedAnyParentData = false;
|
|
5175
6255
|
ch.pendingJoin.clear();
|
|
6256
|
+
ch.pendingParentProbe.clear();
|
|
6257
|
+
ch.parentShadow = undefined;
|
|
6258
|
+
void Promise.all(this.clearParentUpgradeGrace(ch, true, true)).catch(() => { });
|
|
6259
|
+
if (this.parentUpgradeShadowInFlightSuffixKey === ch.id.suffixKey) {
|
|
6260
|
+
this.parentUpgradeShadowInFlightSuffixKey = undefined;
|
|
6261
|
+
}
|
|
5176
6262
|
ch.pendingRouteQuery.clear();
|
|
5177
6263
|
this.abortPendingUnicastAcks(ch, new AbortError("fanout channel kicked"));
|
|
5178
|
-
|
|
5179
|
-
clearTimeout(pending.timer);
|
|
5180
|
-
}
|
|
5181
|
-
ch.pendingRouteProxy.clear();
|
|
6264
|
+
this.clearRouteProxies(ch);
|
|
5182
6265
|
void this.kickChildren(ch).catch(() => { });
|
|
5183
6266
|
this.dispatchEvent(new CustomEvent("fanout:kicked", {
|
|
5184
6267
|
detail: { topic: ch.id.topic, root: ch.id.root, from: fromHash },
|
|
@@ -5186,27 +6269,30 @@ export class FanoutTree extends DirectStream {
|
|
|
5186
6269
|
return true;
|
|
5187
6270
|
}
|
|
5188
6271
|
if (kind === MSG_END) {
|
|
5189
|
-
|
|
6272
|
+
const lastSeqExclusive = this.codec.decodeEnd(data);
|
|
6273
|
+
if (lastSeqExclusive == null)
|
|
5190
6274
|
return false;
|
|
5191
|
-
|
|
6275
|
+
if (ch.parent && fromHash === ch.parent) {
|
|
6276
|
+
ch.lastParentDataAt = Date.now();
|
|
6277
|
+
ch.receivedAnyParentData = true;
|
|
6278
|
+
ch.parentRepairUnansweredStreak = 0;
|
|
6279
|
+
}
|
|
5192
6280
|
ch.endSeqExclusive = Math.max(ch.endSeqExclusive, lastSeqExclusive);
|
|
5193
6281
|
this.touchPeerHint(ch, fromHash);
|
|
5194
6282
|
this.noteEnd(ch, fromHash, lastSeqExclusive);
|
|
5195
6283
|
if (ch.children.size > 0) {
|
|
5196
|
-
void this._sendControlMany([...ch.children.keys()], encodeEnd(ch.id.key, lastSeqExclusive));
|
|
6284
|
+
void this._sendControlMany([...ch.children.keys()], this.codec.encodeEnd(ch.id.key, lastSeqExclusive));
|
|
5197
6285
|
}
|
|
5198
6286
|
void this.tickRepair(ch).catch(() => { });
|
|
5199
6287
|
return true;
|
|
5200
6288
|
}
|
|
5201
6289
|
if (kind === MSG_REPAIR_REQ) {
|
|
5202
|
-
|
|
6290
|
+
const seqs = this.codec.decodeRepairSeqs(data);
|
|
6291
|
+
if (!seqs)
|
|
5203
6292
|
return false;
|
|
5204
6293
|
if (!ch.children.has(fromHash))
|
|
5205
6294
|
return true;
|
|
5206
|
-
const
|
|
5207
|
-
const max = Math.min(count, Math.floor((data.length - 38) / 4));
|
|
5208
|
-
for (let i = 0; i < max; i++) {
|
|
5209
|
-
const seq = readU32BE(data, 38 + i * 4);
|
|
6295
|
+
for (const seq of seqs) {
|
|
5210
6296
|
const cached = this.getCached(ch, seq);
|
|
5211
6297
|
if (!cached) {
|
|
5212
6298
|
ch.metrics.cacheMissesServed += 1;
|
|
@@ -5218,13 +6304,11 @@ export class FanoutTree extends DirectStream {
|
|
|
5218
6304
|
return true;
|
|
5219
6305
|
}
|
|
5220
6306
|
if (kind === MSG_FETCH_REQ) {
|
|
5221
|
-
|
|
6307
|
+
const seqs = this.codec.decodeRepairSeqs(data);
|
|
6308
|
+
if (!seqs)
|
|
5222
6309
|
return false;
|
|
5223
6310
|
this.touchPeerHint(ch, fromHash);
|
|
5224
|
-
const
|
|
5225
|
-
const max = Math.min(count, Math.floor((data.length - 38) / 4));
|
|
5226
|
-
for (let i = 0; i < max; i++) {
|
|
5227
|
-
const seq = readU32BE(data, 38 + i * 4);
|
|
6311
|
+
for (const seq of seqs) {
|
|
5228
6312
|
const cached = this.getCached(ch, seq);
|
|
5229
6313
|
if (!cached) {
|
|
5230
6314
|
ch.metrics.cacheMissesServed += 1;
|
|
@@ -5248,10 +6332,100 @@ export class FanoutTree extends DirectStream {
|
|
|
5248
6332
|
return false;
|
|
5249
6333
|
const seq = readU32BE(id, 4);
|
|
5250
6334
|
if (ch.parent && fromHash === ch.parent) {
|
|
5251
|
-
|
|
6335
|
+
const receivedAt = Date.now();
|
|
6336
|
+
ch.lastParentDataAt = receivedAt;
|
|
6337
|
+
ch.lastParentUpgradeActivityAt = receivedAt;
|
|
5252
6338
|
ch.receivedAnyParentData = true;
|
|
6339
|
+
ch.parentRepairUnansweredStreak = 0;
|
|
6340
|
+
}
|
|
6341
|
+
const hadCachedBeforeData = this.getCached(ch, seq) != null;
|
|
6342
|
+
if (ch.parent && fromHash === ch.parent && !hadCachedBeforeData) {
|
|
6343
|
+
this.noteParentDataLatency(ch, message.header.timestamp);
|
|
5253
6344
|
}
|
|
5254
6345
|
ch.maxSeqSeen = Math.max(ch.maxSeqSeen, seq);
|
|
6346
|
+
if (ch.parentUpgradeRetryAfterSeq >= 0 &&
|
|
6347
|
+
seq > ch.parentUpgradeRetryAfterSeq) {
|
|
6348
|
+
ch.parentUpgradeRetryAfterSeq = -1;
|
|
6349
|
+
}
|
|
6350
|
+
const parentShadow = ch.parentShadow;
|
|
6351
|
+
if (parentShadow != null) {
|
|
6352
|
+
const now = Date.now();
|
|
6353
|
+
const pruneShadowSeqMap = (map) => {
|
|
6354
|
+
while (map.size > 512) {
|
|
6355
|
+
const oldest = map.keys().next().value;
|
|
6356
|
+
if (oldest == null)
|
|
6357
|
+
break;
|
|
6358
|
+
map.delete(oldest);
|
|
6359
|
+
}
|
|
6360
|
+
};
|
|
6361
|
+
const pruneShadowSeqSet = (set) => {
|
|
6362
|
+
while (set.size > 512) {
|
|
6363
|
+
const oldest = set.keys().next().value;
|
|
6364
|
+
if (oldest == null)
|
|
6365
|
+
break;
|
|
6366
|
+
set.delete(oldest);
|
|
6367
|
+
}
|
|
6368
|
+
};
|
|
6369
|
+
const rememberShadowArrival = (map) => {
|
|
6370
|
+
if (map.has(seq))
|
|
6371
|
+
return map.get(seq);
|
|
6372
|
+
map.set(seq, now);
|
|
6373
|
+
pruneShadowSeqMap(map);
|
|
6374
|
+
return now;
|
|
6375
|
+
};
|
|
6376
|
+
const recordCandidateLead = (candidateAt, parentAt) => {
|
|
6377
|
+
if (candidateAt == null ||
|
|
6378
|
+
parentAt == null ||
|
|
6379
|
+
parentShadow.liveComparedSeqs.has(seq)) {
|
|
6380
|
+
return;
|
|
6381
|
+
}
|
|
6382
|
+
parentShadow.liveComparedSeqs.add(seq);
|
|
6383
|
+
pruneShadowSeqSet(parentShadow.liveComparedSeqs);
|
|
6384
|
+
if (candidateAt < parentAt) {
|
|
6385
|
+
parentShadow.liveCandidateLeadSamples += 1;
|
|
6386
|
+
parentShadow.liveCandidateLeadMsTotal += parentAt - candidateAt;
|
|
6387
|
+
}
|
|
6388
|
+
};
|
|
6389
|
+
if (parentShadow.hash === fromHash) {
|
|
6390
|
+
const candidateAt = rememberShadowArrival(parentShadow.liveCandidateFirstAtBySeq);
|
|
6391
|
+
recordCandidateLead(candidateAt, parentShadow.liveParentFirstAtBySeq.get(seq));
|
|
6392
|
+
if (seq > parentShadow.liveMaxSeqSeen) {
|
|
6393
|
+
parentShadow.liveMaxSeqSeen = seq;
|
|
6394
|
+
parentShadow.liveDataMessages += 1;
|
|
6395
|
+
}
|
|
6396
|
+
if (!hadCachedBeforeData) {
|
|
6397
|
+
parentShadow.liveFirstDataMessages += 1;
|
|
6398
|
+
}
|
|
6399
|
+
parentShadow.liveLastDataAt = now;
|
|
6400
|
+
}
|
|
6401
|
+
else if (ch.parent != null && fromHash === ch.parent) {
|
|
6402
|
+
const parentAt = rememberShadowArrival(parentShadow.liveParentFirstAtBySeq);
|
|
6403
|
+
recordCandidateLead(parentShadow.liveCandidateFirstAtBySeq.get(seq), parentAt);
|
|
6404
|
+
if (!hadCachedBeforeData) {
|
|
6405
|
+
parentShadow.liveParentFirstDataMessages += 1;
|
|
6406
|
+
}
|
|
6407
|
+
}
|
|
6408
|
+
}
|
|
6409
|
+
const parentUpgradeGrace = ch.parentUpgradeGrace;
|
|
6410
|
+
if (parentUpgradeGrace != null && !hadCachedBeforeData) {
|
|
6411
|
+
if (fromHash === parentUpgradeGrace.candidateParent) {
|
|
6412
|
+
parentUpgradeGrace.candidateFirstMessages += 1;
|
|
6413
|
+
}
|
|
6414
|
+
else if (fromHash === parentUpgradeGrace.previousParent) {
|
|
6415
|
+
parentUpgradeGrace.previousFirstMessages += 1;
|
|
6416
|
+
}
|
|
6417
|
+
if (parentUpgradeGrace.previousFirstMessages >
|
|
6418
|
+
parentUpgradeGrace.maxPreviousFirstMessages) {
|
|
6419
|
+
this.rollbackParentUpgradeGrace(ch);
|
|
6420
|
+
}
|
|
6421
|
+
else if (parentUpgradeGrace.candidateFirstMessages >=
|
|
6422
|
+
parentUpgradeGrace.minCandidateFirstMessages &&
|
|
6423
|
+
parentUpgradeGrace.candidateFirstMessages >=
|
|
6424
|
+
parentUpgradeGrace.previousFirstMessages +
|
|
6425
|
+
parentUpgradeGrace.minCandidateAdvantageMessages) {
|
|
6426
|
+
this.commitParentUpgradeGrace(ch);
|
|
6427
|
+
}
|
|
6428
|
+
}
|
|
5255
6429
|
this.noteReceivedSeq(ch, fromHash, seq);
|
|
5256
6430
|
const payload = data.subarray(1);
|
|
5257
6431
|
ch.metrics.dataReceives += 1;
|