@peerbit/shared-log 13.2.11 → 13.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/receive-prune.js +31 -29
- package/dist/benchmark/receive-prune.js.map +1 -1
- package/dist/src/checked-prune.d.ts +43 -10
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +257 -27
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +26 -0
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +99 -0
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +71 -14
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3492 -1493
- package/dist/src/index.js.map +1 -1
- package/dist/src/ranges.d.ts +12 -1
- package/dist/src/ranges.d.ts.map +1 -1
- package/dist/src/ranges.js +472 -110
- package/dist/src/ranges.js.map +1 -1
- package/dist/src/sync/index.d.ts +2 -0
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +96 -20
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +1616 -428
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +176 -1
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +2919 -525
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +13 -13
- package/src/checked-prune.ts +338 -29
- package/src/exchange-heads.ts +52 -0
- package/src/index.ts +6370 -2792
- package/src/ranges.ts +648 -127
- package/src/sync/index.ts +2 -1
- package/src/sync/rateless-iblt.ts +2036 -498
- package/src/sync/simple.ts +3779 -634
package/src/sync/simple.ts
CHANGED
|
@@ -21,8 +21,8 @@ import {
|
|
|
21
21
|
import { TransportMessage } from "../message.js";
|
|
22
22
|
import type { EntryReplicated } from "../ranges.js";
|
|
23
23
|
import type {
|
|
24
|
-
HashSymbolResolver,
|
|
25
24
|
HashSymbolHashListResolver,
|
|
25
|
+
HashSymbolResolver,
|
|
26
26
|
RawExchangeHeadsSender,
|
|
27
27
|
RepairSession,
|
|
28
28
|
RepairSessionMode,
|
|
@@ -135,15 +135,26 @@ const getHashesFromSymbols = async (
|
|
|
135
135
|
coordinateToHash: Cache<string>,
|
|
136
136
|
resolveHashesForSymbols?: HashSymbolResolver,
|
|
137
137
|
resolveHashListForSymbols?: HashSymbolHashListResolver,
|
|
138
|
+
maxHashes = 10_000,
|
|
138
139
|
): Promise<Set<string> | string[]> => {
|
|
139
140
|
let queries: IntegerCompare[] = [];
|
|
140
141
|
let batchSize = 128; // TODO arg
|
|
141
142
|
let results = new Set<string>();
|
|
142
143
|
let missingSymbols: bigint[] = [];
|
|
144
|
+
const addHash = (hash: string) => {
|
|
145
|
+
if (results.has(hash)) {
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
if (results.size >= maxHashes) {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
results.add(hash);
|
|
152
|
+
return true;
|
|
153
|
+
};
|
|
143
154
|
const addMissingUnlessCached = (symbol: bigint) => {
|
|
144
155
|
const fromCache = coordinateToHash.get(symbol);
|
|
145
156
|
if (fromCache) {
|
|
146
|
-
|
|
157
|
+
addHash(fromCache);
|
|
147
158
|
return;
|
|
148
159
|
}
|
|
149
160
|
missingSymbols.push(symbol);
|
|
@@ -159,8 +170,13 @@ const getHashesFromSymbols = async (
|
|
|
159
170
|
queries = [];
|
|
160
171
|
|
|
161
172
|
for (const entry of entries) {
|
|
162
|
-
|
|
173
|
+
if (!addHash(entry.value.hash)) {
|
|
174
|
+
break;
|
|
175
|
+
}
|
|
163
176
|
coordinateToHash.add(entry.value.hashNumber, entry.value.hash);
|
|
177
|
+
if (results.size >= maxHashes) {
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
164
180
|
}
|
|
165
181
|
}
|
|
166
182
|
};
|
|
@@ -168,15 +184,31 @@ const getHashesFromSymbols = async (
|
|
|
168
184
|
if (resolveHashListForSymbols) {
|
|
169
185
|
const resolvedHashes = await resolveHashListForSymbols(symbols);
|
|
170
186
|
if (resolvedHashes) {
|
|
171
|
-
const resolvedHashList =
|
|
172
|
-
|
|
173
|
-
|
|
187
|
+
const resolvedHashList: string[] = [];
|
|
188
|
+
const iterator = resolvedHashes[Symbol.iterator]();
|
|
189
|
+
let exhausted = false;
|
|
190
|
+
try {
|
|
191
|
+
while (resolvedHashList.length < maxHashes) {
|
|
192
|
+
const next = iterator.next();
|
|
193
|
+
if (next.done) {
|
|
194
|
+
exhausted = true;
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
resolvedHashList.push(next.value);
|
|
198
|
+
}
|
|
199
|
+
} finally {
|
|
200
|
+
if (!exhausted) {
|
|
201
|
+
iterator.return?.();
|
|
202
|
+
}
|
|
203
|
+
}
|
|
174
204
|
let mergedHashes: Set<string> | undefined;
|
|
175
205
|
for (const symbol of symbols) {
|
|
176
206
|
const fromCache = coordinateToHash.get(symbol);
|
|
177
207
|
if (fromCache) {
|
|
178
208
|
mergedHashes ??= new Set(resolvedHashList);
|
|
179
|
-
mergedHashes.
|
|
209
|
+
if (mergedHashes.size < maxHashes) {
|
|
210
|
+
mergedHashes.add(fromCache);
|
|
211
|
+
}
|
|
180
212
|
}
|
|
181
213
|
}
|
|
182
214
|
return mergedHashes ?? resolvedHashList;
|
|
@@ -186,7 +218,11 @@ const getHashesFromSymbols = async (
|
|
|
186
218
|
if (resolveHashesForSymbols) {
|
|
187
219
|
const resolved = await resolveHashesForSymbols(symbols);
|
|
188
220
|
if (resolved) {
|
|
221
|
+
let resolvedItemCount = 0;
|
|
189
222
|
for (const symbol of symbols) {
|
|
223
|
+
if (resolvedItemCount >= maxHashes) {
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
190
226
|
const hashes = resolved.get(symbol);
|
|
191
227
|
if (!hashes) {
|
|
192
228
|
addMissingUnlessCached(symbol);
|
|
@@ -194,14 +230,37 @@ const getHashesFromSymbols = async (
|
|
|
194
230
|
}
|
|
195
231
|
let singleHash: string | undefined;
|
|
196
232
|
let count = 0;
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
233
|
+
let truncated = false;
|
|
234
|
+
const iterator = hashes[Symbol.iterator]();
|
|
235
|
+
let exhausted = false;
|
|
236
|
+
try {
|
|
237
|
+
while (resolvedItemCount < maxHashes) {
|
|
238
|
+
const next = iterator.next();
|
|
239
|
+
if (next.done) {
|
|
240
|
+
exhausted = true;
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
resolvedItemCount += 1;
|
|
244
|
+
if (!addHash(next.value)) {
|
|
245
|
+
truncated = true;
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
singleHash = next.value;
|
|
249
|
+
count += 1;
|
|
250
|
+
if (results.size >= maxHashes) {
|
|
251
|
+
truncated = true;
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
} finally {
|
|
256
|
+
if (!exhausted) {
|
|
257
|
+
truncated = true;
|
|
258
|
+
iterator.return?.();
|
|
259
|
+
}
|
|
201
260
|
}
|
|
202
261
|
if (count === 0) {
|
|
203
262
|
addMissingUnlessCached(symbol);
|
|
204
|
-
} else if (count === 1) {
|
|
263
|
+
} else if (count === 1 && !truncated) {
|
|
205
264
|
coordinateToHash.add(symbol, singleHash!);
|
|
206
265
|
}
|
|
207
266
|
}
|
|
@@ -217,6 +276,9 @@ const getHashesFromSymbols = async (
|
|
|
217
276
|
}
|
|
218
277
|
|
|
219
278
|
for (const symbol of missingSymbols) {
|
|
279
|
+
if (results.size >= maxHashes) {
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
220
282
|
const matchQuery = new IntegerCompare({
|
|
221
283
|
key: "hashNumber",
|
|
222
284
|
compare: Compare.Equal,
|
|
@@ -241,6 +303,10 @@ const SESSION_POLL_INTERVAL_MS = 100;
|
|
|
241
303
|
const DEFAULT_MAX_HASHES_PER_MESSAGE = 1_024;
|
|
242
304
|
const DEFAULT_MAX_COORDINATES_PER_MESSAGE = 1_024;
|
|
243
305
|
const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
|
|
306
|
+
export const MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS = 1_024;
|
|
307
|
+
export const MAX_SIMPLE_COORDINATE_RESPONSE_HASHES = 10_000;
|
|
308
|
+
export const MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_PER_PEER = 4;
|
|
309
|
+
export const MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_GLOBAL = 32;
|
|
244
310
|
// Keep convergence sync above the default/background lane. Dropping it to the
|
|
245
311
|
// background priority lets repair traffic starve behind foreground work.
|
|
246
312
|
export const SYNC_MESSAGE_PRIORITY = CONVERGENCE_MESSAGE_PRIORITY;
|
|
@@ -250,6 +316,153 @@ export const SYNC_MESSAGE_PRIORITY = CONVERGENCE_MESSAGE_PRIORITY;
|
|
|
250
316
|
const SIMPLE_SYNC_RETRY_AFTER_MS = 10_000;
|
|
251
317
|
const EXCHANGE_HEAD_RESPONSE_DEDUPE_TTL_MS = SIMPLE_SYNC_RETRY_AFTER_MS - 1_000;
|
|
252
318
|
const RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS = 30_000;
|
|
319
|
+
const PENDING_MAYBE_SYNC_RESPONSE_TTL_MS = 30_000;
|
|
320
|
+
// An incoming maybe-sync claim keeps one retry candidate in both
|
|
321
|
+
// syncInFlightQueue and syncInFlightQueueInverted. Bound associations rather
|
|
322
|
+
// than only unique keys: otherwise many peers can grow the claimant array for
|
|
323
|
+
// the same keys without changing syncInFlightQueue.size.
|
|
324
|
+
// The per-peer allowance matches one full 10,000-hash response-authorization
|
|
325
|
+
// window; the global allowance lets four peers make full bounded progress.
|
|
326
|
+
export const MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER = 10_000;
|
|
327
|
+
export const MAX_PENDING_SIMPLE_SYNC_KEYS_GLOBAL = 40_000;
|
|
328
|
+
// Storage presence resolution is not universally abortable. Keep the number of
|
|
329
|
+
// live resolver calls small even when requests use only one key each.
|
|
330
|
+
export const MAX_PENDING_SIMPLE_SYNC_LOOKUPS_PER_PEER = 4;
|
|
331
|
+
export const MAX_PENDING_SIMPLE_SYNC_LOOKUPS_GLOBAL = 32;
|
|
332
|
+
// Retry scanning can touch persistent indexes. Bound each pass so a full
|
|
333
|
+
// adversarial queue cannot force 40,000 lookups in one event-loop turn.
|
|
334
|
+
export const MAX_SIMPLE_SYNC_RETRY_KEYS_PER_TICK = 4_096;
|
|
335
|
+
// Late coordinate-to-hash cache fills are discovered incrementally. Keep this
|
|
336
|
+
// independent of retained queue size so an empty or repeated request cannot
|
|
337
|
+
// force an O(global pending keys) reverse-alias rebuild.
|
|
338
|
+
const MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE = 128;
|
|
339
|
+
const QUEUED_SYNC_ALIAS_REFRESH_PENDING = Symbol(
|
|
340
|
+
"queued-sync-alias-refresh-pending",
|
|
341
|
+
);
|
|
342
|
+
// This is an absolute first-seen lifetime. Repeated claims and additional peers
|
|
343
|
+
// deliberately do not slide the deadline.
|
|
344
|
+
export const PENDING_SIMPLE_SYNC_KEY_TTL_MS = 60_000;
|
|
345
|
+
// Bound retained request/response associations globally. Ten thousand hashes
|
|
346
|
+
// covers several full default-size request batches while keeping adversarial or
|
|
347
|
+
// abandoned requests to a small, predictable amount of heap.
|
|
348
|
+
const MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES = 10_000;
|
|
349
|
+
export const MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_PER_PEER = 4;
|
|
350
|
+
export const MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_GLOBAL = 32;
|
|
351
|
+
const MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITER_BYPASSES = 32;
|
|
352
|
+
const MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITERS = 10_000;
|
|
353
|
+
|
|
354
|
+
type PendingSyncAdmissionReservation = {
|
|
355
|
+
peer: string;
|
|
356
|
+
remaining: number;
|
|
357
|
+
active: boolean;
|
|
358
|
+
released: boolean;
|
|
359
|
+
expiresAt: number;
|
|
360
|
+
identities: Set<SyncableKey>;
|
|
361
|
+
retainedSettled: number;
|
|
362
|
+
};
|
|
363
|
+
|
|
364
|
+
type PendingSyncExpiryNode =
|
|
365
|
+
| {
|
|
366
|
+
kind: "key";
|
|
367
|
+
key: SyncableKey;
|
|
368
|
+
expiresAt: number;
|
|
369
|
+
heapIndex: number;
|
|
370
|
+
}
|
|
371
|
+
| {
|
|
372
|
+
kind: "admission";
|
|
373
|
+
reservation: PendingSyncAdmissionReservation;
|
|
374
|
+
expiresAt: number;
|
|
375
|
+
heapIndex: number;
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
type PendingMaybeSyncResponse = {
|
|
379
|
+
hashes: Set<string>;
|
|
380
|
+
target: string;
|
|
381
|
+
targetLifecycle: SyncDispatchTargetLifecycle;
|
|
382
|
+
expiresAt: number;
|
|
383
|
+
heapIndex: number;
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
type PendingMaybeSyncResponseAuthorizationEvent =
|
|
387
|
+
| "delivered"
|
|
388
|
+
| "fulfilled"
|
|
389
|
+
| "released";
|
|
390
|
+
|
|
391
|
+
type PendingMaybeSyncResponseAuthorization = {
|
|
392
|
+
batch: PendingMaybeSyncResponse;
|
|
393
|
+
hash: string;
|
|
394
|
+
waiters: Set<(event: PendingMaybeSyncResponseAuthorizationEvent) => void>;
|
|
395
|
+
requestDelivered?: boolean;
|
|
396
|
+
deliveryInFlight?: boolean;
|
|
397
|
+
settled?: "fulfilled" | "released";
|
|
398
|
+
active?: boolean;
|
|
399
|
+
};
|
|
400
|
+
|
|
401
|
+
type PendingMaybeSyncResponseReservation = {
|
|
402
|
+
release: () => void;
|
|
403
|
+
beginDelivery: () => void;
|
|
404
|
+
finishDelivery: () => void;
|
|
405
|
+
markDelivered: () => void;
|
|
406
|
+
newlyAuthorizedByTarget: Map<string, string[]>;
|
|
407
|
+
retained: () => boolean;
|
|
408
|
+
signal: AbortSignal;
|
|
409
|
+
};
|
|
410
|
+
|
|
411
|
+
type PendingMaybeSyncResponseReservationAttempt =
|
|
412
|
+
| {
|
|
413
|
+
kind: "reserved";
|
|
414
|
+
reservation: PendingMaybeSyncResponseReservation;
|
|
415
|
+
conflicts: PendingMaybeSyncResponseAuthorization[];
|
|
416
|
+
}
|
|
417
|
+
| {
|
|
418
|
+
kind: "capacity";
|
|
419
|
+
required: number;
|
|
420
|
+
}
|
|
421
|
+
| {
|
|
422
|
+
kind: "inactive";
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
type PendingMaybeSyncResponseWaiter = {
|
|
426
|
+
required: number;
|
|
427
|
+
associations: number;
|
|
428
|
+
order: number;
|
|
429
|
+
bypasses: number;
|
|
430
|
+
heapIndex: number;
|
|
431
|
+
fitHeapIndex: number;
|
|
432
|
+
wake: () => void;
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
export type AuthorizedMaybeSyncResponseLease = {
|
|
436
|
+
hashes: string[];
|
|
437
|
+
signal: AbortSignal;
|
|
438
|
+
release: (options?: { fulfilled?: boolean }) => void;
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
type SyncDispatchLifecycle = {
|
|
442
|
+
ownershipLifecycleController: AbortController;
|
|
443
|
+
callerSignal?: AbortSignal;
|
|
444
|
+
controller: AbortController;
|
|
445
|
+
targets: Map<string, SyncDispatchTargetLifecycle>;
|
|
446
|
+
retainedWork: number;
|
|
447
|
+
onOwnerOrCallerAbort: () => void;
|
|
448
|
+
dispatchFinished: boolean;
|
|
449
|
+
disposed: boolean;
|
|
450
|
+
abortAllOnTargetDisconnect: boolean;
|
|
451
|
+
};
|
|
452
|
+
|
|
453
|
+
type SyncDispatchTargetEpoch = {
|
|
454
|
+
id: number;
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
type SyncDispatchTargetLifecycle = {
|
|
458
|
+
lifecycle: SyncDispatchLifecycle;
|
|
459
|
+
target: string;
|
|
460
|
+
epoch: SyncDispatchTargetEpoch;
|
|
461
|
+
controller: AbortController;
|
|
462
|
+
batches: Set<PendingMaybeSyncResponse>;
|
|
463
|
+
responseLeases: number;
|
|
464
|
+
activeWaiters: number;
|
|
465
|
+
};
|
|
253
466
|
|
|
254
467
|
const createDeferred = <T>() => {
|
|
255
468
|
let resolve!: (value: T | PromiseLike<T>) => void;
|
|
@@ -266,6 +479,7 @@ type RepairSessionTargetState = {
|
|
|
266
479
|
requestedCount: number;
|
|
267
480
|
requestedTotalCount: number;
|
|
268
481
|
attempts: number;
|
|
482
|
+
targetEpoch: SyncDispatchTargetEpoch;
|
|
269
483
|
};
|
|
270
484
|
|
|
271
485
|
type RepairSessionState = {
|
|
@@ -287,9 +501,50 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
287
501
|
// map of hash to public keys that we can ask for entries
|
|
288
502
|
syncInFlightQueue: Map<SyncableKey, PublicSignKey[]>;
|
|
289
503
|
syncInFlightQueueInverted: Map<string, Set<SyncableKey>>;
|
|
504
|
+
private syncInFlightQueueExpiresAt: Map<SyncableKey, number>;
|
|
505
|
+
private syncInFlightQueueExpiryTimer?: ReturnType<typeof setTimeout>;
|
|
506
|
+
private pendingSyncExpiryHeap: PendingSyncExpiryNode[];
|
|
507
|
+
private pendingSyncKeyExpiryNodes: Map<SyncableKey, PendingSyncExpiryNode>;
|
|
508
|
+
private pendingSyncAdmissionExpiryNodes: Map<
|
|
509
|
+
PendingSyncAdmissionReservation,
|
|
510
|
+
PendingSyncExpiryNode
|
|
511
|
+
>;
|
|
512
|
+
private syncInFlightRetryIterator?: IterableIterator<
|
|
513
|
+
[SyncableKey, PublicSignKey[]]
|
|
514
|
+
>;
|
|
515
|
+
private syncInFlightRetryRemaining = 0;
|
|
516
|
+
private syncInFlightQueueClaimants: Map<SyncableKey, Set<string>>;
|
|
517
|
+
private syncInFlightQueueClaimantIndexes: Map<
|
|
518
|
+
SyncableKey,
|
|
519
|
+
Map<string, number>
|
|
520
|
+
>;
|
|
521
|
+
private syncInFlightQueueRoundRobinCursor: Map<SyncableKey, number>;
|
|
522
|
+
private syncInFlightQueuedCoordinates: Set<bigint>;
|
|
523
|
+
private syncInFlightQueuedHashByCoordinate: Map<bigint, string>;
|
|
524
|
+
private syncInFlightQueuedCoordinatesByHash: Map<string, Set<bigint>>;
|
|
525
|
+
private syncInFlightQueuedCoordinateRefreshIterator?: IterableIterator<bigint>;
|
|
526
|
+
private pendingSyncClaimCount: number;
|
|
527
|
+
private pendingSyncAdmissionCount: number;
|
|
528
|
+
private pendingSyncActiveAdmissionReservations: number;
|
|
529
|
+
private pendingSyncAdmissionCountByPeer: Map<string, number>;
|
|
530
|
+
private pendingSyncAdmissionIdentitiesByPeer: Map<string, Set<SyncableKey>>;
|
|
531
|
+
private pendingSyncAdmissionReservations: Set<PendingSyncAdmissionReservation>;
|
|
532
|
+
private pendingSyncAdmissionReservationsByPeer: Map<
|
|
533
|
+
string,
|
|
534
|
+
Set<PendingSyncAdmissionReservation>
|
|
535
|
+
>;
|
|
536
|
+
private pendingSyncAdmissionReservationsByIdentity: Map<
|
|
537
|
+
SyncableKey,
|
|
538
|
+
Set<PendingSyncAdmissionReservation>
|
|
539
|
+
>;
|
|
540
|
+
private pendingCoordinateLookupCount: number;
|
|
541
|
+
private pendingCoordinateLookupCountByPeer: Map<string, number>;
|
|
542
|
+
private pendingCoordinateResponseCount: number;
|
|
543
|
+
private pendingCoordinateResponseCountByPeer: Map<string, number>;
|
|
290
544
|
|
|
291
545
|
// map of hash to public keys that we have asked for entries
|
|
292
546
|
syncInFlight!: Map<string, Map<SyncableKey, { timestamp: number }>>;
|
|
547
|
+
private syncInFlightTargetsByKey: Map<SyncableKey, Set<string>>;
|
|
293
548
|
|
|
294
549
|
rpc: RPC<TransportMessage, TransportMessage>;
|
|
295
550
|
log: Log<any>;
|
|
@@ -305,6 +560,27 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
305
560
|
) => boolean;
|
|
306
561
|
private sendRawExchangeHeads?: RawExchangeHeadsSender;
|
|
307
562
|
private recentlySentExchangeHeads: Map<string, Map<string, number>>;
|
|
563
|
+
private pendingMaybeSyncResponses: Map<
|
|
564
|
+
string,
|
|
565
|
+
Map<string, PendingMaybeSyncResponseAuthorization>
|
|
566
|
+
>;
|
|
567
|
+
private pendingMaybeSyncResponseCount: number;
|
|
568
|
+
private pendingMaybeSyncResponseWaiters: Set<PendingMaybeSyncResponseWaiter>;
|
|
569
|
+
private pendingMaybeSyncResponseWaiterHeap: PendingMaybeSyncResponseWaiter[];
|
|
570
|
+
private pendingMaybeSyncResponseWaiterFitHeap: PendingMaybeSyncResponseWaiter[];
|
|
571
|
+
private pendingMaybeSyncResponseWaiterOrder: number;
|
|
572
|
+
private pendingMaybeSyncResponseWaiterAssociationCount: number;
|
|
573
|
+
private pendingMaybeSyncResponseWakeScheduled: boolean;
|
|
574
|
+
private pendingMaybeSyncResponseConflictWaiterCount: number;
|
|
575
|
+
private pendingMaybeSyncResponseBatches: Set<PendingMaybeSyncResponse>;
|
|
576
|
+
private pendingMaybeSyncResponseExpiryTimer?: ReturnType<typeof setTimeout>;
|
|
577
|
+
private pendingMaybeSyncResponseExpiryHeap: PendingMaybeSyncResponse[];
|
|
578
|
+
private activeMaybeSyncResponseCount: number;
|
|
579
|
+
private activeMaybeSyncResponseCountByPeer: Map<string, number>;
|
|
580
|
+
private syncDispatchLifecycleController: AbortController;
|
|
581
|
+
private syncDispatchTargetEpochCounter: number;
|
|
582
|
+
private syncDispatchTargetEpochs: Map<string, SyncDispatchTargetEpoch>;
|
|
583
|
+
private syncDispatchTargets: Map<string, Set<SyncDispatchTargetLifecycle>>;
|
|
308
584
|
private repairSessionCounter: number;
|
|
309
585
|
private repairSessions: Map<string, RepairSessionState>;
|
|
310
586
|
|
|
@@ -330,7 +606,30 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
330
606
|
}) {
|
|
331
607
|
this.syncInFlightQueue = new Map();
|
|
332
608
|
this.syncInFlightQueueInverted = new Map();
|
|
609
|
+
this.syncInFlightQueueExpiresAt = new Map();
|
|
610
|
+
this.pendingSyncExpiryHeap = [];
|
|
611
|
+
this.pendingSyncKeyExpiryNodes = new Map();
|
|
612
|
+
this.pendingSyncAdmissionExpiryNodes = new Map();
|
|
613
|
+
this.syncInFlightQueueClaimants = new Map();
|
|
614
|
+
this.syncInFlightQueueClaimantIndexes = new Map();
|
|
615
|
+
this.syncInFlightQueueRoundRobinCursor = new Map();
|
|
616
|
+
this.syncInFlightQueuedCoordinates = new Set();
|
|
617
|
+
this.syncInFlightQueuedHashByCoordinate = new Map();
|
|
618
|
+
this.syncInFlightQueuedCoordinatesByHash = new Map();
|
|
619
|
+
this.pendingSyncClaimCount = 0;
|
|
620
|
+
this.pendingSyncAdmissionCount = 0;
|
|
621
|
+
this.pendingSyncActiveAdmissionReservations = 0;
|
|
622
|
+
this.pendingSyncAdmissionCountByPeer = new Map();
|
|
623
|
+
this.pendingSyncAdmissionIdentitiesByPeer = new Map();
|
|
624
|
+
this.pendingSyncAdmissionReservations = new Set();
|
|
625
|
+
this.pendingSyncAdmissionReservationsByPeer = new Map();
|
|
626
|
+
this.pendingSyncAdmissionReservationsByIdentity = new Map();
|
|
627
|
+
this.pendingCoordinateLookupCount = 0;
|
|
628
|
+
this.pendingCoordinateLookupCountByPeer = new Map();
|
|
629
|
+
this.pendingCoordinateResponseCount = 0;
|
|
630
|
+
this.pendingCoordinateResponseCountByPeer = new Map();
|
|
333
631
|
this.syncInFlight = new Map();
|
|
632
|
+
this.syncInFlightTargetsByKey = new Map();
|
|
334
633
|
this.rpc = properties.rpc;
|
|
335
634
|
this.log = properties.log;
|
|
336
635
|
this.entryIndex = properties.entryIndex;
|
|
@@ -341,6 +640,23 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
341
640
|
this.isEntryRecentlyKnownByPeer = properties.isEntryRecentlyKnownByPeer;
|
|
342
641
|
this.sendRawExchangeHeads = properties.sendRawExchangeHeads;
|
|
343
642
|
this.recentlySentExchangeHeads = new Map();
|
|
643
|
+
this.pendingMaybeSyncResponses = new Map();
|
|
644
|
+
this.pendingMaybeSyncResponseCount = 0;
|
|
645
|
+
this.pendingMaybeSyncResponseWaiters = new Set();
|
|
646
|
+
this.pendingMaybeSyncResponseWaiterHeap = [];
|
|
647
|
+
this.pendingMaybeSyncResponseWaiterFitHeap = [];
|
|
648
|
+
this.pendingMaybeSyncResponseWaiterOrder = 0;
|
|
649
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount = 0;
|
|
650
|
+
this.pendingMaybeSyncResponseWakeScheduled = false;
|
|
651
|
+
this.pendingMaybeSyncResponseConflictWaiterCount = 0;
|
|
652
|
+
this.pendingMaybeSyncResponseBatches = new Set();
|
|
653
|
+
this.pendingMaybeSyncResponseExpiryHeap = [];
|
|
654
|
+
this.activeMaybeSyncResponseCount = 0;
|
|
655
|
+
this.activeMaybeSyncResponseCountByPeer = new Map();
|
|
656
|
+
this.syncDispatchLifecycleController = new AbortController();
|
|
657
|
+
this.syncDispatchTargetEpochCounter = 0;
|
|
658
|
+
this.syncDispatchTargetEpochs = new Map();
|
|
659
|
+
this.syncDispatchTargets = new Map();
|
|
344
660
|
this.repairSessionCounter = 0;
|
|
345
661
|
this.repairSessions = new Map();
|
|
346
662
|
}
|
|
@@ -389,21 +705,24 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
389
705
|
private get maxHashesPerMessage() {
|
|
390
706
|
const value = this.syncOptions?.maxSimpleHashesPerMessage;
|
|
391
707
|
return value && Number.isFinite(value) && value > 0
|
|
392
|
-
? Math.floor(value)
|
|
708
|
+
? Math.max(1, Math.floor(value))
|
|
393
709
|
: DEFAULT_MAX_HASHES_PER_MESSAGE;
|
|
394
710
|
}
|
|
395
711
|
|
|
396
712
|
private get maxCoordinatesPerMessage() {
|
|
397
713
|
const value = this.syncOptions?.maxSimpleCoordinatesPerMessage;
|
|
398
714
|
return value && Number.isFinite(value) && value > 0
|
|
399
|
-
? Math.
|
|
715
|
+
? Math.min(
|
|
716
|
+
MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS,
|
|
717
|
+
Math.max(1, Math.floor(value)),
|
|
718
|
+
)
|
|
400
719
|
: DEFAULT_MAX_COORDINATES_PER_MESSAGE;
|
|
401
720
|
}
|
|
402
721
|
|
|
403
722
|
private get maxConvergentTrackedHashes() {
|
|
404
723
|
const value = this.syncOptions?.maxConvergentTrackedHashes;
|
|
405
724
|
return value && Number.isFinite(value) && value > 0
|
|
406
|
-
? Math.floor(value)
|
|
725
|
+
? Math.max(1, Math.floor(value))
|
|
407
726
|
: DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
|
|
408
727
|
}
|
|
409
728
|
|
|
@@ -459,493 +778,2850 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
459
778
|
return out;
|
|
460
779
|
}
|
|
461
780
|
|
|
462
|
-
private
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
781
|
+
private forgetRecentlySentExchangeHeads(
|
|
782
|
+
hashes: Iterable<string>,
|
|
783
|
+
peer: PublicSignKey,
|
|
784
|
+
): void {
|
|
785
|
+
const recentlySent = this.recentlySentExchangeHeads.get(peer.hashcode());
|
|
786
|
+
if (!recentlySent) {
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
for (const hash of hashes) {
|
|
790
|
+
recentlySent.delete(hash);
|
|
791
|
+
}
|
|
792
|
+
if (recentlySent.size === 0) {
|
|
793
|
+
this.recentlySentExchangeHeads.delete(peer.hashcode());
|
|
467
794
|
}
|
|
468
|
-
return true;
|
|
469
795
|
}
|
|
470
796
|
|
|
471
|
-
private
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
797
|
+
private getOrCreateSyncDispatchTargetEpoch(
|
|
798
|
+
target: string,
|
|
799
|
+
): SyncDispatchTargetEpoch {
|
|
800
|
+
let epoch = this.syncDispatchTargetEpochs.get(target);
|
|
801
|
+
if (!epoch) {
|
|
802
|
+
epoch = { id: ++this.syncDispatchTargetEpochCounter };
|
|
803
|
+
this.syncDispatchTargetEpochs.set(target, epoch);
|
|
804
|
+
}
|
|
805
|
+
return epoch;
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
private captureSyncDispatchLifecycle(
|
|
809
|
+
targets: string[],
|
|
810
|
+
callerSignal?: AbortSignal,
|
|
811
|
+
options?: {
|
|
812
|
+
abortAllOnTargetDisconnect?: boolean;
|
|
813
|
+
ownershipLifecycleController?: AbortController;
|
|
814
|
+
targetEpochs?: Map<string, SyncDispatchTargetEpoch>;
|
|
815
|
+
createTargetEpochs?: boolean;
|
|
816
|
+
},
|
|
817
|
+
): SyncDispatchLifecycle {
|
|
818
|
+
const ownershipLifecycleController =
|
|
819
|
+
options?.ownershipLifecycleController ??
|
|
820
|
+
this.syncDispatchLifecycleController;
|
|
821
|
+
const lifecycle = {
|
|
822
|
+
ownershipLifecycleController,
|
|
823
|
+
callerSignal,
|
|
824
|
+
controller: new AbortController(),
|
|
825
|
+
targets: new Map<string, SyncDispatchTargetLifecycle>(),
|
|
826
|
+
retainedWork: 0,
|
|
827
|
+
onOwnerOrCallerAbort: () => {
|
|
828
|
+
const reason =
|
|
829
|
+
callerSignal?.aborted === true
|
|
830
|
+
? callerSignal.reason
|
|
831
|
+
: ownershipLifecycleController.signal.reason;
|
|
832
|
+
this.abortSyncDispatchLifecycle(lifecycle, reason);
|
|
833
|
+
},
|
|
834
|
+
dispatchFinished: false,
|
|
835
|
+
disposed: false,
|
|
836
|
+
abortAllOnTargetDisconnect: options?.abortAllOnTargetDisconnect === true,
|
|
837
|
+
} satisfies SyncDispatchLifecycle;
|
|
838
|
+
|
|
839
|
+
for (const target of [...new Set(targets)]) {
|
|
840
|
+
const expectedEpoch = options?.targetEpochs?.get(target);
|
|
841
|
+
const currentEpoch = this.syncDispatchTargetEpochs.get(target);
|
|
842
|
+
const epoch =
|
|
843
|
+
expectedEpoch ??
|
|
844
|
+
currentEpoch ??
|
|
845
|
+
(options?.createTargetEpochs === false
|
|
846
|
+
? undefined
|
|
847
|
+
: this.getOrCreateSyncDispatchTargetEpoch(target));
|
|
848
|
+
if (!epoch) {
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
const targetLifecycle: SyncDispatchTargetLifecycle = {
|
|
852
|
+
lifecycle,
|
|
480
853
|
target,
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
854
|
+
epoch,
|
|
855
|
+
controller: new AbortController(),
|
|
856
|
+
batches: new Set(),
|
|
857
|
+
responseLeases: 0,
|
|
858
|
+
activeWaiters: 0,
|
|
859
|
+
};
|
|
860
|
+
lifecycle.targets.set(target, targetLifecycle);
|
|
861
|
+
let activeForTarget = this.syncDispatchTargets.get(target);
|
|
862
|
+
if (!activeForTarget) {
|
|
863
|
+
activeForTarget = new Set();
|
|
864
|
+
this.syncDispatchTargets.set(target, activeForTarget);
|
|
865
|
+
}
|
|
866
|
+
activeForTarget.add(targetLifecycle);
|
|
867
|
+
if (this.syncDispatchTargetEpochs.get(target) !== epoch) {
|
|
868
|
+
this.abortSyncDispatchTarget(
|
|
869
|
+
targetLifecycle,
|
|
870
|
+
new Error("sync target lifecycle is stale"),
|
|
871
|
+
);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
ownershipLifecycleController.signal.addEventListener(
|
|
876
|
+
"abort",
|
|
877
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
878
|
+
{ once: true },
|
|
879
|
+
);
|
|
880
|
+
if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
|
|
881
|
+
callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
|
|
882
|
+
once: true,
|
|
489
883
|
});
|
|
490
884
|
}
|
|
491
|
-
|
|
885
|
+
if (
|
|
886
|
+
this.closed === true ||
|
|
887
|
+
ownershipLifecycleController !== this.syncDispatchLifecycleController ||
|
|
888
|
+
ownershipLifecycleController.signal.aborted ||
|
|
889
|
+
callerSignal?.aborted
|
|
890
|
+
) {
|
|
891
|
+
lifecycle.onOwnerOrCallerAbort();
|
|
892
|
+
}
|
|
893
|
+
return lifecycle;
|
|
492
894
|
}
|
|
493
895
|
|
|
494
|
-
private
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
896
|
+
private abortSyncDispatchTarget(
|
|
897
|
+
targetLifecycle: SyncDispatchTargetLifecycle,
|
|
898
|
+
reason?: unknown,
|
|
899
|
+
): void {
|
|
900
|
+
if (!targetLifecycle.controller.signal.aborted) {
|
|
901
|
+
targetLifecycle.controller.abort(reason);
|
|
498
902
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
if (session.timer) {
|
|
502
|
-
clearTimeout(session.timer);
|
|
903
|
+
for (const batch of [...targetLifecycle.batches]) {
|
|
904
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
503
905
|
}
|
|
504
|
-
|
|
906
|
+
this.maybeDisposeSyncDispatchLifecycle(targetLifecycle.lifecycle);
|
|
505
907
|
}
|
|
506
908
|
|
|
507
|
-
private
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
909
|
+
private abortSyncDispatchLifecycle(
|
|
910
|
+
lifecycle: SyncDispatchLifecycle,
|
|
911
|
+
reason?: unknown,
|
|
912
|
+
): void {
|
|
913
|
+
if (!lifecycle.controller.signal.aborted) {
|
|
914
|
+
lifecycle.controller.abort(reason);
|
|
511
915
|
}
|
|
512
|
-
for (const
|
|
513
|
-
|
|
514
|
-
typeof this.log.hasMany === "function"
|
|
515
|
-
? await this.log.hasMany(state.unresolved)
|
|
516
|
-
: await this.getExistingRepairHashes(state.unresolved);
|
|
517
|
-
for (const hash of resolved) {
|
|
518
|
-
state.unresolved.delete(hash);
|
|
519
|
-
}
|
|
916
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
917
|
+
this.abortSyncDispatchTarget(targetLifecycle, reason);
|
|
520
918
|
}
|
|
919
|
+
this.maybeDisposeSyncDispatchLifecycle(lifecycle);
|
|
521
920
|
}
|
|
522
921
|
|
|
523
|
-
private
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
const resolved = new Set<string>();
|
|
527
|
-
for (const hash of hashes) {
|
|
528
|
-
if (await this.log.has(hash)) {
|
|
529
|
-
resolved.add(hash);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
return resolved;
|
|
922
|
+
private finishSyncDispatchLifecycle(lifecycle: SyncDispatchLifecycle): void {
|
|
923
|
+
lifecycle.dispatchFinished = true;
|
|
924
|
+
this.maybeDisposeSyncDispatchLifecycle(lifecycle);
|
|
533
925
|
}
|
|
534
926
|
|
|
535
|
-
private
|
|
536
|
-
|
|
927
|
+
private maybeDisposeSyncDispatchLifecycle(
|
|
928
|
+
lifecycle: SyncDispatchLifecycle,
|
|
929
|
+
): void {
|
|
930
|
+
if (
|
|
931
|
+
lifecycle.disposed ||
|
|
932
|
+
!lifecycle.dispatchFinished ||
|
|
933
|
+
lifecycle.retainedWork > 0
|
|
934
|
+
) {
|
|
537
935
|
return;
|
|
538
936
|
}
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
937
|
+
lifecycle.disposed = true;
|
|
938
|
+
lifecycle.ownershipLifecycleController.signal.removeEventListener(
|
|
939
|
+
"abort",
|
|
940
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
941
|
+
);
|
|
942
|
+
if (
|
|
943
|
+
lifecycle.callerSignal &&
|
|
944
|
+
lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
|
|
945
|
+
) {
|
|
946
|
+
lifecycle.callerSignal.removeEventListener(
|
|
947
|
+
"abort",
|
|
948
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
949
|
+
);
|
|
950
|
+
}
|
|
951
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
952
|
+
const activeForTarget = this.syncDispatchTargets.get(
|
|
953
|
+
targetLifecycle.target,
|
|
954
|
+
);
|
|
955
|
+
activeForTarget?.delete(targetLifecycle);
|
|
956
|
+
if (activeForTarget?.size === 0) {
|
|
957
|
+
this.syncDispatchTargets.delete(targetLifecycle.target);
|
|
547
958
|
}
|
|
548
959
|
}
|
|
549
960
|
}
|
|
550
961
|
|
|
551
|
-
private
|
|
552
|
-
|
|
553
|
-
|
|
962
|
+
private isSyncDispatchLifecycleActive(
|
|
963
|
+
lifecycle: SyncDispatchLifecycle,
|
|
964
|
+
target?: string,
|
|
965
|
+
): boolean {
|
|
966
|
+
if (
|
|
967
|
+
this.closed === true ||
|
|
968
|
+
lifecycle.disposed ||
|
|
969
|
+
lifecycle.ownershipLifecycleController !==
|
|
970
|
+
this.syncDispatchLifecycleController ||
|
|
971
|
+
lifecycle.ownershipLifecycleController.signal.aborted ||
|
|
972
|
+
lifecycle.callerSignal?.aborted ||
|
|
973
|
+
lifecycle.controller.signal.aborted
|
|
974
|
+
) {
|
|
975
|
+
return false;
|
|
554
976
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
state.unresolved.delete(hash);
|
|
558
|
-
}
|
|
559
|
-
if (this.isRepairSessionComplete(session)) {
|
|
560
|
-
this.finalizeRepairSession(sessionId, true);
|
|
561
|
-
}
|
|
977
|
+
if (target === undefined) {
|
|
978
|
+
return true;
|
|
562
979
|
}
|
|
980
|
+
const targetLifecycle = lifecycle.targets.get(target);
|
|
981
|
+
return (
|
|
982
|
+
targetLifecycle !== undefined &&
|
|
983
|
+
!targetLifecycle.controller.signal.aborted &&
|
|
984
|
+
this.syncDispatchTargetEpochs.get(target) === targetLifecycle.epoch
|
|
985
|
+
);
|
|
563
986
|
}
|
|
564
987
|
|
|
565
|
-
private
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
return;
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
const waitMs = Math.max(0, delayMs - previousDelay);
|
|
578
|
-
previousDelay = delayMs;
|
|
579
|
-
if (waitMs > 0) {
|
|
580
|
-
await new Promise<void>((resolve) => {
|
|
581
|
-
const timer = setTimeout(resolve, waitMs);
|
|
582
|
-
timer.unref?.();
|
|
583
|
-
});
|
|
584
|
-
}
|
|
585
|
-
if (!this.repairSessions.has(sessionId) || this.closed) {
|
|
586
|
-
return;
|
|
587
|
-
}
|
|
988
|
+
private getSyncDispatchSignal(
|
|
989
|
+
lifecycle: SyncDispatchLifecycle,
|
|
990
|
+
target: string,
|
|
991
|
+
): AbortSignal {
|
|
992
|
+
return (
|
|
993
|
+
lifecycle.targets.get(target)?.controller.signal ??
|
|
994
|
+
lifecycle.controller.signal
|
|
995
|
+
);
|
|
996
|
+
}
|
|
588
997
|
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
this.finalizeRepairSession(sessionId, true);
|
|
596
|
-
return;
|
|
597
|
-
}
|
|
998
|
+
private pendingMaybeSyncResponseWaiterBefore(
|
|
999
|
+
left: PendingMaybeSyncResponseWaiter,
|
|
1000
|
+
right: PendingMaybeSyncResponseWaiter,
|
|
1001
|
+
): boolean {
|
|
1002
|
+
return left.order < right.order;
|
|
1003
|
+
}
|
|
598
1004
|
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
1005
|
+
private swapPendingMaybeSyncResponseWaiters(
|
|
1006
|
+
left: number,
|
|
1007
|
+
right: number,
|
|
1008
|
+
): void {
|
|
1009
|
+
const leftWaiter = this.pendingMaybeSyncResponseWaiterHeap[left]!;
|
|
1010
|
+
const rightWaiter = this.pendingMaybeSyncResponseWaiterHeap[right]!;
|
|
1011
|
+
this.pendingMaybeSyncResponseWaiterHeap[left] = rightWaiter;
|
|
1012
|
+
this.pendingMaybeSyncResponseWaiterHeap[right] = leftWaiter;
|
|
1013
|
+
rightWaiter.heapIndex = left;
|
|
1014
|
+
leftWaiter.heapIndex = right;
|
|
1015
|
+
}
|
|
610
1016
|
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
1017
|
+
private pushPendingMaybeSyncResponseWaiter(
|
|
1018
|
+
waiter: PendingMaybeSyncResponseWaiter,
|
|
1019
|
+
): void {
|
|
1020
|
+
waiter.heapIndex = this.pendingMaybeSyncResponseWaiterHeap.length;
|
|
1021
|
+
this.pendingMaybeSyncResponseWaiterHeap.push(waiter);
|
|
1022
|
+
let index = waiter.heapIndex;
|
|
1023
|
+
while (index > 0) {
|
|
1024
|
+
const parent = Math.floor((index - 1) / 2);
|
|
1025
|
+
if (
|
|
1026
|
+
this.pendingMaybeSyncResponseWaiterBefore(
|
|
1027
|
+
this.pendingMaybeSyncResponseWaiterHeap[parent]!,
|
|
1028
|
+
this.pendingMaybeSyncResponseWaiterHeap[index]!,
|
|
1029
|
+
)
|
|
1030
|
+
) {
|
|
1031
|
+
break;
|
|
619
1032
|
}
|
|
1033
|
+
this.swapPendingMaybeSyncResponseWaiters(parent, index);
|
|
1034
|
+
index = parent;
|
|
1035
|
+
}
|
|
1036
|
+
this.pushPendingMaybeSyncResponseFitWaiter(waiter);
|
|
1037
|
+
}
|
|
620
1038
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
1039
|
+
private removePendingMaybeSyncResponseWaiter(
|
|
1040
|
+
waiter: PendingMaybeSyncResponseWaiter,
|
|
1041
|
+
): void {
|
|
1042
|
+
this.removePendingMaybeSyncResponseFitWaiter(waiter);
|
|
1043
|
+
const index = waiter.heapIndex;
|
|
1044
|
+
if (
|
|
1045
|
+
index < 0 ||
|
|
1046
|
+
index >= this.pendingMaybeSyncResponseWaiterHeap.length ||
|
|
1047
|
+
this.pendingMaybeSyncResponseWaiterHeap[index] !== waiter
|
|
1048
|
+
) {
|
|
1049
|
+
return;
|
|
1050
|
+
}
|
|
1051
|
+
const last = this.pendingMaybeSyncResponseWaiterHeap.pop()!;
|
|
1052
|
+
waiter.heapIndex = -1;
|
|
1053
|
+
if (index >= this.pendingMaybeSyncResponseWaiterHeap.length) {
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
this.pendingMaybeSyncResponseWaiterHeap[index] = last;
|
|
1057
|
+
last.heapIndex = index;
|
|
1058
|
+
let current = index;
|
|
1059
|
+
while (current > 0) {
|
|
1060
|
+
const parent = Math.floor((current - 1) / 2);
|
|
1061
|
+
if (
|
|
1062
|
+
this.pendingMaybeSyncResponseWaiterBefore(
|
|
1063
|
+
this.pendingMaybeSyncResponseWaiterHeap[parent]!,
|
|
1064
|
+
this.pendingMaybeSyncResponseWaiterHeap[current]!,
|
|
1065
|
+
)
|
|
1066
|
+
) {
|
|
1067
|
+
break;
|
|
624
1068
|
}
|
|
1069
|
+
this.swapPendingMaybeSyncResponseWaiters(parent, current);
|
|
1070
|
+
current = parent;
|
|
625
1071
|
}
|
|
626
|
-
|
|
627
1072
|
for (;;) {
|
|
628
|
-
|
|
629
|
-
|
|
1073
|
+
const left = current * 2 + 1;
|
|
1074
|
+
const right = left + 1;
|
|
1075
|
+
let smallest = current;
|
|
1076
|
+
if (
|
|
1077
|
+
left < this.pendingMaybeSyncResponseWaiterHeap.length &&
|
|
1078
|
+
this.pendingMaybeSyncResponseWaiterBefore(
|
|
1079
|
+
this.pendingMaybeSyncResponseWaiterHeap[left]!,
|
|
1080
|
+
this.pendingMaybeSyncResponseWaiterHeap[smallest]!,
|
|
1081
|
+
)
|
|
1082
|
+
) {
|
|
1083
|
+
smallest = left;
|
|
630
1084
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
1085
|
+
if (
|
|
1086
|
+
right < this.pendingMaybeSyncResponseWaiterHeap.length &&
|
|
1087
|
+
this.pendingMaybeSyncResponseWaiterBefore(
|
|
1088
|
+
this.pendingMaybeSyncResponseWaiterHeap[right]!,
|
|
1089
|
+
this.pendingMaybeSyncResponseWaiterHeap[smallest]!,
|
|
1090
|
+
)
|
|
1091
|
+
) {
|
|
1092
|
+
smallest = right;
|
|
635
1093
|
}
|
|
636
|
-
if (
|
|
637
|
-
|
|
638
|
-
return;
|
|
1094
|
+
if (smallest === current) {
|
|
1095
|
+
break;
|
|
639
1096
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
timer.unref?.();
|
|
643
|
-
});
|
|
1097
|
+
this.swapPendingMaybeSyncResponseWaiters(current, smallest);
|
|
1098
|
+
current = smallest;
|
|
644
1099
|
}
|
|
645
1100
|
}
|
|
646
1101
|
|
|
647
|
-
private
|
|
648
|
-
|
|
1102
|
+
private pendingMaybeSyncResponseFitWaiterBefore(
|
|
1103
|
+
left: PendingMaybeSyncResponseWaiter,
|
|
1104
|
+
right: PendingMaybeSyncResponseWaiter,
|
|
1105
|
+
): boolean {
|
|
1106
|
+
return (
|
|
1107
|
+
left.required < right.required ||
|
|
1108
|
+
(left.required === right.required && left.order < right.order)
|
|
1109
|
+
);
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
private swapPendingMaybeSyncResponseFitWaiters(
|
|
1113
|
+
left: number,
|
|
1114
|
+
right: number,
|
|
1115
|
+
): void {
|
|
1116
|
+
const leftWaiter = this.pendingMaybeSyncResponseWaiterFitHeap[left]!;
|
|
1117
|
+
const rightWaiter = this.pendingMaybeSyncResponseWaiterFitHeap[right]!;
|
|
1118
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[left] = rightWaiter;
|
|
1119
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[right] = leftWaiter;
|
|
1120
|
+
rightWaiter.fitHeapIndex = left;
|
|
1121
|
+
leftWaiter.fitHeapIndex = right;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
private pushPendingMaybeSyncResponseFitWaiter(
|
|
1125
|
+
waiter: PendingMaybeSyncResponseWaiter,
|
|
1126
|
+
): void {
|
|
1127
|
+
waiter.fitHeapIndex = this.pendingMaybeSyncResponseWaiterFitHeap.length;
|
|
1128
|
+
this.pendingMaybeSyncResponseWaiterFitHeap.push(waiter);
|
|
1129
|
+
let index = waiter.fitHeapIndex;
|
|
1130
|
+
while (index > 0) {
|
|
1131
|
+
const parent = Math.floor((index - 1) / 2);
|
|
1132
|
+
if (
|
|
1133
|
+
this.pendingMaybeSyncResponseFitWaiterBefore(
|
|
1134
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[parent]!,
|
|
1135
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[index]!,
|
|
1136
|
+
)
|
|
1137
|
+
) {
|
|
1138
|
+
break;
|
|
1139
|
+
}
|
|
1140
|
+
this.swapPendingMaybeSyncResponseFitWaiters(parent, index);
|
|
1141
|
+
index = parent;
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
private removePendingMaybeSyncResponseFitWaiter(
|
|
1146
|
+
waiter: PendingMaybeSyncResponseWaiter,
|
|
1147
|
+
): void {
|
|
1148
|
+
const index = waiter.fitHeapIndex;
|
|
1149
|
+
if (
|
|
1150
|
+
index < 0 ||
|
|
1151
|
+
index >= this.pendingMaybeSyncResponseWaiterFitHeap.length ||
|
|
1152
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[index] !== waiter
|
|
1153
|
+
) {
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
const last = this.pendingMaybeSyncResponseWaiterFitHeap.pop()!;
|
|
1157
|
+
waiter.fitHeapIndex = -1;
|
|
1158
|
+
if (index >= this.pendingMaybeSyncResponseWaiterFitHeap.length) {
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[index] = last;
|
|
1162
|
+
last.fitHeapIndex = index;
|
|
1163
|
+
let current = index;
|
|
1164
|
+
while (current > 0) {
|
|
1165
|
+
const parent = Math.floor((current - 1) / 2);
|
|
1166
|
+
if (
|
|
1167
|
+
this.pendingMaybeSyncResponseFitWaiterBefore(
|
|
1168
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[parent]!,
|
|
1169
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[current]!,
|
|
1170
|
+
)
|
|
1171
|
+
) {
|
|
1172
|
+
break;
|
|
1173
|
+
}
|
|
1174
|
+
this.swapPendingMaybeSyncResponseFitWaiters(parent, current);
|
|
1175
|
+
current = parent;
|
|
1176
|
+
}
|
|
1177
|
+
for (;;) {
|
|
1178
|
+
const left = current * 2 + 1;
|
|
1179
|
+
const right = left + 1;
|
|
1180
|
+
let smallest = current;
|
|
1181
|
+
if (
|
|
1182
|
+
left < this.pendingMaybeSyncResponseWaiterFitHeap.length &&
|
|
1183
|
+
this.pendingMaybeSyncResponseFitWaiterBefore(
|
|
1184
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[left]!,
|
|
1185
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[smallest]!,
|
|
1186
|
+
)
|
|
1187
|
+
) {
|
|
1188
|
+
smallest = left;
|
|
1189
|
+
}
|
|
1190
|
+
if (
|
|
1191
|
+
right < this.pendingMaybeSyncResponseWaiterFitHeap.length &&
|
|
1192
|
+
this.pendingMaybeSyncResponseFitWaiterBefore(
|
|
1193
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[right]!,
|
|
1194
|
+
this.pendingMaybeSyncResponseWaiterFitHeap[smallest]!,
|
|
1195
|
+
)
|
|
1196
|
+
) {
|
|
1197
|
+
smallest = right;
|
|
1198
|
+
}
|
|
1199
|
+
if (smallest === current) {
|
|
1200
|
+
break;
|
|
1201
|
+
}
|
|
1202
|
+
this.swapPendingMaybeSyncResponseFitWaiters(current, smallest);
|
|
1203
|
+
current = smallest;
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
private notifyPendingMaybeSyncResponseWaiter(): void {
|
|
1208
|
+
const waiter = this.pendingMaybeSyncResponseWaiterHeap[0];
|
|
1209
|
+
const available =
|
|
1210
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
|
|
1211
|
+
this.pendingMaybeSyncResponseCount;
|
|
1212
|
+
if (!waiter) {
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
if (waiter.required <= available) {
|
|
1216
|
+
waiter.wake();
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1219
|
+
if (waiter.bypasses >= MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITER_BYPASSES) {
|
|
1220
|
+
// Reserve newly freed capacity for the oldest large request after a
|
|
1221
|
+
// bounded number of smaller requests have bypassed it.
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
const candidate = this.pendingMaybeSyncResponseWaiterFitHeap[0];
|
|
1225
|
+
if (candidate && candidate.required <= available) {
|
|
1226
|
+
waiter.bypasses += 1;
|
|
1227
|
+
candidate.wake();
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
private schedulePendingMaybeSyncResponseWaiter(): void {
|
|
1232
|
+
if (this.pendingMaybeSyncResponseWakeScheduled) {
|
|
1233
|
+
return;
|
|
1234
|
+
}
|
|
1235
|
+
this.pendingMaybeSyncResponseWakeScheduled = true;
|
|
1236
|
+
queueMicrotask(() => {
|
|
1237
|
+
this.pendingMaybeSyncResponseWakeScheduled = false;
|
|
1238
|
+
this.notifyPendingMaybeSyncResponseWaiter();
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
private swapPendingMaybeSyncResponseExpiry(
|
|
1243
|
+
left: number,
|
|
1244
|
+
right: number,
|
|
1245
|
+
): void {
|
|
1246
|
+
const leftBatch = this.pendingMaybeSyncResponseExpiryHeap[left]!;
|
|
1247
|
+
const rightBatch = this.pendingMaybeSyncResponseExpiryHeap[right]!;
|
|
1248
|
+
this.pendingMaybeSyncResponseExpiryHeap[left] = rightBatch;
|
|
1249
|
+
this.pendingMaybeSyncResponseExpiryHeap[right] = leftBatch;
|
|
1250
|
+
rightBatch.heapIndex = left;
|
|
1251
|
+
leftBatch.heapIndex = right;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
private pushPendingMaybeSyncResponseExpiry(
|
|
1255
|
+
batch: PendingMaybeSyncResponse,
|
|
1256
|
+
): void {
|
|
1257
|
+
batch.heapIndex = this.pendingMaybeSyncResponseExpiryHeap.length;
|
|
1258
|
+
this.pendingMaybeSyncResponseExpiryHeap.push(batch);
|
|
1259
|
+
let index = batch.heapIndex;
|
|
1260
|
+
while (index > 0) {
|
|
1261
|
+
const parent = Math.floor((index - 1) / 2);
|
|
1262
|
+
if (
|
|
1263
|
+
this.pendingMaybeSyncResponseExpiryHeap[parent]!.expiresAt <=
|
|
1264
|
+
this.pendingMaybeSyncResponseExpiryHeap[index]!.expiresAt
|
|
1265
|
+
) {
|
|
1266
|
+
break;
|
|
1267
|
+
}
|
|
1268
|
+
this.swapPendingMaybeSyncResponseExpiry(parent, index);
|
|
1269
|
+
index = parent;
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
private removePendingMaybeSyncResponseExpiry(
|
|
1274
|
+
batch: PendingMaybeSyncResponse,
|
|
1275
|
+
): void {
|
|
1276
|
+
const index = batch.heapIndex;
|
|
1277
|
+
if (
|
|
1278
|
+
index < 0 ||
|
|
1279
|
+
index >= this.pendingMaybeSyncResponseExpiryHeap.length ||
|
|
1280
|
+
this.pendingMaybeSyncResponseExpiryHeap[index] !== batch
|
|
1281
|
+
) {
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
const last = this.pendingMaybeSyncResponseExpiryHeap.pop()!;
|
|
1285
|
+
batch.heapIndex = -1;
|
|
1286
|
+
if (index >= this.pendingMaybeSyncResponseExpiryHeap.length) {
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
this.pendingMaybeSyncResponseExpiryHeap[index] = last;
|
|
1290
|
+
last.heapIndex = index;
|
|
1291
|
+
let current = index;
|
|
1292
|
+
while (current > 0) {
|
|
1293
|
+
const parent = Math.floor((current - 1) / 2);
|
|
1294
|
+
if (
|
|
1295
|
+
this.pendingMaybeSyncResponseExpiryHeap[parent]!.expiresAt <=
|
|
1296
|
+
this.pendingMaybeSyncResponseExpiryHeap[current]!.expiresAt
|
|
1297
|
+
) {
|
|
1298
|
+
break;
|
|
1299
|
+
}
|
|
1300
|
+
this.swapPendingMaybeSyncResponseExpiry(parent, current);
|
|
1301
|
+
current = parent;
|
|
1302
|
+
}
|
|
1303
|
+
for (;;) {
|
|
1304
|
+
const left = current * 2 + 1;
|
|
1305
|
+
const right = left + 1;
|
|
1306
|
+
let smallest = current;
|
|
1307
|
+
if (
|
|
1308
|
+
left < this.pendingMaybeSyncResponseExpiryHeap.length &&
|
|
1309
|
+
this.pendingMaybeSyncResponseExpiryHeap[left]!.expiresAt <
|
|
1310
|
+
this.pendingMaybeSyncResponseExpiryHeap[smallest]!.expiresAt
|
|
1311
|
+
) {
|
|
1312
|
+
smallest = left;
|
|
1313
|
+
}
|
|
1314
|
+
if (
|
|
1315
|
+
right < this.pendingMaybeSyncResponseExpiryHeap.length &&
|
|
1316
|
+
this.pendingMaybeSyncResponseExpiryHeap[right]!.expiresAt <
|
|
1317
|
+
this.pendingMaybeSyncResponseExpiryHeap[smallest]!.expiresAt
|
|
1318
|
+
) {
|
|
1319
|
+
smallest = right;
|
|
1320
|
+
}
|
|
1321
|
+
if (smallest === current) {
|
|
1322
|
+
break;
|
|
1323
|
+
}
|
|
1324
|
+
this.swapPendingMaybeSyncResponseExpiry(current, smallest);
|
|
1325
|
+
current = smallest;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
private schedulePendingMaybeSyncResponseExpiry(): void {
|
|
1330
|
+
if (
|
|
1331
|
+
this.pendingMaybeSyncResponseExpiryTimer ||
|
|
1332
|
+
this.pendingMaybeSyncResponseExpiryHeap.length === 0
|
|
1333
|
+
) {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
const earliest = this.pendingMaybeSyncResponseExpiryHeap[0]!.expiresAt;
|
|
1337
|
+
this.pendingMaybeSyncResponseExpiryTimer = setTimeout(
|
|
1338
|
+
() => {
|
|
1339
|
+
this.pendingMaybeSyncResponseExpiryTimer = undefined;
|
|
1340
|
+
this.expirePendingMaybeSyncResponses();
|
|
1341
|
+
this.schedulePendingMaybeSyncResponseExpiry();
|
|
1342
|
+
},
|
|
1343
|
+
Math.max(0, earliest - Date.now()),
|
|
1344
|
+
);
|
|
1345
|
+
this.pendingMaybeSyncResponseExpiryTimer.unref?.();
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
private expirePendingMaybeSyncResponses(now = Date.now()): void {
|
|
1349
|
+
for (;;) {
|
|
1350
|
+
const batch = this.pendingMaybeSyncResponseExpiryHeap[0];
|
|
1351
|
+
if (!batch || batch.expiresAt > now) {
|
|
1352
|
+
break;
|
|
1353
|
+
}
|
|
1354
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
private settlePendingMaybeSyncResponseAuthorization(
|
|
1359
|
+
authorization: PendingMaybeSyncResponseAuthorization,
|
|
1360
|
+
fulfilled: boolean,
|
|
1361
|
+
): void {
|
|
1362
|
+
if (authorization.settled) {
|
|
1363
|
+
return;
|
|
1364
|
+
}
|
|
1365
|
+
authorization.settled = fulfilled ? "fulfilled" : "released";
|
|
1366
|
+
const waiters = [...authorization.waiters];
|
|
1367
|
+
authorization.waiters.clear();
|
|
1368
|
+
for (const waiter of waiters) {
|
|
1369
|
+
waiter(fulfilled ? "fulfilled" : "released");
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
private doesPendingMaybeSyncResponseScopeMatch(
|
|
1374
|
+
authorization: PendingMaybeSyncResponseAuthorization,
|
|
1375
|
+
lifecycle: SyncDispatchLifecycle,
|
|
1376
|
+
target: string,
|
|
1377
|
+
): boolean {
|
|
1378
|
+
const owner = authorization.batch.targetLifecycle;
|
|
1379
|
+
return (
|
|
1380
|
+
owner.epoch === lifecycle.targets.get(target)?.epoch &&
|
|
1381
|
+
owner.lifecycle.ownershipLifecycleController ===
|
|
1382
|
+
lifecycle.ownershipLifecycleController &&
|
|
1383
|
+
owner.lifecycle.callerSignal === lifecycle.callerSignal
|
|
1384
|
+
);
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
private waitForPendingMaybeSyncResponseConflicts(
|
|
1388
|
+
conflicts: PendingMaybeSyncResponseAuthorization[],
|
|
1389
|
+
lifecycle: SyncDispatchLifecycle,
|
|
1390
|
+
target: string,
|
|
1391
|
+
retainedAssociations: number,
|
|
1392
|
+
): Promise<string[]> {
|
|
1393
|
+
const unique = [...new Set(conflicts)];
|
|
1394
|
+
if (
|
|
1395
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount +
|
|
1396
|
+
retainedAssociations >
|
|
1397
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES
|
|
1398
|
+
) {
|
|
1399
|
+
return Promise.resolve([]);
|
|
1400
|
+
}
|
|
1401
|
+
const available =
|
|
1402
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
|
|
1403
|
+
this.pendingMaybeSyncResponseConflictWaiterCount;
|
|
1404
|
+
const admitted = unique.slice(0, Math.max(0, available));
|
|
1405
|
+
if (admitted.length === 0) {
|
|
1406
|
+
return Promise.resolve([]);
|
|
1407
|
+
}
|
|
1408
|
+
this.pendingMaybeSyncResponseConflictWaiterCount += admitted.length;
|
|
1409
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount += retainedAssociations;
|
|
1410
|
+
const targetSignal =
|
|
1411
|
+
lifecycle.targets.get(target)?.controller.signal ??
|
|
1412
|
+
lifecycle.controller.signal;
|
|
1413
|
+
return new Promise<string[]>((resolve) => {
|
|
1414
|
+
const retry = new Set<string>();
|
|
1415
|
+
const callbacks = new Map<
|
|
1416
|
+
PendingMaybeSyncResponseAuthorization,
|
|
1417
|
+
(event: PendingMaybeSyncResponseAuthorizationEvent) => void
|
|
1418
|
+
>();
|
|
1419
|
+
let remaining = admitted.length;
|
|
1420
|
+
let groupSettled = false;
|
|
1421
|
+
const finishGroup = () => {
|
|
1422
|
+
if (groupSettled || remaining !== 0) {
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1425
|
+
groupSettled = true;
|
|
1426
|
+
lifecycle.controller.signal.removeEventListener("abort", abort);
|
|
1427
|
+
if (targetSignal !== lifecycle.controller.signal) {
|
|
1428
|
+
targetSignal.removeEventListener("abort", abort);
|
|
1429
|
+
}
|
|
1430
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount -=
|
|
1431
|
+
retainedAssociations;
|
|
1432
|
+
resolve([...retry]);
|
|
1433
|
+
};
|
|
1434
|
+
const finishOne = (
|
|
1435
|
+
authorization: PendingMaybeSyncResponseAuthorization,
|
|
1436
|
+
event: PendingMaybeSyncResponseAuthorizationEvent,
|
|
1437
|
+
) => {
|
|
1438
|
+
if (
|
|
1439
|
+
event === "delivered" &&
|
|
1440
|
+
(authorization.active === true ||
|
|
1441
|
+
!this.doesPendingMaybeSyncResponseScopeMatch(
|
|
1442
|
+
authorization,
|
|
1443
|
+
lifecycle,
|
|
1444
|
+
target,
|
|
1445
|
+
))
|
|
1446
|
+
) {
|
|
1447
|
+
return;
|
|
1448
|
+
}
|
|
1449
|
+
const callback = callbacks.get(authorization);
|
|
1450
|
+
if (!callback) {
|
|
1451
|
+
return;
|
|
1452
|
+
}
|
|
1453
|
+
callbacks.delete(authorization);
|
|
1454
|
+
authorization.waiters.delete(callback);
|
|
1455
|
+
this.pendingMaybeSyncResponseConflictWaiterCount -= 1;
|
|
1456
|
+
if (
|
|
1457
|
+
event === "released" &&
|
|
1458
|
+
this.isSyncDispatchLifecycleActive(lifecycle, target)
|
|
1459
|
+
) {
|
|
1460
|
+
retry.add(authorization.hash);
|
|
1461
|
+
}
|
|
1462
|
+
remaining -= 1;
|
|
1463
|
+
finishGroup();
|
|
1464
|
+
};
|
|
1465
|
+
const abort = () => {
|
|
1466
|
+
for (const authorization of [...callbacks.keys()]) {
|
|
1467
|
+
finishOne(authorization, "fulfilled");
|
|
1468
|
+
}
|
|
1469
|
+
};
|
|
1470
|
+
lifecycle.controller.signal.addEventListener("abort", abort, {
|
|
1471
|
+
once: true,
|
|
1472
|
+
});
|
|
1473
|
+
if (targetSignal !== lifecycle.controller.signal) {
|
|
1474
|
+
targetSignal.addEventListener("abort", abort, { once: true });
|
|
1475
|
+
}
|
|
1476
|
+
for (const authorization of admitted) {
|
|
1477
|
+
const callback = (event: PendingMaybeSyncResponseAuthorizationEvent) =>
|
|
1478
|
+
finishOne(authorization, event);
|
|
1479
|
+
callbacks.set(authorization, callback);
|
|
1480
|
+
authorization.waiters.add(callback);
|
|
1481
|
+
if (authorization.settled) {
|
|
1482
|
+
callback(authorization.settled);
|
|
1483
|
+
} else if (
|
|
1484
|
+
authorization.requestDelivered === true &&
|
|
1485
|
+
authorization.active !== true
|
|
1486
|
+
) {
|
|
1487
|
+
callback("delivered");
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
1491
|
+
abort();
|
|
1492
|
+
}
|
|
1493
|
+
});
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
private removePendingMaybeSyncResponseBatch(
|
|
1497
|
+
batch: PendingMaybeSyncResponse,
|
|
1498
|
+
): void {
|
|
1499
|
+
const pendingForTarget = this.pendingMaybeSyncResponses.get(batch.target);
|
|
1500
|
+
let removed = 0;
|
|
1501
|
+
if (pendingForTarget) {
|
|
1502
|
+
for (const hash of batch.hashes) {
|
|
1503
|
+
const authorization = pendingForTarget.get(hash);
|
|
1504
|
+
if (authorization?.batch !== batch) {
|
|
1505
|
+
continue;
|
|
1506
|
+
}
|
|
1507
|
+
this.settlePendingMaybeSyncResponseAuthorization(authorization, false);
|
|
1508
|
+
pendingForTarget.delete(hash);
|
|
1509
|
+
if (authorization.deliveryInFlight !== true) {
|
|
1510
|
+
removed += 1;
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
if (pendingForTarget.size === 0) {
|
|
1514
|
+
this.pendingMaybeSyncResponses.delete(batch.target);
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
this.pendingMaybeSyncResponseCount -= removed;
|
|
1518
|
+
this.pendingMaybeSyncResponseBatches.delete(batch);
|
|
1519
|
+
this.removePendingMaybeSyncResponseExpiry(batch);
|
|
1520
|
+
if (batch.targetLifecycle.batches.delete(batch)) {
|
|
1521
|
+
batch.targetLifecycle.lifecycle.retainedWork -= 1;
|
|
1522
|
+
}
|
|
1523
|
+
batch.hashes.clear();
|
|
1524
|
+
if (
|
|
1525
|
+
this.pendingMaybeSyncResponseExpiryHeap.length === 0 &&
|
|
1526
|
+
this.pendingMaybeSyncResponseExpiryTimer
|
|
1527
|
+
) {
|
|
1528
|
+
clearTimeout(this.pendingMaybeSyncResponseExpiryTimer);
|
|
1529
|
+
this.pendingMaybeSyncResponseExpiryTimer = undefined;
|
|
1530
|
+
}
|
|
1531
|
+
if (removed > 0) {
|
|
1532
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1533
|
+
}
|
|
1534
|
+
this.maybeDisposeSyncDispatchLifecycle(batch.targetLifecycle.lifecycle);
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
private clearPendingMaybeSyncResponses(target?: string): void {
|
|
1538
|
+
const batches =
|
|
1539
|
+
target === undefined
|
|
1540
|
+
? [...this.pendingMaybeSyncResponseBatches]
|
|
1541
|
+
: [
|
|
1542
|
+
...new Set(
|
|
1543
|
+
[
|
|
1544
|
+
...(this.pendingMaybeSyncResponses.get(target)?.values() ?? []),
|
|
1545
|
+
].map((authorization) => authorization.batch),
|
|
1546
|
+
),
|
|
1547
|
+
];
|
|
1548
|
+
for (const batch of batches) {
|
|
1549
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
private tryReservePendingMaybeSyncResponse(properties: {
|
|
1554
|
+
hashes: Iterable<string>;
|
|
1555
|
+
targets: string[];
|
|
1556
|
+
lifecycle: SyncDispatchLifecycle;
|
|
1557
|
+
}): PendingMaybeSyncResponseReservationAttempt {
|
|
1558
|
+
// Timers are only a cleanup aid. Enforce absolute deadlines at the
|
|
1559
|
+
// admission boundary as well, including for unrelated fresh hashes.
|
|
1560
|
+
this.expirePendingMaybeSyncResponses();
|
|
1561
|
+
const hashes = [...new Set(properties.hashes)];
|
|
1562
|
+
const targets = [...new Set(properties.targets)];
|
|
1563
|
+
if (
|
|
1564
|
+
!this.isSyncDispatchLifecycleActive(properties.lifecycle) ||
|
|
1565
|
+
targets.some(
|
|
1566
|
+
(target) =>
|
|
1567
|
+
!this.isSyncDispatchLifecycleActive(properties.lifecycle, target),
|
|
1568
|
+
)
|
|
1569
|
+
) {
|
|
1570
|
+
return { kind: "inactive" };
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
const hashesToAddByTarget = new Map<string, string[]>();
|
|
1574
|
+
const conflicts: PendingMaybeSyncResponseAuthorization[] = [];
|
|
1575
|
+
let required = 0;
|
|
1576
|
+
for (const target of targets) {
|
|
1577
|
+
const hashesToAdd: string[] = [];
|
|
1578
|
+
for (const hash of hashes) {
|
|
1579
|
+
let existing = this.pendingMaybeSyncResponses.get(target)?.get(hash);
|
|
1580
|
+
if (
|
|
1581
|
+
existing &&
|
|
1582
|
+
existing.active !== true &&
|
|
1583
|
+
(existing.batch.expiresAt <= Date.now() ||
|
|
1584
|
+
!this.isSyncDispatchLifecycleActive(
|
|
1585
|
+
existing.batch.targetLifecycle.lifecycle,
|
|
1586
|
+
target,
|
|
1587
|
+
))
|
|
1588
|
+
) {
|
|
1589
|
+
this.removePendingMaybeSyncResponseBatch(existing.batch);
|
|
1590
|
+
existing = undefined;
|
|
1591
|
+
}
|
|
1592
|
+
if (existing) {
|
|
1593
|
+
const existingTarget = existing.batch.targetLifecycle;
|
|
1594
|
+
if (
|
|
1595
|
+
existingTarget.lifecycle === properties.lifecycle ||
|
|
1596
|
+
(existing.active !== true &&
|
|
1597
|
+
existing.requestDelivered === true &&
|
|
1598
|
+
this.doesPendingMaybeSyncResponseScopeMatch(
|
|
1599
|
+
existing,
|
|
1600
|
+
properties.lifecycle,
|
|
1601
|
+
target,
|
|
1602
|
+
))
|
|
1603
|
+
) {
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
// Another live caller already owns the authorization for this
|
|
1607
|
+
// exact target/hash. Send unrelated hashes now, then wait for this
|
|
1608
|
+
// authorization to be fulfilled or released before deciding
|
|
1609
|
+
// whether this caller must retry it.
|
|
1610
|
+
conflicts.push(existing);
|
|
1611
|
+
continue;
|
|
1612
|
+
}
|
|
1613
|
+
hashesToAdd.push(hash);
|
|
1614
|
+
}
|
|
1615
|
+
if (hashesToAdd.length > 0) {
|
|
1616
|
+
hashesToAddByTarget.set(target, hashesToAdd);
|
|
1617
|
+
required += hashesToAdd.length;
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
if (
|
|
1622
|
+
required >
|
|
1623
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
|
|
1624
|
+
this.pendingMaybeSyncResponseCount
|
|
1625
|
+
) {
|
|
1626
|
+
return { kind: "capacity", required };
|
|
1627
|
+
}
|
|
1628
|
+
|
|
1629
|
+
const addedBatches: PendingMaybeSyncResponse[] = [];
|
|
1630
|
+
const addedAuthorizations: PendingMaybeSyncResponseAuthorization[] = [];
|
|
1631
|
+
for (const [target, hashesToAdd] of hashesToAddByTarget) {
|
|
1632
|
+
const targetLifecycle = properties.lifecycle.targets.get(target)!;
|
|
1633
|
+
const batch: PendingMaybeSyncResponse = {
|
|
1634
|
+
hashes: new Set(hashesToAdd),
|
|
1635
|
+
target,
|
|
1636
|
+
targetLifecycle,
|
|
1637
|
+
// Start the response deadline only after rpc.send succeeds. Keeping
|
|
1638
|
+
// pre-delivery work charged prevents a slow/non-abortable transport
|
|
1639
|
+
// from rolling over an unbounded number of sends.
|
|
1640
|
+
expiresAt: Infinity,
|
|
1641
|
+
heapIndex: -1,
|
|
1642
|
+
};
|
|
1643
|
+
let pendingForTarget = this.pendingMaybeSyncResponses.get(target);
|
|
1644
|
+
if (!pendingForTarget) {
|
|
1645
|
+
pendingForTarget = new Map();
|
|
1646
|
+
this.pendingMaybeSyncResponses.set(target, pendingForTarget);
|
|
1647
|
+
}
|
|
1648
|
+
for (const hash of hashesToAdd) {
|
|
1649
|
+
const authorization: PendingMaybeSyncResponseAuthorization = {
|
|
1650
|
+
batch,
|
|
1651
|
+
hash,
|
|
1652
|
+
waiters: new Set(),
|
|
1653
|
+
};
|
|
1654
|
+
pendingForTarget.set(hash, authorization);
|
|
1655
|
+
addedAuthorizations.push(authorization);
|
|
1656
|
+
}
|
|
1657
|
+
this.pendingMaybeSyncResponseCount += hashesToAdd.length;
|
|
1658
|
+
this.pendingMaybeSyncResponseBatches.add(batch);
|
|
1659
|
+
targetLifecycle.batches.add(batch);
|
|
1660
|
+
targetLifecycle.lifecycle.retainedWork += 1;
|
|
1661
|
+
addedBatches.push(batch);
|
|
1662
|
+
}
|
|
1663
|
+
this.schedulePendingMaybeSyncResponseExpiry();
|
|
1664
|
+
if (this.pendingMaybeSyncResponseWaiters.size > 0) {
|
|
1665
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
let released = false;
|
|
1669
|
+
const release = () => {
|
|
1670
|
+
if (released) {
|
|
1671
|
+
return;
|
|
1672
|
+
}
|
|
1673
|
+
released = true;
|
|
1674
|
+
for (const batch of addedBatches) {
|
|
1675
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
1676
|
+
}
|
|
1677
|
+
};
|
|
1678
|
+
const retained = () =>
|
|
1679
|
+
this.isSyncDispatchLifecycleActive(properties.lifecycle) &&
|
|
1680
|
+
addedBatches.every(
|
|
1681
|
+
(batch) =>
|
|
1682
|
+
this.pendingMaybeSyncResponseBatches.has(batch) &&
|
|
1683
|
+
batch.expiresAt > Date.now() &&
|
|
1684
|
+
batch.hashes.size > 0,
|
|
1685
|
+
);
|
|
1686
|
+
if (!this.isSyncDispatchLifecycleActive(properties.lifecycle)) {
|
|
1687
|
+
release();
|
|
1688
|
+
return { kind: "inactive" };
|
|
1689
|
+
}
|
|
1690
|
+
return {
|
|
1691
|
+
kind: "reserved",
|
|
1692
|
+
reservation: {
|
|
1693
|
+
release,
|
|
1694
|
+
beginDelivery: () => {
|
|
1695
|
+
for (const authorization of addedAuthorizations) {
|
|
1696
|
+
if (!authorization.settled) {
|
|
1697
|
+
authorization.deliveryInFlight = true;
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
},
|
|
1701
|
+
finishDelivery: () => {
|
|
1702
|
+
let releasedCount = 0;
|
|
1703
|
+
for (const authorization of addedAuthorizations) {
|
|
1704
|
+
if (authorization.deliveryInFlight !== true) {
|
|
1705
|
+
continue;
|
|
1706
|
+
}
|
|
1707
|
+
authorization.deliveryInFlight = false;
|
|
1708
|
+
if (
|
|
1709
|
+
authorization.settled ||
|
|
1710
|
+
this.pendingMaybeSyncResponses
|
|
1711
|
+
.get(authorization.batch.target)
|
|
1712
|
+
?.get(authorization.hash) !== authorization
|
|
1713
|
+
) {
|
|
1714
|
+
releasedCount += 1;
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
if (releasedCount > 0) {
|
|
1718
|
+
this.pendingMaybeSyncResponseCount -= releasedCount;
|
|
1719
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1720
|
+
}
|
|
1721
|
+
},
|
|
1722
|
+
markDelivered: () => {
|
|
1723
|
+
const delivered: PendingMaybeSyncResponseAuthorization[] = [];
|
|
1724
|
+
const deliveredBatches = new Set<PendingMaybeSyncResponse>();
|
|
1725
|
+
for (const authorization of addedAuthorizations) {
|
|
1726
|
+
if (
|
|
1727
|
+
authorization.settled ||
|
|
1728
|
+
authorization.active === true ||
|
|
1729
|
+
this.pendingMaybeSyncResponses
|
|
1730
|
+
.get(authorization.batch.target)
|
|
1731
|
+
?.get(authorization.hash) !== authorization ||
|
|
1732
|
+
!this.pendingMaybeSyncResponseBatches.has(authorization.batch)
|
|
1733
|
+
) {
|
|
1734
|
+
continue;
|
|
1735
|
+
}
|
|
1736
|
+
authorization.requestDelivered = true;
|
|
1737
|
+
delivered.push(authorization);
|
|
1738
|
+
deliveredBatches.add(authorization.batch);
|
|
1739
|
+
}
|
|
1740
|
+
const expiresAt = Date.now() + PENDING_MAYBE_SYNC_RESPONSE_TTL_MS;
|
|
1741
|
+
for (const batch of deliveredBatches) {
|
|
1742
|
+
if (batch.heapIndex >= 0) {
|
|
1743
|
+
this.removePendingMaybeSyncResponseExpiry(batch);
|
|
1744
|
+
}
|
|
1745
|
+
batch.expiresAt = expiresAt;
|
|
1746
|
+
this.pushPendingMaybeSyncResponseExpiry(batch);
|
|
1747
|
+
}
|
|
1748
|
+
this.schedulePendingMaybeSyncResponseExpiry();
|
|
1749
|
+
for (const authorization of delivered) {
|
|
1750
|
+
if (authorization.settled || authorization.active === true) {
|
|
1751
|
+
continue;
|
|
1752
|
+
}
|
|
1753
|
+
for (const waiter of [...authorization.waiters]) {
|
|
1754
|
+
waiter("delivered");
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
},
|
|
1758
|
+
newlyAuthorizedByTarget: hashesToAddByTarget,
|
|
1759
|
+
retained,
|
|
1760
|
+
signal: properties.lifecycle.controller.signal,
|
|
1761
|
+
},
|
|
1762
|
+
conflicts,
|
|
1763
|
+
};
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
private waitForPendingMaybeSyncResponseChange(
|
|
1767
|
+
lifecycle: SyncDispatchLifecycle,
|
|
1768
|
+
targets: string[],
|
|
1769
|
+
required: number,
|
|
1770
|
+
associations: number,
|
|
1771
|
+
): Promise<void> {
|
|
1772
|
+
if (
|
|
1773
|
+
!this.isSyncDispatchLifecycleActive(lifecycle) ||
|
|
1774
|
+
targets.some(
|
|
1775
|
+
(target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
|
|
1776
|
+
)
|
|
1777
|
+
) {
|
|
1778
|
+
return Promise.resolve();
|
|
1779
|
+
}
|
|
1780
|
+
const targetLifecycles = targets
|
|
1781
|
+
.map((target) => lifecycle.targets.get(target))
|
|
1782
|
+
.filter(
|
|
1783
|
+
(target): target is SyncDispatchTargetLifecycle => target !== undefined,
|
|
1784
|
+
);
|
|
1785
|
+
for (const targetLifecycle of targetLifecycles) {
|
|
1786
|
+
targetLifecycle.activeWaiters += 1;
|
|
1787
|
+
targetLifecycle.lifecycle.retainedWork += 1;
|
|
1788
|
+
}
|
|
1789
|
+
return new Promise<void>((resolve) => {
|
|
1790
|
+
let settled = false;
|
|
1791
|
+
let waiter!: PendingMaybeSyncResponseWaiter;
|
|
1792
|
+
const abortSignals = [
|
|
1793
|
+
lifecycle.controller.signal,
|
|
1794
|
+
...targetLifecycles.map(
|
|
1795
|
+
(targetLifecycle) => targetLifecycle.controller.signal,
|
|
1796
|
+
),
|
|
1797
|
+
];
|
|
1798
|
+
const wake = () => {
|
|
1799
|
+
if (settled) {
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
const advanceWaiters =
|
|
1803
|
+
!this.isSyncDispatchLifecycleActive(lifecycle) ||
|
|
1804
|
+
targets.some(
|
|
1805
|
+
(target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
|
|
1806
|
+
);
|
|
1807
|
+
settled = true;
|
|
1808
|
+
this.pendingMaybeSyncResponseWaiters.delete(waiter);
|
|
1809
|
+
this.removePendingMaybeSyncResponseWaiter(waiter);
|
|
1810
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount -=
|
|
1811
|
+
waiter.associations;
|
|
1812
|
+
for (const signal of abortSignals) {
|
|
1813
|
+
signal.removeEventListener("abort", wake);
|
|
1814
|
+
}
|
|
1815
|
+
for (const targetLifecycle of targetLifecycles) {
|
|
1816
|
+
targetLifecycle.activeWaiters -= 1;
|
|
1817
|
+
targetLifecycle.lifecycle.retainedWork -= 1;
|
|
1818
|
+
}
|
|
1819
|
+
resolve();
|
|
1820
|
+
this.maybeDisposeSyncDispatchLifecycle(lifecycle);
|
|
1821
|
+
if (advanceWaiters) {
|
|
1822
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1823
|
+
}
|
|
1824
|
+
};
|
|
1825
|
+
waiter = {
|
|
1826
|
+
required,
|
|
1827
|
+
associations,
|
|
1828
|
+
order: ++this.pendingMaybeSyncResponseWaiterOrder,
|
|
1829
|
+
bypasses: 0,
|
|
1830
|
+
heapIndex: -1,
|
|
1831
|
+
fitHeapIndex: -1,
|
|
1832
|
+
wake,
|
|
1833
|
+
};
|
|
1834
|
+
this.pendingMaybeSyncResponseWaiters.add(waiter);
|
|
1835
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount += associations;
|
|
1836
|
+
this.pushPendingMaybeSyncResponseWaiter(waiter);
|
|
1837
|
+
// This exact request does not fit, but an older/stale requirement may
|
|
1838
|
+
// have changed and another smaller waiter can still use the capacity.
|
|
1839
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1840
|
+
for (const signal of abortSignals) {
|
|
1841
|
+
signal.addEventListener("abort", wake, { once: true });
|
|
1842
|
+
}
|
|
1843
|
+
if (
|
|
1844
|
+
!this.isSyncDispatchLifecycleActive(lifecycle) ||
|
|
1845
|
+
targets.some(
|
|
1846
|
+
(target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
|
|
1847
|
+
)
|
|
1848
|
+
) {
|
|
1849
|
+
wake();
|
|
1850
|
+
}
|
|
1851
|
+
});
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
private async reservePendingMaybeSyncResponse(properties: {
|
|
1855
|
+
hashes: Iterable<string>;
|
|
1856
|
+
targets: string[];
|
|
1857
|
+
lifecycle: SyncDispatchLifecycle;
|
|
1858
|
+
}): Promise<
|
|
1859
|
+
| {
|
|
1860
|
+
reservation: PendingMaybeSyncResponseReservation;
|
|
1861
|
+
conflicts: PendingMaybeSyncResponseAuthorization[];
|
|
1862
|
+
}
|
|
1863
|
+
| undefined
|
|
1864
|
+
> {
|
|
1865
|
+
const hashes = [...new Set(properties.hashes)];
|
|
1866
|
+
const targets = [...new Set(properties.targets)];
|
|
1867
|
+
const associations = Math.max(
|
|
1868
|
+
1,
|
|
1869
|
+
Math.min(MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES, hashes.length) *
|
|
1870
|
+
Math.min(MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES, targets.length),
|
|
1871
|
+
);
|
|
1872
|
+
while (
|
|
1873
|
+
this.isSyncDispatchLifecycleActive(properties.lifecycle) &&
|
|
1874
|
+
targets.every((target) =>
|
|
1875
|
+
this.isSyncDispatchLifecycleActive(properties.lifecycle, target),
|
|
1876
|
+
)
|
|
1877
|
+
) {
|
|
1878
|
+
const attempt = this.tryReservePendingMaybeSyncResponse({
|
|
1879
|
+
hashes,
|
|
1880
|
+
targets,
|
|
1881
|
+
lifecycle: properties.lifecycle,
|
|
1882
|
+
});
|
|
1883
|
+
if (attempt.kind === "reserved") {
|
|
1884
|
+
return {
|
|
1885
|
+
reservation: attempt.reservation,
|
|
1886
|
+
conflicts: attempt.conflicts,
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
if (attempt.kind !== "capacity") {
|
|
1890
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
1891
|
+
return undefined;
|
|
1892
|
+
}
|
|
1893
|
+
if (
|
|
1894
|
+
this.pendingMaybeSyncResponseWaiters.size >=
|
|
1895
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITERS ||
|
|
1896
|
+
this.pendingMaybeSyncResponseWaiterAssociationCount + associations >
|
|
1897
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES
|
|
1898
|
+
) {
|
|
1899
|
+
return undefined;
|
|
1900
|
+
}
|
|
1901
|
+
await this.waitForPendingMaybeSyncResponseChange(
|
|
1902
|
+
properties.lifecycle,
|
|
1903
|
+
targets,
|
|
1904
|
+
attempt.required,
|
|
1905
|
+
associations,
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
return undefined;
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1911
|
+
expectMaybeSyncResponse(properties: {
|
|
1912
|
+
hashes: Iterable<string>;
|
|
1913
|
+
targets: string[];
|
|
1914
|
+
signal?: AbortSignal;
|
|
1915
|
+
}): PendingMaybeSyncResponseReservation | undefined {
|
|
1916
|
+
const targets = [...new Set(properties.targets)];
|
|
1917
|
+
const lifecycle = this.captureSyncDispatchLifecycle(
|
|
1918
|
+
targets,
|
|
1919
|
+
properties.signal,
|
|
1920
|
+
{ abortAllOnTargetDisconnect: true },
|
|
1921
|
+
);
|
|
1922
|
+
const attempt = this.tryReservePendingMaybeSyncResponse({
|
|
1923
|
+
hashes: properties.hashes,
|
|
1924
|
+
targets,
|
|
1925
|
+
lifecycle,
|
|
1926
|
+
});
|
|
1927
|
+
const reservation =
|
|
1928
|
+
attempt.kind === "reserved" ? attempt.reservation : undefined;
|
|
1929
|
+
let retainedReservation: PendingMaybeSyncResponseReservation | undefined;
|
|
1930
|
+
if (reservation) {
|
|
1931
|
+
// This synchronous helper represents an already-issued request.
|
|
1932
|
+
reservation.markDelivered();
|
|
1933
|
+
const leasedTargets = [...lifecycle.targets.values()];
|
|
1934
|
+
for (const targetLifecycle of leasedTargets) {
|
|
1935
|
+
targetLifecycle.responseLeases += 1;
|
|
1936
|
+
targetLifecycle.lifecycle.retainedWork += 1;
|
|
1937
|
+
}
|
|
1938
|
+
let released = false;
|
|
1939
|
+
retainedReservation = {
|
|
1940
|
+
...reservation,
|
|
1941
|
+
release: () => {
|
|
1942
|
+
if (released) {
|
|
1943
|
+
return;
|
|
1944
|
+
}
|
|
1945
|
+
released = true;
|
|
1946
|
+
reservation.release();
|
|
1947
|
+
for (const targetLifecycle of leasedTargets) {
|
|
1948
|
+
targetLifecycle.responseLeases -= 1;
|
|
1949
|
+
targetLifecycle.lifecycle.retainedWork -= 1;
|
|
1950
|
+
}
|
|
1951
|
+
this.maybeDisposeSyncDispatchLifecycle(lifecycle);
|
|
1952
|
+
},
|
|
1953
|
+
};
|
|
1954
|
+
}
|
|
1955
|
+
this.finishSyncDispatchLifecycle(lifecycle);
|
|
1956
|
+
return retainedReservation;
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
consumeAuthorizedMaybeSyncResponse(
|
|
1960
|
+
hashes: Iterable<string>,
|
|
1961
|
+
from: PublicSignKey,
|
|
1962
|
+
): AuthorizedMaybeSyncResponseLease[] {
|
|
1963
|
+
const fromHash = from.hashcode();
|
|
1964
|
+
const pendingForTarget = this.pendingMaybeSyncResponses.get(fromHash);
|
|
1965
|
+
if (!pendingForTarget) {
|
|
1966
|
+
return [];
|
|
1967
|
+
}
|
|
1968
|
+
if (
|
|
1969
|
+
this.activeMaybeSyncResponseCount >=
|
|
1970
|
+
MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_GLOBAL ||
|
|
1971
|
+
(this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 0) >=
|
|
1972
|
+
MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_PER_PEER
|
|
1973
|
+
) {
|
|
1974
|
+
return [];
|
|
1975
|
+
}
|
|
1976
|
+
const acceptedByLifecycle = new Map<
|
|
1977
|
+
SyncDispatchTargetLifecycle,
|
|
1978
|
+
{
|
|
1979
|
+
hashes: string[];
|
|
1980
|
+
authorizations: PendingMaybeSyncResponseAuthorization[];
|
|
1981
|
+
}
|
|
1982
|
+
>();
|
|
1983
|
+
const seen = new Set<string>();
|
|
1984
|
+
let inspected = 0;
|
|
1985
|
+
const iterator = hashes[Symbol.iterator]();
|
|
1986
|
+
let exhausted = false;
|
|
1987
|
+
try {
|
|
1988
|
+
while (
|
|
1989
|
+
inspected < MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES &&
|
|
1990
|
+
pendingForTarget.size > 0
|
|
1991
|
+
) {
|
|
1992
|
+
const next = iterator.next();
|
|
1993
|
+
if (next.done) {
|
|
1994
|
+
exhausted = true;
|
|
1995
|
+
break;
|
|
1996
|
+
}
|
|
1997
|
+
inspected += 1;
|
|
1998
|
+
const hash = next.value;
|
|
1999
|
+
if (seen.has(hash)) {
|
|
2000
|
+
continue;
|
|
2001
|
+
}
|
|
2002
|
+
seen.add(hash);
|
|
2003
|
+
const authorization = pendingForTarget.get(hash);
|
|
2004
|
+
if (!authorization || authorization.active === true) {
|
|
2005
|
+
continue;
|
|
2006
|
+
}
|
|
2007
|
+
const batch = authorization.batch;
|
|
2008
|
+
const targetLifecycle = batch.targetLifecycle;
|
|
2009
|
+
if (
|
|
2010
|
+
batch.expiresAt <= Date.now() ||
|
|
2011
|
+
!this.isSyncDispatchLifecycleActive(
|
|
2012
|
+
targetLifecycle.lifecycle,
|
|
2013
|
+
fromHash,
|
|
2014
|
+
)
|
|
2015
|
+
) {
|
|
2016
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
2017
|
+
continue;
|
|
2018
|
+
}
|
|
2019
|
+
if (pendingForTarget.get(hash)?.batch !== batch) {
|
|
2020
|
+
continue;
|
|
2021
|
+
}
|
|
2022
|
+
let accepted = acceptedByLifecycle.get(targetLifecycle);
|
|
2023
|
+
if (!accepted) {
|
|
2024
|
+
accepted = { hashes: [], authorizations: [] };
|
|
2025
|
+
acceptedByLifecycle.set(targetLifecycle, accepted);
|
|
2026
|
+
targetLifecycle.responseLeases += 1;
|
|
2027
|
+
targetLifecycle.lifecycle.retainedWork += 1;
|
|
2028
|
+
}
|
|
2029
|
+
authorization.active = true;
|
|
2030
|
+
batch.hashes.delete(hash);
|
|
2031
|
+
accepted.hashes.push(hash);
|
|
2032
|
+
accepted.authorizations.push(authorization);
|
|
2033
|
+
if (batch.hashes.size === 0) {
|
|
2034
|
+
this.removePendingMaybeSyncResponseBatch(batch);
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
} finally {
|
|
2038
|
+
if (!exhausted) {
|
|
2039
|
+
iterator.return?.();
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
if (pendingForTarget.size === 0) {
|
|
2043
|
+
this.pendingMaybeSyncResponses.delete(fromHash);
|
|
2044
|
+
}
|
|
2045
|
+
if (acceptedByLifecycle.size === 0) {
|
|
2046
|
+
return [];
|
|
2047
|
+
}
|
|
2048
|
+
this.activeMaybeSyncResponseCount += 1;
|
|
2049
|
+
this.activeMaybeSyncResponseCountByPeer.set(
|
|
2050
|
+
fromHash,
|
|
2051
|
+
(this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 0) + 1,
|
|
2052
|
+
);
|
|
2053
|
+
let remainingLeases = acceptedByLifecycle.size;
|
|
2054
|
+
return [...acceptedByLifecycle].map(
|
|
2055
|
+
([targetLifecycle, { hashes: acceptedHashes, authorizations }]) => {
|
|
2056
|
+
let released = false;
|
|
2057
|
+
return {
|
|
2058
|
+
hashes: acceptedHashes,
|
|
2059
|
+
signal: targetLifecycle.controller.signal,
|
|
2060
|
+
release: (options?: { fulfilled?: boolean }) => {
|
|
2061
|
+
if (released) {
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
released = true;
|
|
2065
|
+
const pendingForTarget =
|
|
2066
|
+
this.pendingMaybeSyncResponses.get(fromHash);
|
|
2067
|
+
for (const authorization of authorizations) {
|
|
2068
|
+
this.settlePendingMaybeSyncResponseAuthorization(
|
|
2069
|
+
authorization,
|
|
2070
|
+
options?.fulfilled === true,
|
|
2071
|
+
);
|
|
2072
|
+
if (pendingForTarget?.get(authorization.hash) === authorization) {
|
|
2073
|
+
pendingForTarget.delete(authorization.hash);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
if (pendingForTarget?.size === 0) {
|
|
2077
|
+
this.pendingMaybeSyncResponses.delete(fromHash);
|
|
2078
|
+
}
|
|
2079
|
+
this.pendingMaybeSyncResponseCount -= authorizations.filter(
|
|
2080
|
+
(authorization) => authorization.deliveryInFlight !== true,
|
|
2081
|
+
).length;
|
|
2082
|
+
targetLifecycle.responseLeases -= 1;
|
|
2083
|
+
targetLifecycle.lifecycle.retainedWork -= 1;
|
|
2084
|
+
remainingLeases -= 1;
|
|
2085
|
+
if (remainingLeases === 0) {
|
|
2086
|
+
this.activeMaybeSyncResponseCount -= 1;
|
|
2087
|
+
const activeForPeer =
|
|
2088
|
+
(this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 1) -
|
|
2089
|
+
1;
|
|
2090
|
+
if (activeForPeer === 0) {
|
|
2091
|
+
this.activeMaybeSyncResponseCountByPeer.delete(fromHash);
|
|
2092
|
+
} else {
|
|
2093
|
+
this.activeMaybeSyncResponseCountByPeer.set(
|
|
2094
|
+
fromHash,
|
|
2095
|
+
activeForPeer,
|
|
2096
|
+
);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
this.schedulePendingMaybeSyncResponseWaiter();
|
|
2100
|
+
this.maybeDisposeSyncDispatchLifecycle(targetLifecycle.lifecycle);
|
|
2101
|
+
},
|
|
2102
|
+
};
|
|
2103
|
+
},
|
|
2104
|
+
);
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
private isRepairSessionComplete(session: RepairSessionState): boolean {
|
|
2108
|
+
for (const state of session.targets.values()) {
|
|
2109
|
+
if (state.unresolved.size > 0) {
|
|
2110
|
+
return false;
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
return true;
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
private buildRepairSessionResult(
|
|
2117
|
+
session: RepairSessionState,
|
|
2118
|
+
completed: boolean,
|
|
2119
|
+
): RepairSessionResult[] {
|
|
2120
|
+
const durationMs = Date.now() - session.startedAt;
|
|
2121
|
+
const out: RepairSessionResult[] = [];
|
|
2122
|
+
for (const [target, state] of session.targets) {
|
|
2123
|
+
const unresolved = [...state.unresolved];
|
|
2124
|
+
out.push({
|
|
2125
|
+
target,
|
|
2126
|
+
requested: state.requestedCount,
|
|
2127
|
+
resolved: state.requestedCount - unresolved.length,
|
|
2128
|
+
unresolved,
|
|
2129
|
+
attempts: state.attempts,
|
|
2130
|
+
durationMs,
|
|
2131
|
+
completed,
|
|
2132
|
+
requestedTotal: state.requestedTotalCount,
|
|
2133
|
+
truncated: session.truncated,
|
|
2134
|
+
});
|
|
2135
|
+
}
|
|
2136
|
+
return out;
|
|
2137
|
+
}
|
|
2138
|
+
|
|
2139
|
+
private finalizeRepairSession(sessionId: string, completed: boolean): void {
|
|
2140
|
+
const session = this.repairSessions.get(sessionId);
|
|
2141
|
+
if (!session) {
|
|
2142
|
+
return;
|
|
2143
|
+
}
|
|
2144
|
+
this.repairSessions.delete(sessionId);
|
|
2145
|
+
session.cancelled = true;
|
|
2146
|
+
if (session.timer) {
|
|
2147
|
+
clearTimeout(session.timer);
|
|
2148
|
+
}
|
|
2149
|
+
session.deferred.resolve(this.buildRepairSessionResult(session, completed));
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
private async refreshRepairSessionState(sessionId: string): Promise<void> {
|
|
2153
|
+
const session = this.repairSessions.get(sessionId);
|
|
2154
|
+
if (!session) {
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
2157
|
+
for (const state of session.targets.values()) {
|
|
2158
|
+
const resolved =
|
|
2159
|
+
typeof this.log.hasMany === "function"
|
|
2160
|
+
? await this.log.hasMany(state.unresolved)
|
|
2161
|
+
: await this.getExistingRepairHashes(state.unresolved);
|
|
2162
|
+
for (const hash of resolved) {
|
|
2163
|
+
state.unresolved.delete(hash);
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
private async getExistingRepairHashes(
|
|
2169
|
+
hashes: Iterable<string>,
|
|
2170
|
+
): Promise<Set<string>> {
|
|
2171
|
+
const resolved = new Set<string>();
|
|
2172
|
+
for (const hash of hashes) {
|
|
2173
|
+
if (await this.log.has(hash)) {
|
|
2174
|
+
resolved.add(hash);
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
return resolved;
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
private markRepairSessionResolvedHashes(hashes: string[]): void {
|
|
2181
|
+
if (hashes.length === 0 || this.repairSessions.size === 0) {
|
|
2182
|
+
return;
|
|
2183
|
+
}
|
|
2184
|
+
for (const [sessionId, session] of this.repairSessions) {
|
|
2185
|
+
for (const state of session.targets.values()) {
|
|
2186
|
+
for (const hash of hashes) {
|
|
2187
|
+
state.unresolved.delete(hash);
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
if (this.isRepairSessionComplete(session)) {
|
|
2191
|
+
this.finalizeRepairSession(sessionId, true);
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
private markRepairSessionResolvedHash(hash: string): void {
|
|
2197
|
+
if (this.repairSessions.size === 0) {
|
|
2198
|
+
return;
|
|
2199
|
+
}
|
|
2200
|
+
for (const [sessionId, session] of this.repairSessions) {
|
|
2201
|
+
for (const state of session.targets.values()) {
|
|
2202
|
+
state.unresolved.delete(hash);
|
|
2203
|
+
}
|
|
2204
|
+
if (this.isRepairSessionComplete(session)) {
|
|
2205
|
+
this.finalizeRepairSession(sessionId, true);
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
private async runRepairSession(sessionId: string): Promise<void> {
|
|
2211
|
+
const session = this.repairSessions.get(sessionId);
|
|
2212
|
+
if (!session) {
|
|
2213
|
+
return;
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
let previousDelay = 0;
|
|
2217
|
+
for (const delayMs of session.retryIntervalsMs) {
|
|
2218
|
+
if (!this.repairSessions.has(sessionId) || this.closed) {
|
|
2219
|
+
return;
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
const waitMs = Math.max(0, delayMs - previousDelay);
|
|
2223
|
+
previousDelay = delayMs;
|
|
2224
|
+
if (waitMs > 0) {
|
|
2225
|
+
await new Promise<void>((resolve) => {
|
|
2226
|
+
const timer = setTimeout(resolve, waitMs);
|
|
2227
|
+
timer.unref?.();
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
if (!this.repairSessions.has(sessionId) || this.closed) {
|
|
2231
|
+
return;
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
await this.refreshRepairSessionState(sessionId);
|
|
2235
|
+
const current = this.repairSessions.get(sessionId);
|
|
2236
|
+
if (!current) {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
if (this.isRepairSessionComplete(current)) {
|
|
2240
|
+
this.finalizeRepairSession(sessionId, true);
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
|
|
2244
|
+
for (const [target, state] of current.targets) {
|
|
2245
|
+
if (state.unresolved.size === 0) {
|
|
2246
|
+
continue;
|
|
2247
|
+
}
|
|
2248
|
+
state.attempts += 1;
|
|
2249
|
+
try {
|
|
2250
|
+
await this.requestSync([...state.unresolved], [target], {
|
|
2251
|
+
targetEpochs: new Map([[target, state.targetEpoch]]),
|
|
2252
|
+
createTargetEpochs: false,
|
|
2253
|
+
});
|
|
2254
|
+
} catch {
|
|
2255
|
+
// Best-effort: keep unresolved and let retries/timeout determine outcome.
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
|
|
2259
|
+
await this.refreshRepairSessionState(sessionId);
|
|
2260
|
+
const afterSend = this.repairSessions.get(sessionId);
|
|
2261
|
+
if (!afterSend) {
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
if (this.isRepairSessionComplete(afterSend)) {
|
|
2265
|
+
this.finalizeRepairSession(sessionId, true);
|
|
2266
|
+
return;
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
if (afterSend.mode === "best-effort") {
|
|
2270
|
+
this.finalizeRepairSession(sessionId, false);
|
|
2271
|
+
return;
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
for (;;) {
|
|
2276
|
+
if (!this.repairSessions.has(sessionId) || this.closed) {
|
|
2277
|
+
return;
|
|
2278
|
+
}
|
|
2279
|
+
await this.refreshRepairSessionState(sessionId);
|
|
2280
|
+
const current = this.repairSessions.get(sessionId);
|
|
2281
|
+
if (!current) {
|
|
2282
|
+
return;
|
|
2283
|
+
}
|
|
2284
|
+
if (this.isRepairSessionComplete(current)) {
|
|
2285
|
+
this.finalizeRepairSession(sessionId, true);
|
|
2286
|
+
return;
|
|
2287
|
+
}
|
|
2288
|
+
await new Promise<void>((resolve) => {
|
|
2289
|
+
const timer = setTimeout(resolve, SESSION_POLL_INTERVAL_MS);
|
|
2290
|
+
timer.unref?.();
|
|
2291
|
+
});
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
private getQueuedSyncKey(key: SyncableKey): SyncableKey | undefined {
|
|
2296
|
+
if (this.syncInFlightQueue.has(key)) {
|
|
2297
|
+
return key;
|
|
2298
|
+
}
|
|
2299
|
+
if (typeof key === "string") {
|
|
2300
|
+
for (const queuedKey of this.syncInFlightQueue.keys()) {
|
|
2301
|
+
if (
|
|
2302
|
+
typeof queuedKey === "bigint" &&
|
|
2303
|
+
this.coordinateToHash.get(queuedKey) === key
|
|
2304
|
+
) {
|
|
2305
|
+
return queuedKey;
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
return undefined;
|
|
2309
|
+
}
|
|
2310
|
+
const hash = this.coordinateToHash.get(key);
|
|
2311
|
+
return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
|
|
2312
|
+
}
|
|
2313
|
+
|
|
2314
|
+
startRepairSession(properties: {
|
|
2315
|
+
entries: Map<string, SyncEntryCoordinates<R>>;
|
|
2316
|
+
targets: string[];
|
|
2317
|
+
mode?: RepairSessionMode;
|
|
2318
|
+
timeoutMs?: number;
|
|
2319
|
+
retryIntervalsMs?: number[];
|
|
2320
|
+
}): RepairSession {
|
|
2321
|
+
const mode = properties.mode ?? "best-effort";
|
|
2322
|
+
const startedAt = Date.now();
|
|
2323
|
+
const timeoutMs = Math.max(
|
|
2324
|
+
1,
|
|
2325
|
+
Math.floor(
|
|
2326
|
+
properties.timeoutMs ??
|
|
2327
|
+
(mode === "convergent"
|
|
2328
|
+
? DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS
|
|
2329
|
+
: DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS),
|
|
2330
|
+
),
|
|
2331
|
+
);
|
|
2332
|
+
const retryIntervalsMs = this.normalizeRetryIntervals(
|
|
2333
|
+
mode,
|
|
2334
|
+
properties.retryIntervalsMs,
|
|
2335
|
+
);
|
|
2336
|
+
const allHashes = this.getPrioritizedHashes(properties.entries);
|
|
2337
|
+
const trackedHashes =
|
|
2338
|
+
mode === "convergent" &&
|
|
2339
|
+
allHashes.length > this.maxConvergentTrackedHashes
|
|
2340
|
+
? allHashes.slice(0, this.maxConvergentTrackedHashes)
|
|
2341
|
+
: allHashes;
|
|
2342
|
+
const truncated = trackedHashes.length < allHashes.length;
|
|
2343
|
+
const targets = [...new Set(properties.targets)];
|
|
2344
|
+
const id = `repair-${++this.repairSessionCounter}`;
|
|
2345
|
+
const deferred = createDeferred<RepairSessionResult[]>();
|
|
2346
|
+
|
|
2347
|
+
const targetStates = new Map<string, RepairSessionTargetState>();
|
|
2348
|
+
for (const target of targets) {
|
|
2349
|
+
targetStates.set(target, {
|
|
2350
|
+
unresolved: new Set(trackedHashes),
|
|
2351
|
+
requestedCount: trackedHashes.length,
|
|
2352
|
+
requestedTotalCount: allHashes.length,
|
|
2353
|
+
attempts: 0,
|
|
2354
|
+
targetEpoch: this.getOrCreateSyncDispatchTargetEpoch(target),
|
|
2355
|
+
});
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
const session: RepairSessionState = {
|
|
2359
|
+
id,
|
|
2360
|
+
mode,
|
|
2361
|
+
startedAt,
|
|
2362
|
+
timeoutMs,
|
|
2363
|
+
retryIntervalsMs,
|
|
2364
|
+
targets: targetStates,
|
|
2365
|
+
truncated,
|
|
2366
|
+
deferred,
|
|
2367
|
+
cancelled: false,
|
|
2368
|
+
};
|
|
2369
|
+
|
|
2370
|
+
if (allHashes.length === 0 || targets.length === 0) {
|
|
2371
|
+
deferred.resolve(this.buildRepairSessionResult(session, true));
|
|
2372
|
+
return {
|
|
2373
|
+
id,
|
|
2374
|
+
done: deferred.promise,
|
|
2375
|
+
cancel: () => {
|
|
2376
|
+
// no-op
|
|
2377
|
+
},
|
|
2378
|
+
};
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
// For capped convergent sessions, still dispatch the full set once so large
|
|
2382
|
+
// repairs are not limited to tracked hashes.
|
|
2383
|
+
if (mode === "convergent" && truncated) {
|
|
2384
|
+
void this.onMaybeMissingEntries({
|
|
2385
|
+
entries: properties.entries,
|
|
2386
|
+
targets,
|
|
2387
|
+
}).catch(() => {
|
|
2388
|
+
// Best-effort: retries on tracked hashes continue via runRepairSession.
|
|
2389
|
+
});
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
session.timer = setTimeout(() => {
|
|
2393
|
+
this.finalizeRepairSession(id, false);
|
|
2394
|
+
}, timeoutMs);
|
|
2395
|
+
session.timer.unref?.();
|
|
2396
|
+
|
|
2397
|
+
this.repairSessions.set(id, session);
|
|
2398
|
+
void this.runRepairSession(id).catch(() => {
|
|
2399
|
+
this.finalizeRepairSession(id, false);
|
|
2400
|
+
});
|
|
2401
|
+
|
|
2402
|
+
return {
|
|
2403
|
+
id,
|
|
2404
|
+
done: deferred.promise,
|
|
2405
|
+
cancel: () => {
|
|
2406
|
+
this.finalizeRepairSession(id, false);
|
|
2407
|
+
},
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
async onMaybeMissingEntries(properties: {
|
|
2412
|
+
entries: Map<string, SyncEntryCoordinates<R>>;
|
|
2413
|
+
targets: string[];
|
|
2414
|
+
signal?: AbortSignal;
|
|
2415
|
+
}): Promise<void> {
|
|
2416
|
+
if (properties.signal?.aborted) {
|
|
2417
|
+
return;
|
|
2418
|
+
}
|
|
2419
|
+
await this.onMaybeMissingHashes({
|
|
2420
|
+
hashes: this.getPrioritizedHashes(properties.entries),
|
|
2421
|
+
targets: properties.targets,
|
|
2422
|
+
signal: properties.signal,
|
|
2423
|
+
});
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
async onMaybeMissingHashes(properties: {
|
|
2427
|
+
hashes: Iterable<string>;
|
|
2428
|
+
targets: string[];
|
|
2429
|
+
signal?: AbortSignal;
|
|
2430
|
+
}): Promise<void> {
|
|
2431
|
+
const targets = [...new Set(properties.targets)];
|
|
2432
|
+
const lifecycle = this.captureSyncDispatchLifecycle(
|
|
2433
|
+
targets,
|
|
2434
|
+
properties.signal,
|
|
2435
|
+
);
|
|
2436
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
|
|
2437
|
+
this.finishSyncDispatchLifecycle(lifecycle);
|
|
2438
|
+
return;
|
|
2439
|
+
}
|
|
2440
|
+
const profile = this.syncOptions?.profile;
|
|
2441
|
+
const startedAt = syncProfileStart(profile);
|
|
2442
|
+
const hashes = [...new Set(properties.hashes)];
|
|
2443
|
+
const targetsPerAuthorizationWindow = Math.max(
|
|
2444
|
+
1,
|
|
2445
|
+
Math.min(targets.length, MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES),
|
|
2446
|
+
);
|
|
2447
|
+
const chunks = this.chunk(
|
|
2448
|
+
hashes,
|
|
2449
|
+
Math.max(
|
|
2450
|
+
1,
|
|
2451
|
+
Math.min(
|
|
2452
|
+
this.maxHashesPerMessage,
|
|
2453
|
+
Math.floor(
|
|
2454
|
+
MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES /
|
|
2455
|
+
targetsPerAuthorizationWindow,
|
|
2456
|
+
),
|
|
2457
|
+
),
|
|
2458
|
+
),
|
|
2459
|
+
);
|
|
2460
|
+
let messages = 0;
|
|
2461
|
+
try {
|
|
2462
|
+
for (const chunk of chunks) {
|
|
2463
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
|
|
2464
|
+
break;
|
|
2465
|
+
}
|
|
2466
|
+
for (const target of targets) {
|
|
2467
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
2468
|
+
continue;
|
|
2469
|
+
}
|
|
2470
|
+
let hashesToAuthorize = chunk;
|
|
2471
|
+
while (
|
|
2472
|
+
hashesToAuthorize.length > 0 &&
|
|
2473
|
+
this.isSyncDispatchLifecycleActive(lifecycle, target)
|
|
2474
|
+
) {
|
|
2475
|
+
const reserved = await this.reservePendingMaybeSyncResponse({
|
|
2476
|
+
hashes: hashesToAuthorize,
|
|
2477
|
+
targets: [target],
|
|
2478
|
+
lifecycle,
|
|
2479
|
+
});
|
|
2480
|
+
if (
|
|
2481
|
+
!reserved ||
|
|
2482
|
+
!this.isSyncDispatchLifecycleActive(lifecycle, target)
|
|
2483
|
+
) {
|
|
2484
|
+
break;
|
|
2485
|
+
}
|
|
2486
|
+
const { reservation, conflicts } = reserved;
|
|
2487
|
+
const hashesToSend =
|
|
2488
|
+
reservation.newlyAuthorizedByTarget.get(target) ?? [];
|
|
2489
|
+
if (!reservation.retained()) {
|
|
2490
|
+
reservation.release();
|
|
2491
|
+
break;
|
|
2492
|
+
}
|
|
2493
|
+
if (hashesToSend.length > 0) {
|
|
2494
|
+
reservation.beginDelivery();
|
|
2495
|
+
try {
|
|
2496
|
+
await this.rpc.send(
|
|
2497
|
+
new RequestMaybeSync({ hashes: hashesToSend }),
|
|
2498
|
+
{
|
|
2499
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2500
|
+
mode: new SilentDelivery({
|
|
2501
|
+
to: [target],
|
|
2502
|
+
redundancy: 1,
|
|
2503
|
+
}),
|
|
2504
|
+
signal: this.getSyncDispatchSignal(lifecycle, target),
|
|
2505
|
+
},
|
|
2506
|
+
);
|
|
2507
|
+
reservation.markDelivered();
|
|
2508
|
+
messages += 1;
|
|
2509
|
+
} catch (error) {
|
|
2510
|
+
reservation.release();
|
|
2511
|
+
if (
|
|
2512
|
+
!this.isSyncDispatchLifecycleActive(lifecycle) ||
|
|
2513
|
+
!this.isSyncDispatchLifecycleActive(lifecycle, target)
|
|
2514
|
+
) {
|
|
2515
|
+
break;
|
|
2516
|
+
}
|
|
2517
|
+
throw error;
|
|
2518
|
+
} finally {
|
|
2519
|
+
reservation.finishDelivery();
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
hashesToAuthorize =
|
|
2523
|
+
await this.waitForPendingMaybeSyncResponseConflicts(
|
|
2524
|
+
conflicts,
|
|
2525
|
+
lifecycle,
|
|
2526
|
+
target,
|
|
2527
|
+
hashesToAuthorize.length,
|
|
2528
|
+
);
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
|
|
2532
|
+
break;
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
} finally {
|
|
2536
|
+
this.finishSyncDispatchLifecycle(lifecycle);
|
|
2537
|
+
if (profile) {
|
|
2538
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
2539
|
+
name: "simple.onMaybeMissingEntries",
|
|
2540
|
+
entries: hashes.length,
|
|
2541
|
+
messages,
|
|
2542
|
+
targets: targets.length,
|
|
2543
|
+
details: !this.isSyncDispatchLifecycleActive(lifecycle)
|
|
2544
|
+
? { cancelled: true }
|
|
2545
|
+
: undefined,
|
|
2546
|
+
});
|
|
2547
|
+
}
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
|
|
2551
|
+
/**
|
|
2552
|
+
* Ship exchange heads to one peer: the fused native raw path when the
|
|
2553
|
+
* peer advertised raw capability and the shared log provided a fused
|
|
2554
|
+
* sender, otherwise the TS message path (raw or plain by capability).
|
|
2555
|
+
* Returns the number of messages sent and whether the fused path ran.
|
|
2556
|
+
*/
|
|
2557
|
+
private async shipExchangeHeads(
|
|
2558
|
+
hashes: string[],
|
|
2559
|
+
to: PublicSignKey,
|
|
2560
|
+
canReceiveRaw: boolean,
|
|
2561
|
+
signal?: AbortSignal,
|
|
2562
|
+
): Promise<{ messages: number; fused: boolean }> {
|
|
2563
|
+
if (signal?.aborted) {
|
|
2564
|
+
return { messages: 0, fused: false };
|
|
2565
|
+
}
|
|
2566
|
+
if (canReceiveRaw && this.sendRawExchangeHeads) {
|
|
2567
|
+
let sentMessages: number | undefined;
|
|
2568
|
+
try {
|
|
2569
|
+
sentMessages = await this.sendRawExchangeHeads(
|
|
2570
|
+
hashes,
|
|
2571
|
+
[to.hashcode()],
|
|
2572
|
+
{ signal },
|
|
2573
|
+
);
|
|
2574
|
+
} catch (error) {
|
|
2575
|
+
if (signal?.aborted) {
|
|
2576
|
+
return { messages: 0, fused: true };
|
|
2577
|
+
}
|
|
2578
|
+
throw error;
|
|
2579
|
+
}
|
|
2580
|
+
if (sentMessages !== undefined) {
|
|
2581
|
+
return { messages: sentMessages, fused: true };
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
let messages = 0;
|
|
2585
|
+
const messageGenerator = canReceiveRaw
|
|
2586
|
+
? createRawExchangeHeadsMessages(
|
|
2587
|
+
this.log,
|
|
2588
|
+
hashes,
|
|
2589
|
+
this.syncOptions?.profile,
|
|
2590
|
+
)
|
|
2591
|
+
: createExchangeHeadsMessages(this.log, hashes);
|
|
2592
|
+
for await (const message of messageGenerator) {
|
|
2593
|
+
if (signal?.aborted) {
|
|
2594
|
+
break;
|
|
2595
|
+
}
|
|
2596
|
+
try {
|
|
2597
|
+
await this.rpc.send(message, {
|
|
2598
|
+
mode: new SilentDelivery({ to: [to], redundancy: 1 }),
|
|
2599
|
+
signal,
|
|
2600
|
+
});
|
|
2601
|
+
messages += 1;
|
|
2602
|
+
} catch (error) {
|
|
2603
|
+
if (signal?.aborted) {
|
|
2604
|
+
break;
|
|
2605
|
+
}
|
|
2606
|
+
throw error;
|
|
2607
|
+
}
|
|
2608
|
+
}
|
|
2609
|
+
return { messages, fused: false };
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
/**
|
|
2613
|
+
* Ships hashes that the caller has already authorized for this exact sender.
|
|
2614
|
+
*
|
|
2615
|
+
* This deliberately does not consult or extend Simple's bounded pending-response
|
|
2616
|
+
* window. Rateless sync owns a separate, target-scoped authorization lifecycle
|
|
2617
|
+
* and uses this helper only after intersecting the response with that process's
|
|
2618
|
+
* exact advertised hash set.
|
|
2619
|
+
*/
|
|
2620
|
+
async shipAuthorizedMaybeSyncResponse(properties: {
|
|
2621
|
+
hashes: string[];
|
|
2622
|
+
from: PublicSignKey;
|
|
2623
|
+
response: ResponseMaybeSync | ResponseMaybeSyncCapabilities;
|
|
2624
|
+
signal: AbortSignal;
|
|
2625
|
+
}): Promise<{ messages: number; fused: boolean; entries: number }> {
|
|
2626
|
+
const hashes = this.filterRecentlySentExchangeHeads(
|
|
2627
|
+
properties.hashes,
|
|
2628
|
+
properties.from,
|
|
2629
|
+
);
|
|
2630
|
+
try {
|
|
2631
|
+
const shipped = await this.shipExchangeHeads(
|
|
2632
|
+
hashes,
|
|
2633
|
+
properties.from,
|
|
2634
|
+
canReceiveRawExchangeHeads(properties.response),
|
|
2635
|
+
properties.signal,
|
|
2636
|
+
);
|
|
2637
|
+
return {
|
|
2638
|
+
...shipped,
|
|
2639
|
+
entries: hashes.length,
|
|
2640
|
+
};
|
|
2641
|
+
} catch (error) {
|
|
2642
|
+
this.forgetRecentlySentExchangeHeads(hashes, properties.from);
|
|
2643
|
+
throw error;
|
|
2644
|
+
} finally {
|
|
2645
|
+
if (properties.signal.aborted) {
|
|
2646
|
+
this.forgetRecentlySentExchangeHeads(hashes, properties.from);
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
async shipAuthorizedMaybeSyncResponseLeases(properties: {
|
|
2652
|
+
leases: AuthorizedMaybeSyncResponseLease[];
|
|
2653
|
+
from: PublicSignKey;
|
|
2654
|
+
response: ResponseMaybeSync | ResponseMaybeSyncCapabilities;
|
|
2655
|
+
source?: string;
|
|
2656
|
+
}): Promise<{ messages: number; fused: boolean; entries: number }> {
|
|
2657
|
+
if (properties.leases.length === 0) {
|
|
2658
|
+
return { messages: 0, fused: false, entries: 0 };
|
|
2659
|
+
}
|
|
2660
|
+
const profile = this.syncOptions?.profile;
|
|
2661
|
+
const startedAt = syncProfileStart(profile);
|
|
2662
|
+
let messages = 0;
|
|
2663
|
+
let fused = false;
|
|
2664
|
+
let entries = 0;
|
|
2665
|
+
let firstError: unknown;
|
|
2666
|
+
for (const lease of properties.leases) {
|
|
2667
|
+
let fulfilled = false;
|
|
2668
|
+
try {
|
|
2669
|
+
const shipped = await this.shipAuthorizedMaybeSyncResponse({
|
|
2670
|
+
hashes: lease.hashes,
|
|
2671
|
+
from: properties.from,
|
|
2672
|
+
response: properties.response,
|
|
2673
|
+
signal: lease.signal,
|
|
2674
|
+
});
|
|
2675
|
+
messages += shipped.messages;
|
|
2676
|
+
fused ||= shipped.fused;
|
|
2677
|
+
entries += shipped.entries;
|
|
2678
|
+
fulfilled = !lease.signal.aborted;
|
|
2679
|
+
} catch (error) {
|
|
2680
|
+
firstError ??= error;
|
|
2681
|
+
} finally {
|
|
2682
|
+
lease.release({ fulfilled });
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
if (profile) {
|
|
2686
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
2687
|
+
name: "simple.exchangeHeads",
|
|
2688
|
+
entries,
|
|
2689
|
+
messages,
|
|
2690
|
+
targets: 1,
|
|
2691
|
+
details: {
|
|
2692
|
+
source: properties.source ?? "responseMaybeSync",
|
|
2693
|
+
fused,
|
|
2694
|
+
},
|
|
2695
|
+
});
|
|
2696
|
+
}
|
|
2697
|
+
if (firstError !== undefined) {
|
|
2698
|
+
throw firstError;
|
|
2699
|
+
}
|
|
2700
|
+
return { messages, fused, entries };
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
async onMessage(
|
|
2704
|
+
msg: TransportMessage,
|
|
2705
|
+
context: RequestContext,
|
|
2706
|
+
): Promise<boolean> {
|
|
2707
|
+
const from = context.from!;
|
|
2708
|
+
if (msg instanceof RequestMaybeSync) {
|
|
2709
|
+
await this.queueSync(msg.hashes, from);
|
|
2710
|
+
return true;
|
|
2711
|
+
} else if (
|
|
2712
|
+
msg instanceof ResponseMaybeSync ||
|
|
2713
|
+
msg instanceof ResponseMaybeSyncCapabilities
|
|
2714
|
+
) {
|
|
2715
|
+
// TODO perhaps send less messages to more receivers for performance reasons?
|
|
2716
|
+
// TODO wait for previous send to target before trying to send more?
|
|
2717
|
+
|
|
2718
|
+
const pending = this.consumeAuthorizedMaybeSyncResponse(msg.hashes, from);
|
|
2719
|
+
if (pending.length === 0) {
|
|
2720
|
+
return true;
|
|
2721
|
+
}
|
|
2722
|
+
await this.shipAuthorizedMaybeSyncResponseLeases({
|
|
2723
|
+
leases: pending,
|
|
2724
|
+
from,
|
|
2725
|
+
response: msg,
|
|
2726
|
+
});
|
|
2727
|
+
return true;
|
|
2728
|
+
} else if (
|
|
2729
|
+
msg instanceof RequestMaybeSyncCoordinate ||
|
|
2730
|
+
msg instanceof RequestMaybeSyncCoordinateCapabilities
|
|
2731
|
+
) {
|
|
2732
|
+
if (
|
|
2733
|
+
msg.hashNumbers.length === 0 ||
|
|
2734
|
+
msg.hashNumbers.length > MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS
|
|
2735
|
+
) {
|
|
2736
|
+
return true;
|
|
2737
|
+
}
|
|
2738
|
+
const target = from.hashcode();
|
|
2739
|
+
const releaseLookup = this.tryAcquireCoordinateLookup(target);
|
|
2740
|
+
if (!releaseLookup) {
|
|
2741
|
+
return true;
|
|
2742
|
+
}
|
|
2743
|
+
const lifecycle = this.captureSyncDispatchLifecycle([target]);
|
|
2744
|
+
let lookupReleased = false;
|
|
2745
|
+
const finishLookup = () => {
|
|
2746
|
+
if (lookupReleased) {
|
|
2747
|
+
return;
|
|
2748
|
+
}
|
|
2749
|
+
lookupReleased = true;
|
|
2750
|
+
releaseLookup();
|
|
2751
|
+
};
|
|
2752
|
+
try {
|
|
2753
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
const symbols = [...new Set(msg.hashNumbers)];
|
|
2757
|
+
const profile = this.syncOptions?.profile;
|
|
2758
|
+
const lookupStartedAt = syncProfileStart(profile);
|
|
2759
|
+
const hashes = await getHashesFromSymbols(
|
|
2760
|
+
symbols,
|
|
2761
|
+
this.entryIndex,
|
|
2762
|
+
this.coordinateToHash,
|
|
2763
|
+
this.resolveHashesForSymbols,
|
|
2764
|
+
this.resolveHashListForSymbols,
|
|
2765
|
+
MAX_SIMPLE_COORDINATE_RESPONSE_HASHES,
|
|
2766
|
+
);
|
|
2767
|
+
finishLookup();
|
|
2768
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
2769
|
+
return true;
|
|
2770
|
+
}
|
|
2771
|
+
if (profile) {
|
|
2772
|
+
emitSyncProfileDuration(profile, lookupStartedAt, {
|
|
2773
|
+
name: "simple.coordinateLookup",
|
|
2774
|
+
entries: hashLookupResultSize(hashes),
|
|
2775
|
+
symbols: symbols.length,
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2779
|
+
const releaseResponse = this.tryAcquireCoordinateResponse(target);
|
|
2780
|
+
if (!releaseResponse) {
|
|
2781
|
+
return true;
|
|
2782
|
+
}
|
|
2783
|
+
const exchangeStartedAt = syncProfileStart(profile);
|
|
2784
|
+
let hashesToSend: string[] = [];
|
|
2785
|
+
let messages = 0;
|
|
2786
|
+
let fused = false;
|
|
2787
|
+
try {
|
|
2788
|
+
hashesToSend = this.filterRecentlySentExchangeHeads(hashes, from);
|
|
2789
|
+
// dont set priority 1 here because this will block other messages that should higher priority
|
|
2790
|
+
({ messages, fused } = await this.shipExchangeHeads(
|
|
2791
|
+
hashesToSend,
|
|
2792
|
+
context.from!,
|
|
2793
|
+
canReceiveRawExchangeHeads(msg),
|
|
2794
|
+
this.getSyncDispatchSignal(lifecycle, target),
|
|
2795
|
+
));
|
|
2796
|
+
} finally {
|
|
2797
|
+
releaseResponse();
|
|
2798
|
+
if (profile) {
|
|
2799
|
+
emitSyncProfileDuration(profile, exchangeStartedAt, {
|
|
2800
|
+
name: "simple.exchangeHeads",
|
|
2801
|
+
entries: hashesToSend.length,
|
|
2802
|
+
messages,
|
|
2803
|
+
targets: 1,
|
|
2804
|
+
details: {
|
|
2805
|
+
source: "requestMaybeSyncCoordinate",
|
|
2806
|
+
fused,
|
|
2807
|
+
},
|
|
2808
|
+
});
|
|
2809
|
+
}
|
|
2810
|
+
}
|
|
2811
|
+
|
|
2812
|
+
return true;
|
|
2813
|
+
} finally {
|
|
2814
|
+
finishLookup();
|
|
2815
|
+
this.finishSyncDispatchLifecycle(lifecycle);
|
|
2816
|
+
}
|
|
2817
|
+
} else {
|
|
2818
|
+
return false; // no message was consumed
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
onReceivedEntries(properties: {
|
|
2823
|
+
entries: EntryWithRefs<any>[];
|
|
2824
|
+
from: PublicSignKey;
|
|
2825
|
+
}): Promise<void> | void {
|
|
2826
|
+
return this.onReceivedEntryHashes({
|
|
2827
|
+
hashes: properties.entries.map((entry) => entry.entry.hash),
|
|
2828
|
+
from: properties.from,
|
|
2829
|
+
});
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
onReceivedEntryHashes(properties: {
|
|
2833
|
+
hashes: string[];
|
|
2834
|
+
from: PublicSignKey;
|
|
2835
|
+
}): Promise<void> | void {
|
|
2836
|
+
this.clearSyncInFlightForPeerHashes(
|
|
2837
|
+
properties.from.hashcode(),
|
|
2838
|
+
properties.hashes,
|
|
2839
|
+
);
|
|
2840
|
+
this.markRepairSessionResolvedHashes(properties.hashes);
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2843
|
+
private getPendingSyncKeyIdentity(key: SyncableKey): SyncableKey {
|
|
2844
|
+
if (typeof key === "string") {
|
|
2845
|
+
return key;
|
|
2846
|
+
}
|
|
2847
|
+
const hash = this.coordinateToHash.get(key);
|
|
2848
|
+
return hash ?? key;
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
private removeQueuedSyncCoordinateAlias(key: bigint): void {
|
|
2852
|
+
this.syncInFlightQueuedCoordinates.delete(key);
|
|
2853
|
+
if (this.syncInFlightQueuedCoordinates.size === 0) {
|
|
2854
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
2855
|
+
}
|
|
2856
|
+
const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
|
|
2857
|
+
this.syncInFlightQueuedHashByCoordinate.delete(key);
|
|
2858
|
+
if (previousHash != null) {
|
|
2859
|
+
const coordinates =
|
|
2860
|
+
this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
|
|
2861
|
+
coordinates?.delete(key);
|
|
2862
|
+
if (coordinates?.size === 0) {
|
|
2863
|
+
this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
|
|
2864
|
+
}
|
|
2865
|
+
}
|
|
2866
|
+
}
|
|
2867
|
+
|
|
2868
|
+
private refreshQueuedSyncCoordinateAlias(key: bigint): void {
|
|
2869
|
+
if (!this.syncInFlightQueue.has(key)) {
|
|
2870
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
2871
|
+
return;
|
|
2872
|
+
}
|
|
2873
|
+
this.syncInFlightQueuedCoordinates.add(key);
|
|
2874
|
+
const hash = this.coordinateToHash.get(key) ?? undefined;
|
|
2875
|
+
const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
|
|
2876
|
+
if (previousHash !== hash) {
|
|
2877
|
+
if (previousHash != null) {
|
|
2878
|
+
const coordinates =
|
|
2879
|
+
this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
|
|
2880
|
+
coordinates?.delete(key);
|
|
2881
|
+
if (coordinates?.size === 0) {
|
|
2882
|
+
this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
|
|
2883
|
+
}
|
|
2884
|
+
}
|
|
2885
|
+
if (hash == null) {
|
|
2886
|
+
this.syncInFlightQueuedHashByCoordinate.delete(key);
|
|
2887
|
+
} else {
|
|
2888
|
+
this.syncInFlightQueuedHashByCoordinate.set(key, hash);
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
if (hash != null) {
|
|
2892
|
+
let coordinates = this.syncInFlightQueuedCoordinatesByHash.get(hash);
|
|
2893
|
+
if (!coordinates) {
|
|
2894
|
+
coordinates = new Set();
|
|
2895
|
+
this.syncInFlightQueuedCoordinatesByHash.set(hash, coordinates);
|
|
2896
|
+
}
|
|
2897
|
+
coordinates.add(key);
|
|
2898
|
+
this.reconcileQueuedSyncCoordinateAlias(key, hash);
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
|
|
2902
|
+
private reconcileQueuedSyncCoordinateAlias(
|
|
2903
|
+
coordinate: bigint,
|
|
2904
|
+
hash: string,
|
|
2905
|
+
): void {
|
|
2906
|
+
const now = Date.now();
|
|
2907
|
+
const coordinateExpiresAt = this.syncInFlightQueueExpiresAt.get(coordinate);
|
|
2908
|
+
if (coordinateExpiresAt != null && coordinateExpiresAt <= now) {
|
|
2909
|
+
this.clearSyncProcessKey(coordinate);
|
|
2910
|
+
return;
|
|
2911
|
+
}
|
|
2912
|
+
const hashExpiresAt = this.syncInFlightQueueExpiresAt.get(hash);
|
|
2913
|
+
if (hashExpiresAt != null && hashExpiresAt <= now) {
|
|
2914
|
+
this.clearSyncProcessKey(hash);
|
|
2915
|
+
return;
|
|
2916
|
+
}
|
|
2917
|
+
const hashClaimants = this.syncInFlightQueue.get(hash);
|
|
2918
|
+
if (!hashClaimants || !this.syncInFlightQueue.has(coordinate)) {
|
|
2919
|
+
return;
|
|
2920
|
+
}
|
|
2921
|
+
const expiresAt = Math.min(
|
|
2922
|
+
this.syncInFlightQueueExpiresAt.get(coordinate) ?? Infinity,
|
|
2923
|
+
this.syncInFlightQueueExpiresAt.get(hash) ?? Infinity,
|
|
2924
|
+
);
|
|
2925
|
+
for (const claimant of [...hashClaimants]) {
|
|
2926
|
+
this.addPendingSyncClaim(coordinate, claimant, expiresAt);
|
|
2927
|
+
}
|
|
2928
|
+
if (Number.isFinite(expiresAt)) {
|
|
2929
|
+
this.movePendingSyncKeyExpiryEarlier(coordinate, expiresAt);
|
|
2930
|
+
}
|
|
2931
|
+
for (const target of [...(this.syncInFlightTargetsByKey.get(hash) ?? [])]) {
|
|
2932
|
+
const state = this.syncInFlight.get(target)?.get(hash);
|
|
2933
|
+
if (state) {
|
|
2934
|
+
this.setSyncInFlightTargetKey(target, coordinate, state.timestamp);
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
this.clearSyncProcessKey(hash);
|
|
2938
|
+
}
|
|
2939
|
+
|
|
2940
|
+
private refreshQueuedSyncCoordinateAliases(): void {
|
|
2941
|
+
if (
|
|
2942
|
+
this.syncInFlightQueuedCoordinates.size === 0 &&
|
|
2943
|
+
this.syncInFlightQueueClaimants.size < this.syncInFlightQueue.size
|
|
2944
|
+
) {
|
|
2945
|
+
// Defensive compatibility for callers/tests that seed the public queue
|
|
2946
|
+
// directly. Keep hydration bounded; internal writes register coordinates
|
|
2947
|
+
// when the key is first admitted.
|
|
2948
|
+
let inspected = 0;
|
|
2949
|
+
for (const key of this.syncInFlightQueue.keys()) {
|
|
2950
|
+
if (typeof key === "bigint") {
|
|
2951
|
+
this.syncInFlightQueuedCoordinates.add(key);
|
|
2952
|
+
}
|
|
2953
|
+
inspected += 1;
|
|
2954
|
+
if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
|
|
2955
|
+
break;
|
|
2956
|
+
}
|
|
2957
|
+
}
|
|
2958
|
+
}
|
|
2959
|
+
if (this.syncInFlightQueuedCoordinates.size === 0) {
|
|
2960
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
2961
|
+
return;
|
|
2962
|
+
}
|
|
2963
|
+
this.syncInFlightQueuedCoordinateRefreshIterator ??=
|
|
2964
|
+
this.syncInFlightQueuedCoordinates.values();
|
|
2965
|
+
for (
|
|
2966
|
+
let refreshed = 0;
|
|
2967
|
+
refreshed < MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE;
|
|
2968
|
+
refreshed += 1
|
|
2969
|
+
) {
|
|
2970
|
+
const next = this.syncInFlightQueuedCoordinateRefreshIterator.next();
|
|
2971
|
+
if (next.done) {
|
|
2972
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
2973
|
+
break;
|
|
2974
|
+
}
|
|
2975
|
+
this.refreshQueuedSyncCoordinateAlias(next.value);
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
private getQueuedSyncKeyForAdmission(
|
|
2980
|
+
key: SyncableKey,
|
|
2981
|
+
): SyncableKey | typeof QUEUED_SYNC_ALIAS_REFRESH_PENDING | undefined {
|
|
2982
|
+
const getValidQueuedKey = (
|
|
2983
|
+
candidate: SyncableKey,
|
|
2984
|
+
): SyncableKey | undefined => {
|
|
2985
|
+
if (!this.syncInFlightQueue.has(candidate)) {
|
|
2986
|
+
return undefined;
|
|
2987
|
+
}
|
|
2988
|
+
const expiresAt = this.syncInFlightQueueExpiresAt.get(candidate);
|
|
2989
|
+
if (expiresAt != null && expiresAt <= Date.now()) {
|
|
2990
|
+
this.clearSyncProcessKey(candidate);
|
|
2991
|
+
return undefined;
|
|
2992
|
+
}
|
|
2993
|
+
return candidate;
|
|
2994
|
+
};
|
|
2995
|
+
if (getValidQueuedKey(key) != null) {
|
|
2996
|
+
if (typeof key === "bigint") {
|
|
2997
|
+
this.refreshQueuedSyncCoordinateAlias(key);
|
|
2998
|
+
return getValidQueuedKey(key);
|
|
2999
|
+
}
|
|
649
3000
|
return key;
|
|
650
3001
|
}
|
|
651
3002
|
if (typeof key === "string") {
|
|
652
|
-
|
|
3003
|
+
const aliases = this.syncInFlightQueuedCoordinatesByHash.get(key);
|
|
3004
|
+
if (aliases) {
|
|
3005
|
+
let inspected = 0;
|
|
3006
|
+
for (const alias of aliases) {
|
|
3007
|
+
if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
|
|
3008
|
+
return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
|
|
3009
|
+
}
|
|
3010
|
+
inspected += 1;
|
|
3011
|
+
this.refreshQueuedSyncCoordinateAlias(alias);
|
|
3012
|
+
if (
|
|
3013
|
+
this.syncInFlightQueuedCoordinatesByHash.get(key)?.has(alias) ===
|
|
3014
|
+
true
|
|
3015
|
+
) {
|
|
3016
|
+
const validAlias = getValidQueuedKey(alias);
|
|
3017
|
+
if (validAlias != null) {
|
|
3018
|
+
return validAlias;
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
653
3022
|
if (
|
|
654
|
-
|
|
655
|
-
this.coordinateToHash.get(queuedKey) === key
|
|
3023
|
+
(this.syncInFlightQueuedCoordinatesByHash.get(key)?.size ?? 0) > 0
|
|
656
3024
|
) {
|
|
657
|
-
return
|
|
3025
|
+
return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
|
|
658
3026
|
}
|
|
659
3027
|
}
|
|
660
3028
|
return undefined;
|
|
661
3029
|
}
|
|
662
3030
|
const hash = this.coordinateToHash.get(key);
|
|
663
|
-
return hash
|
|
3031
|
+
return hash != null ? getValidQueuedKey(hash) : undefined;
|
|
664
3032
|
}
|
|
665
3033
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
)
|
|
3034
|
+
private addPendingSyncClaim(
|
|
3035
|
+
key: SyncableKey,
|
|
3036
|
+
from: PublicSignKey,
|
|
3037
|
+
expiresAt?: number,
|
|
3038
|
+
): boolean {
|
|
3039
|
+
const fromHash = from.hashcode();
|
|
3040
|
+
let peers = this.syncInFlightQueue.get(key);
|
|
3041
|
+
let claimants = this.syncInFlightQueueClaimants.get(key);
|
|
3042
|
+
let claimantIndexes = this.syncInFlightQueueClaimantIndexes.get(key);
|
|
3043
|
+
if (!peers) {
|
|
3044
|
+
peers = [];
|
|
3045
|
+
this.syncInFlightQueue.set(key, peers);
|
|
3046
|
+
claimants = new Set();
|
|
3047
|
+
this.syncInFlightQueueClaimants.set(key, claimants);
|
|
3048
|
+
claimantIndexes = new Map();
|
|
3049
|
+
this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
|
|
3050
|
+
const deadline = expiresAt ?? Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS;
|
|
3051
|
+
this.syncInFlightQueueExpiresAt.set(key, deadline);
|
|
3052
|
+
const expiryNode: PendingSyncExpiryNode = {
|
|
3053
|
+
kind: "key",
|
|
3054
|
+
key,
|
|
3055
|
+
expiresAt: deadline,
|
|
3056
|
+
heapIndex: -1,
|
|
3057
|
+
};
|
|
3058
|
+
this.pendingSyncKeyExpiryNodes.set(key, expiryNode);
|
|
3059
|
+
this.pushPendingSyncExpiry(expiryNode);
|
|
3060
|
+
if (typeof key === "bigint") {
|
|
3061
|
+
this.refreshQueuedSyncCoordinateAlias(key);
|
|
3062
|
+
}
|
|
3063
|
+
} else if (!claimants) {
|
|
3064
|
+
// Defensive compatibility for callers/tests that seed the public queue
|
|
3065
|
+
// maps directly. Internally every retained key gets this set at creation.
|
|
3066
|
+
claimants = new Set(peers.map((peer) => peer.hashcode()));
|
|
3067
|
+
this.syncInFlightQueueClaimants.set(key, claimants);
|
|
3068
|
+
this.pendingSyncClaimCount += claimants.size;
|
|
3069
|
+
}
|
|
3070
|
+
if (!claimantIndexes) {
|
|
3071
|
+
claimantIndexes = new Map();
|
|
3072
|
+
for (let index = 0; index < peers.length; index += 1) {
|
|
3073
|
+
claimantIndexes.set(peers[index]!.hashcode(), index);
|
|
3074
|
+
}
|
|
3075
|
+
this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
|
|
3076
|
+
}
|
|
3077
|
+
if (claimants.has(fromHash)) {
|
|
3078
|
+
return false;
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
claimantIndexes.set(fromHash, peers.length);
|
|
3082
|
+
peers.push(from);
|
|
3083
|
+
claimants.add(fromHash);
|
|
3084
|
+
let inverted = this.syncInFlightQueueInverted.get(fromHash);
|
|
3085
|
+
if (!inverted) {
|
|
3086
|
+
inverted = new Set();
|
|
3087
|
+
this.syncInFlightQueueInverted.set(fromHash, inverted);
|
|
3088
|
+
}
|
|
3089
|
+
inverted.add(key);
|
|
3090
|
+
this.pendingSyncClaimCount += 1;
|
|
3091
|
+
this.schedulePendingSyncKeyExpiry();
|
|
3092
|
+
return true;
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
private hasPendingSyncClaim(key: SyncableKey, peer: string): boolean {
|
|
3096
|
+
const claimants = this.syncInFlightQueueClaimants.get(key);
|
|
3097
|
+
if (claimants) {
|
|
3098
|
+
return claimants.has(peer);
|
|
3099
|
+
}
|
|
3100
|
+
const peers = this.syncInFlightQueue.get(key);
|
|
3101
|
+
if (!peers) {
|
|
3102
|
+
return false;
|
|
3103
|
+
}
|
|
3104
|
+
const hydrated = new Set(peers.map((candidate) => candidate.hashcode()));
|
|
3105
|
+
this.syncInFlightQueueClaimants.set(key, hydrated);
|
|
3106
|
+
const indexes = new Map<string, number>();
|
|
3107
|
+
for (let index = 0; index < peers.length; index += 1) {
|
|
3108
|
+
indexes.set(peers[index]!.hashcode(), index);
|
|
3109
|
+
}
|
|
3110
|
+
this.syncInFlightQueueClaimantIndexes.set(key, indexes);
|
|
3111
|
+
this.pendingSyncClaimCount += hydrated.size;
|
|
3112
|
+
return hydrated.has(peer);
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
private filterDispatchablePendingSyncClaims(
|
|
3116
|
+
keys: SyncableKey[],
|
|
3117
|
+
peer: string,
|
|
3118
|
+
epoch: SyncDispatchTargetEpoch,
|
|
3119
|
+
): SyncableKey[] {
|
|
3120
|
+
if (this.syncDispatchTargetEpochs.get(peer) !== epoch) {
|
|
3121
|
+
return [];
|
|
3122
|
+
}
|
|
3123
|
+
const now = Date.now();
|
|
3124
|
+
const dispatchable: SyncableKey[] = [];
|
|
3125
|
+
for (const key of keys) {
|
|
3126
|
+
const expiresAt = this.syncInFlightQueueExpiresAt.get(key);
|
|
3127
|
+
if (expiresAt != null && expiresAt <= now) {
|
|
3128
|
+
this.clearSyncProcessKey(key);
|
|
3129
|
+
continue;
|
|
3130
|
+
}
|
|
3131
|
+
if (
|
|
3132
|
+
this.syncInFlightQueue.has(key) &&
|
|
3133
|
+
this.hasPendingSyncClaim(key, peer)
|
|
3134
|
+
) {
|
|
3135
|
+
dispatchable.push(key);
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
return dispatchable;
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
private canStartPendingSyncLookup(peer: string): boolean {
|
|
3142
|
+
return (
|
|
3143
|
+
this.pendingSyncAdmissionReservations.size +
|
|
3144
|
+
this.pendingCoordinateLookupCount <
|
|
3145
|
+
MAX_PENDING_SIMPLE_SYNC_LOOKUPS_GLOBAL &&
|
|
3146
|
+
(this.pendingSyncAdmissionReservationsByPeer.get(peer)?.size ?? 0) +
|
|
3147
|
+
(this.pendingCoordinateLookupCountByPeer.get(peer) ?? 0) <
|
|
3148
|
+
MAX_PENDING_SIMPLE_SYNC_LOOKUPS_PER_PEER
|
|
683
3149
|
);
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
3150
|
+
}
|
|
3151
|
+
|
|
3152
|
+
private tryAcquireCoordinateLookup(peer: string): (() => void) | undefined {
|
|
3153
|
+
if (!this.canStartPendingSyncLookup(peer)) {
|
|
3154
|
+
return undefined;
|
|
3155
|
+
}
|
|
3156
|
+
this.pendingCoordinateLookupCount += 1;
|
|
3157
|
+
this.pendingCoordinateLookupCountByPeer.set(
|
|
3158
|
+
peer,
|
|
3159
|
+
(this.pendingCoordinateLookupCountByPeer.get(peer) ?? 0) + 1,
|
|
687
3160
|
);
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
3161
|
+
let released = false;
|
|
3162
|
+
return () => {
|
|
3163
|
+
if (released) {
|
|
3164
|
+
return;
|
|
3165
|
+
}
|
|
3166
|
+
released = true;
|
|
3167
|
+
this.pendingCoordinateLookupCount -= 1;
|
|
3168
|
+
const remaining =
|
|
3169
|
+
(this.pendingCoordinateLookupCountByPeer.get(peer) ?? 1) - 1;
|
|
3170
|
+
if (remaining === 0) {
|
|
3171
|
+
this.pendingCoordinateLookupCountByPeer.delete(peer);
|
|
3172
|
+
} else {
|
|
3173
|
+
this.pendingCoordinateLookupCountByPeer.set(peer, remaining);
|
|
3174
|
+
}
|
|
3175
|
+
};
|
|
3176
|
+
}
|
|
698
3177
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
3178
|
+
private tryAcquireCoordinateResponse(peer: string): (() => void) | undefined {
|
|
3179
|
+
if (
|
|
3180
|
+
this.pendingCoordinateResponseCount >=
|
|
3181
|
+
MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_GLOBAL ||
|
|
3182
|
+
(this.pendingCoordinateResponseCountByPeer.get(peer) ?? 0) >=
|
|
3183
|
+
MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_PER_PEER
|
|
3184
|
+
) {
|
|
3185
|
+
return undefined;
|
|
707
3186
|
}
|
|
3187
|
+
this.pendingCoordinateResponseCount += 1;
|
|
3188
|
+
this.pendingCoordinateResponseCountByPeer.set(
|
|
3189
|
+
peer,
|
|
3190
|
+
(this.pendingCoordinateResponseCountByPeer.get(peer) ?? 0) + 1,
|
|
3191
|
+
);
|
|
3192
|
+
let released = false;
|
|
3193
|
+
return () => {
|
|
3194
|
+
if (released) {
|
|
3195
|
+
return;
|
|
3196
|
+
}
|
|
3197
|
+
released = true;
|
|
3198
|
+
this.pendingCoordinateResponseCount -= 1;
|
|
3199
|
+
const remaining =
|
|
3200
|
+
(this.pendingCoordinateResponseCountByPeer.get(peer) ?? 1) - 1;
|
|
3201
|
+
if (remaining === 0) {
|
|
3202
|
+
this.pendingCoordinateResponseCountByPeer.delete(peer);
|
|
3203
|
+
} else {
|
|
3204
|
+
this.pendingCoordinateResponseCountByPeer.set(peer, remaining);
|
|
3205
|
+
}
|
|
3206
|
+
};
|
|
3207
|
+
}
|
|
708
3208
|
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
3209
|
+
private reservePendingSyncAdmission(
|
|
3210
|
+
peer: string,
|
|
3211
|
+
identities: SyncableKey[],
|
|
3212
|
+
): PendingSyncAdmissionReservation | undefined {
|
|
3213
|
+
const count = identities.length;
|
|
3214
|
+
if (count <= 0) {
|
|
3215
|
+
return undefined;
|
|
3216
|
+
}
|
|
3217
|
+
const reservation: PendingSyncAdmissionReservation = {
|
|
3218
|
+
peer,
|
|
3219
|
+
remaining: count,
|
|
3220
|
+
active: true,
|
|
3221
|
+
released: false,
|
|
3222
|
+
expiresAt: Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS,
|
|
3223
|
+
identities: new Set(identities),
|
|
3224
|
+
retainedSettled: 0,
|
|
3225
|
+
};
|
|
3226
|
+
this.pendingSyncAdmissionReservations.add(reservation);
|
|
3227
|
+
let peerReservations =
|
|
3228
|
+
this.pendingSyncAdmissionReservationsByPeer.get(peer);
|
|
3229
|
+
if (!peerReservations) {
|
|
3230
|
+
peerReservations = new Set();
|
|
3231
|
+
this.pendingSyncAdmissionReservationsByPeer.set(peer, peerReservations);
|
|
3232
|
+
}
|
|
3233
|
+
peerReservations.add(reservation);
|
|
3234
|
+
this.pendingSyncActiveAdmissionReservations += 1;
|
|
3235
|
+
this.pendingSyncAdmissionCount += count;
|
|
3236
|
+
this.pendingSyncAdmissionCountByPeer.set(
|
|
3237
|
+
peer,
|
|
3238
|
+
(this.pendingSyncAdmissionCountByPeer.get(peer) ?? 0) + count,
|
|
3239
|
+
);
|
|
3240
|
+
let reservedIdentities =
|
|
3241
|
+
this.pendingSyncAdmissionIdentitiesByPeer.get(peer);
|
|
3242
|
+
if (!reservedIdentities) {
|
|
3243
|
+
reservedIdentities = new Set();
|
|
3244
|
+
this.pendingSyncAdmissionIdentitiesByPeer.set(peer, reservedIdentities);
|
|
3245
|
+
}
|
|
3246
|
+
for (const identity of identities) {
|
|
3247
|
+
reservedIdentities.add(identity);
|
|
3248
|
+
let reservationsForIdentity =
|
|
3249
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
3250
|
+
if (!reservationsForIdentity) {
|
|
3251
|
+
reservationsForIdentity = new Set();
|
|
3252
|
+
this.pendingSyncAdmissionReservationsByIdentity.set(
|
|
3253
|
+
identity,
|
|
3254
|
+
reservationsForIdentity,
|
|
3255
|
+
);
|
|
3256
|
+
}
|
|
3257
|
+
reservationsForIdentity.add(reservation);
|
|
3258
|
+
}
|
|
3259
|
+
const expiryNode: PendingSyncExpiryNode = {
|
|
3260
|
+
kind: "admission",
|
|
3261
|
+
reservation,
|
|
3262
|
+
expiresAt: reservation.expiresAt,
|
|
3263
|
+
heapIndex: -1,
|
|
719
3264
|
};
|
|
3265
|
+
this.pendingSyncAdmissionExpiryNodes.set(reservation, expiryNode);
|
|
3266
|
+
this.pushPendingSyncExpiry(expiryNode);
|
|
3267
|
+
this.schedulePendingSyncKeyExpiry();
|
|
3268
|
+
return reservation;
|
|
3269
|
+
}
|
|
720
3270
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
3271
|
+
private removePendingSyncAdmissionIdentity(
|
|
3272
|
+
reservation: PendingSyncAdmissionReservation,
|
|
3273
|
+
identity: SyncableKey,
|
|
3274
|
+
options?: { retainQuota?: boolean },
|
|
3275
|
+
): boolean {
|
|
3276
|
+
if (!reservation.identities.delete(identity)) {
|
|
3277
|
+
return false;
|
|
3278
|
+
}
|
|
3279
|
+
if (options?.retainQuota === true) {
|
|
3280
|
+
reservation.retainedSettled += 1;
|
|
3281
|
+
} else {
|
|
3282
|
+
reservation.remaining -= 1;
|
|
3283
|
+
this.pendingSyncAdmissionCount -= 1;
|
|
3284
|
+
const peerCount =
|
|
3285
|
+
(this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) - 1;
|
|
3286
|
+
if (peerCount === 0) {
|
|
3287
|
+
this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
|
|
3288
|
+
} else {
|
|
3289
|
+
this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
const reservedIdentities = this.pendingSyncAdmissionIdentitiesByPeer.get(
|
|
3293
|
+
reservation.peer,
|
|
3294
|
+
);
|
|
3295
|
+
reservedIdentities?.delete(identity);
|
|
3296
|
+
if (reservedIdentities?.size === 0) {
|
|
3297
|
+
this.pendingSyncAdmissionIdentitiesByPeer.delete(reservation.peer);
|
|
3298
|
+
}
|
|
3299
|
+
const reservationsForIdentity =
|
|
3300
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
3301
|
+
reservationsForIdentity?.delete(reservation);
|
|
3302
|
+
if (reservationsForIdentity?.size === 0) {
|
|
3303
|
+
this.pendingSyncAdmissionReservationsByIdentity.delete(identity);
|
|
730
3304
|
}
|
|
3305
|
+
return true;
|
|
3306
|
+
}
|
|
731
3307
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
3308
|
+
private clearPendingSyncAdmissionIdentity(identity: SyncableKey): void {
|
|
3309
|
+
const reservations =
|
|
3310
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
3311
|
+
if (!reservations) {
|
|
3312
|
+
return;
|
|
3313
|
+
}
|
|
3314
|
+
for (const reservation of [...reservations]) {
|
|
3315
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity, {
|
|
3316
|
+
retainQuota: true,
|
|
740
3317
|
});
|
|
741
3318
|
}
|
|
3319
|
+
}
|
|
742
3320
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
3321
|
+
private consumePendingSyncAdmission(
|
|
3322
|
+
reservation: PendingSyncAdmissionReservation,
|
|
3323
|
+
identity: SyncableKey,
|
|
3324
|
+
): "consumed" | "settled" | "invalid" {
|
|
3325
|
+
if (!reservation.active || reservation.expiresAt <= Date.now()) {
|
|
3326
|
+
if (reservation.expiresAt <= Date.now()) {
|
|
3327
|
+
this.invalidatePendingSyncAdmission(reservation);
|
|
3328
|
+
}
|
|
3329
|
+
return "invalid";
|
|
3330
|
+
}
|
|
3331
|
+
if (!reservation.identities.has(identity)) {
|
|
3332
|
+
return "settled";
|
|
3333
|
+
}
|
|
3334
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity);
|
|
3335
|
+
if (reservation.remaining === 0) {
|
|
3336
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
3337
|
+
reservation.active = false;
|
|
3338
|
+
reservation.released = true;
|
|
3339
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
3340
|
+
this.pendingSyncAdmissionReservations.delete(reservation);
|
|
3341
|
+
const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
|
|
3342
|
+
reservation.peer,
|
|
3343
|
+
);
|
|
3344
|
+
peerReservations?.delete(reservation);
|
|
3345
|
+
if (peerReservations?.size === 0) {
|
|
3346
|
+
this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
|
|
3347
|
+
}
|
|
3348
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
3349
|
+
}
|
|
3350
|
+
return "consumed";
|
|
3351
|
+
}
|
|
752
3352
|
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
3353
|
+
private transferPendingSyncAdmissionIdentity(
|
|
3354
|
+
peer: string,
|
|
3355
|
+
identity: SyncableKey,
|
|
3356
|
+
): number | undefined {
|
|
3357
|
+
const reservations =
|
|
3358
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
3359
|
+
if (!reservations) {
|
|
3360
|
+
return undefined;
|
|
3361
|
+
}
|
|
3362
|
+
for (const reservation of reservations) {
|
|
3363
|
+
if (
|
|
3364
|
+
reservation.peer !== peer ||
|
|
3365
|
+
reservation.released ||
|
|
3366
|
+
reservation.expiresAt <= Date.now() ||
|
|
3367
|
+
!this.removePendingSyncAdmissionIdentity(reservation, identity, {
|
|
3368
|
+
retainQuota: true,
|
|
3369
|
+
})
|
|
3370
|
+
) {
|
|
3371
|
+
continue;
|
|
3372
|
+
}
|
|
3373
|
+
// The original resolver may be non-abortable and still retains its input
|
|
3374
|
+
// arrays. Keep that reservation charged until its queueSync finally
|
|
3375
|
+
// settles; the queued claim is counted separately and can disappear
|
|
3376
|
+
// without returning the resolver's quota early.
|
|
3377
|
+
return reservation.expiresAt;
|
|
3378
|
+
}
|
|
3379
|
+
return undefined;
|
|
760
3380
|
}
|
|
761
3381
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
3382
|
+
private invalidatePendingSyncAdmission(
|
|
3383
|
+
reservation?: PendingSyncAdmissionReservation,
|
|
3384
|
+
): void {
|
|
3385
|
+
if (!reservation || reservation.released || !reservation.active) {
|
|
3386
|
+
return;
|
|
3387
|
+
}
|
|
3388
|
+
// Expiry/disconnect invalidates late lookup results, but it must not return
|
|
3389
|
+
// the quota slot while the underlying storage/index work is still alive.
|
|
3390
|
+
// Those lookups are not generally abortable; only queueSync's finally block
|
|
3391
|
+
// may release their active-work accounting.
|
|
3392
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
3393
|
+
reservation.active = false;
|
|
3394
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
3395
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
770
3396
|
}
|
|
771
3397
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
const
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
} finally {
|
|
795
|
-
if (profile) {
|
|
796
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
797
|
-
name: "simple.onMaybeMissingEntries",
|
|
798
|
-
entries: hashes.length,
|
|
799
|
-
messages: chunks.length,
|
|
800
|
-
targets: properties.targets.length,
|
|
801
|
-
});
|
|
3398
|
+
private releasePendingSyncAdmission(
|
|
3399
|
+
reservation?: PendingSyncAdmissionReservation,
|
|
3400
|
+
): void {
|
|
3401
|
+
if (!reservation || reservation.released) {
|
|
3402
|
+
return;
|
|
3403
|
+
}
|
|
3404
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
3405
|
+
for (const identity of [...reservation.identities]) {
|
|
3406
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity);
|
|
3407
|
+
}
|
|
3408
|
+
if (reservation.retainedSettled > 0) {
|
|
3409
|
+
const retainedSettled = reservation.retainedSettled;
|
|
3410
|
+
reservation.retainedSettled = 0;
|
|
3411
|
+
reservation.remaining -= retainedSettled;
|
|
3412
|
+
this.pendingSyncAdmissionCount -= retainedSettled;
|
|
3413
|
+
const peerCount =
|
|
3414
|
+
(this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) -
|
|
3415
|
+
retainedSettled;
|
|
3416
|
+
if (peerCount === 0) {
|
|
3417
|
+
this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
|
|
3418
|
+
} else {
|
|
3419
|
+
this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
|
|
802
3420
|
}
|
|
803
3421
|
}
|
|
3422
|
+
if (reservation.active) {
|
|
3423
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
3424
|
+
}
|
|
3425
|
+
reservation.active = false;
|
|
3426
|
+
reservation.released = true;
|
|
3427
|
+
this.pendingSyncAdmissionReservations.delete(reservation);
|
|
3428
|
+
const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
|
|
3429
|
+
reservation.peer,
|
|
3430
|
+
);
|
|
3431
|
+
peerReservations?.delete(reservation);
|
|
3432
|
+
if (peerReservations?.size === 0) {
|
|
3433
|
+
this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
|
|
3434
|
+
}
|
|
3435
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
804
3436
|
}
|
|
805
3437
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
hashes: string[],
|
|
814
|
-
to: PublicSignKey,
|
|
815
|
-
canReceiveRaw: boolean,
|
|
816
|
-
): Promise<{ messages: number; fused: boolean }> {
|
|
817
|
-
if (canReceiveRaw && this.sendRawExchangeHeads) {
|
|
818
|
-
const sentMessages = await this.sendRawExchangeHeads(hashes, [
|
|
819
|
-
to.hashcode(),
|
|
820
|
-
]);
|
|
821
|
-
if (sentMessages !== undefined) {
|
|
822
|
-
return { messages: sentMessages, fused: true };
|
|
823
|
-
}
|
|
3438
|
+
private clearPendingSyncAdmissions(peer?: string): void {
|
|
3439
|
+
const reservations =
|
|
3440
|
+
peer == null
|
|
3441
|
+
? this.pendingSyncAdmissionReservations
|
|
3442
|
+
: this.pendingSyncAdmissionReservationsByPeer.get(peer);
|
|
3443
|
+
if (!reservations) {
|
|
3444
|
+
return;
|
|
824
3445
|
}
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
? createRawExchangeHeadsMessages(
|
|
828
|
-
this.log,
|
|
829
|
-
hashes,
|
|
830
|
-
this.syncOptions?.profile,
|
|
831
|
-
)
|
|
832
|
-
: createExchangeHeadsMessages(this.log, hashes);
|
|
833
|
-
for await (const message of messageGenerator) {
|
|
834
|
-
messages += 1;
|
|
835
|
-
await this.rpc.send(message, {
|
|
836
|
-
mode: new SilentDelivery({ to: [to], redundancy: 1 }),
|
|
837
|
-
});
|
|
3446
|
+
for (const reservation of [...reservations]) {
|
|
3447
|
+
this.invalidatePendingSyncAdmission(reservation);
|
|
838
3448
|
}
|
|
839
|
-
return { messages, fused: false };
|
|
840
3449
|
}
|
|
841
3450
|
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
} else if (
|
|
851
|
-
msg instanceof ResponseMaybeSync ||
|
|
852
|
-
msg instanceof ResponseMaybeSyncCapabilities
|
|
853
|
-
) {
|
|
854
|
-
// TODO perhaps send less messages to more receivers for performance reasons?
|
|
855
|
-
// TODO wait for previous send to target before trying to send more?
|
|
3451
|
+
private swapPendingSyncExpiry(left: number, right: number): void {
|
|
3452
|
+
const leftNode = this.pendingSyncExpiryHeap[left]!;
|
|
3453
|
+
const rightNode = this.pendingSyncExpiryHeap[right]!;
|
|
3454
|
+
this.pendingSyncExpiryHeap[left] = rightNode;
|
|
3455
|
+
this.pendingSyncExpiryHeap[right] = leftNode;
|
|
3456
|
+
rightNode.heapIndex = left;
|
|
3457
|
+
leftNode.heapIndex = right;
|
|
3458
|
+
}
|
|
856
3459
|
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
} finally {
|
|
869
|
-
if (profile) {
|
|
870
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
871
|
-
name: "simple.exchangeHeads",
|
|
872
|
-
entries: hashes.length,
|
|
873
|
-
messages,
|
|
874
|
-
targets: 1,
|
|
875
|
-
details: { source: "responseMaybeSync", fused },
|
|
876
|
-
});
|
|
877
|
-
}
|
|
3460
|
+
private pushPendingSyncExpiry(node: PendingSyncExpiryNode): void {
|
|
3461
|
+
node.heapIndex = this.pendingSyncExpiryHeap.length;
|
|
3462
|
+
this.pendingSyncExpiryHeap.push(node);
|
|
3463
|
+
let index = node.heapIndex;
|
|
3464
|
+
while (index > 0) {
|
|
3465
|
+
const parent = Math.floor((index - 1) / 2);
|
|
3466
|
+
if (
|
|
3467
|
+
this.pendingSyncExpiryHeap[parent]!.expiresAt <=
|
|
3468
|
+
this.pendingSyncExpiryHeap[index]!.expiresAt
|
|
3469
|
+
) {
|
|
3470
|
+
break;
|
|
878
3471
|
}
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
3472
|
+
this.swapPendingSyncExpiry(parent, index);
|
|
3473
|
+
index = parent;
|
|
3474
|
+
}
|
|
3475
|
+
}
|
|
3476
|
+
|
|
3477
|
+
private removePendingSyncExpiry(node: PendingSyncExpiryNode): void {
|
|
3478
|
+
const index = node.heapIndex;
|
|
3479
|
+
if (
|
|
3480
|
+
index < 0 ||
|
|
3481
|
+
index >= this.pendingSyncExpiryHeap.length ||
|
|
3482
|
+
this.pendingSyncExpiryHeap[index] !== node
|
|
883
3483
|
) {
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
3484
|
+
return;
|
|
3485
|
+
}
|
|
3486
|
+
const last = this.pendingSyncExpiryHeap.pop()!;
|
|
3487
|
+
node.heapIndex = -1;
|
|
3488
|
+
if (index >= this.pendingSyncExpiryHeap.length) {
|
|
3489
|
+
return;
|
|
3490
|
+
}
|
|
3491
|
+
this.pendingSyncExpiryHeap[index] = last;
|
|
3492
|
+
last.heapIndex = index;
|
|
3493
|
+
|
|
3494
|
+
let current = index;
|
|
3495
|
+
while (current > 0) {
|
|
3496
|
+
const parent = Math.floor((current - 1) / 2);
|
|
3497
|
+
if (
|
|
3498
|
+
this.pendingSyncExpiryHeap[parent]!.expiresAt <=
|
|
3499
|
+
this.pendingSyncExpiryHeap[current]!.expiresAt
|
|
3500
|
+
) {
|
|
3501
|
+
break;
|
|
3502
|
+
}
|
|
3503
|
+
this.swapPendingSyncExpiry(parent, current);
|
|
3504
|
+
current = parent;
|
|
3505
|
+
}
|
|
3506
|
+
for (;;) {
|
|
3507
|
+
const left = current * 2 + 1;
|
|
3508
|
+
const right = left + 1;
|
|
3509
|
+
let smallest = current;
|
|
3510
|
+
if (
|
|
3511
|
+
left < this.pendingSyncExpiryHeap.length &&
|
|
3512
|
+
this.pendingSyncExpiryHeap[left]!.expiresAt <
|
|
3513
|
+
this.pendingSyncExpiryHeap[smallest]!.expiresAt
|
|
3514
|
+
) {
|
|
3515
|
+
smallest = left;
|
|
3516
|
+
}
|
|
3517
|
+
if (
|
|
3518
|
+
right < this.pendingSyncExpiryHeap.length &&
|
|
3519
|
+
this.pendingSyncExpiryHeap[right]!.expiresAt <
|
|
3520
|
+
this.pendingSyncExpiryHeap[smallest]!.expiresAt
|
|
3521
|
+
) {
|
|
3522
|
+
smallest = right;
|
|
3523
|
+
}
|
|
3524
|
+
if (smallest === current) {
|
|
3525
|
+
break;
|
|
899
3526
|
}
|
|
3527
|
+
this.swapPendingSyncExpiry(current, smallest);
|
|
3528
|
+
current = smallest;
|
|
3529
|
+
}
|
|
3530
|
+
}
|
|
900
3531
|
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
3532
|
+
private removePendingSyncKeyExpiry(key: SyncableKey): void {
|
|
3533
|
+
const node = this.pendingSyncKeyExpiryNodes.get(key);
|
|
3534
|
+
if (!node) {
|
|
3535
|
+
return;
|
|
3536
|
+
}
|
|
3537
|
+
this.pendingSyncKeyExpiryNodes.delete(key);
|
|
3538
|
+
this.removePendingSyncExpiry(node);
|
|
3539
|
+
}
|
|
3540
|
+
|
|
3541
|
+
private movePendingSyncKeyExpiryEarlier(
|
|
3542
|
+
key: SyncableKey,
|
|
3543
|
+
expiresAt: number,
|
|
3544
|
+
): void {
|
|
3545
|
+
const current = this.syncInFlightQueueExpiresAt.get(key);
|
|
3546
|
+
if (current == null || current <= expiresAt) {
|
|
3547
|
+
return;
|
|
3548
|
+
}
|
|
3549
|
+
this.removePendingSyncKeyExpiry(key);
|
|
3550
|
+
this.syncInFlightQueueExpiresAt.set(key, expiresAt);
|
|
3551
|
+
const node: PendingSyncExpiryNode = {
|
|
3552
|
+
kind: "key",
|
|
3553
|
+
key,
|
|
3554
|
+
expiresAt,
|
|
3555
|
+
heapIndex: -1,
|
|
3556
|
+
};
|
|
3557
|
+
this.pendingSyncKeyExpiryNodes.set(key, node);
|
|
3558
|
+
this.pushPendingSyncExpiry(node);
|
|
3559
|
+
this.schedulePendingSyncKeyExpiry();
|
|
3560
|
+
}
|
|
3561
|
+
|
|
3562
|
+
private removePendingSyncAdmissionExpiry(
|
|
3563
|
+
reservation: PendingSyncAdmissionReservation,
|
|
3564
|
+
): void {
|
|
3565
|
+
const node = this.pendingSyncAdmissionExpiryNodes.get(reservation);
|
|
3566
|
+
if (!node) {
|
|
3567
|
+
return;
|
|
3568
|
+
}
|
|
3569
|
+
this.pendingSyncAdmissionExpiryNodes.delete(reservation);
|
|
3570
|
+
this.removePendingSyncExpiry(node);
|
|
3571
|
+
}
|
|
3572
|
+
|
|
3573
|
+
private expirePendingSyncKeys(now = Date.now()): void {
|
|
3574
|
+
for (;;) {
|
|
3575
|
+
const node = this.pendingSyncExpiryHeap[0];
|
|
3576
|
+
if (!node || node.expiresAt > now) {
|
|
3577
|
+
break;
|
|
3578
|
+
}
|
|
3579
|
+
this.removePendingSyncExpiry(node);
|
|
3580
|
+
if (node.kind === "key") {
|
|
3581
|
+
if (this.pendingSyncKeyExpiryNodes.get(node.key) !== node) {
|
|
3582
|
+
continue;
|
|
3583
|
+
}
|
|
3584
|
+
this.pendingSyncKeyExpiryNodes.delete(node.key);
|
|
3585
|
+
this.clearSyncProcessKey(node.key);
|
|
3586
|
+
} else {
|
|
3587
|
+
if (
|
|
3588
|
+
this.pendingSyncAdmissionExpiryNodes.get(node.reservation) !== node
|
|
3589
|
+
) {
|
|
3590
|
+
continue;
|
|
921
3591
|
}
|
|
3592
|
+
this.pendingSyncAdmissionExpiryNodes.delete(node.reservation);
|
|
3593
|
+
this.invalidatePendingSyncAdmission(node.reservation);
|
|
922
3594
|
}
|
|
923
|
-
|
|
924
|
-
return true;
|
|
925
|
-
} else {
|
|
926
|
-
return false; // no message was consumed
|
|
927
3595
|
}
|
|
928
3596
|
}
|
|
929
3597
|
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
}
|
|
3598
|
+
private clearPendingSyncExpiryTimerIfIdle(): void {
|
|
3599
|
+
if (
|
|
3600
|
+
this.pendingSyncExpiryHeap.length === 0 &&
|
|
3601
|
+
this.syncInFlightQueueExpiryTimer != null
|
|
3602
|
+
) {
|
|
3603
|
+
clearTimeout(this.syncInFlightQueueExpiryTimer);
|
|
3604
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
3605
|
+
}
|
|
938
3606
|
}
|
|
939
3607
|
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
3608
|
+
private schedulePendingSyncKeyExpiry(): void {
|
|
3609
|
+
if (
|
|
3610
|
+
this.syncInFlightQueueExpiryTimer != null ||
|
|
3611
|
+
this.pendingSyncExpiryHeap.length === 0
|
|
3612
|
+
) {
|
|
3613
|
+
return;
|
|
3614
|
+
}
|
|
3615
|
+
const expiresAt = this.pendingSyncExpiryHeap[0]!.expiresAt;
|
|
3616
|
+
this.syncInFlightQueueExpiryTimer = setTimeout(
|
|
3617
|
+
() => {
|
|
3618
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
3619
|
+
this.expirePendingSyncKeys();
|
|
3620
|
+
this.schedulePendingSyncKeyExpiry();
|
|
3621
|
+
},
|
|
3622
|
+
Math.max(0, expiresAt - Date.now()),
|
|
947
3623
|
);
|
|
948
|
-
this.
|
|
3624
|
+
this.syncInFlightQueueExpiryTimer.unref?.();
|
|
949
3625
|
}
|
|
950
3626
|
|
|
951
3627
|
async queueSync(
|
|
@@ -953,111 +3629,273 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
953
3629
|
from: PublicSignKey,
|
|
954
3630
|
options?: { skipCheck?: boolean },
|
|
955
3631
|
) {
|
|
3632
|
+
if (this.closed === true || keys.length === 0) {
|
|
3633
|
+
return;
|
|
3634
|
+
}
|
|
3635
|
+
// A delayed timer must not let expired claims or admission reservations
|
|
3636
|
+
// keep the exact per-peer/global quota closed to fresh work.
|
|
3637
|
+
this.expirePendingSyncKeys();
|
|
3638
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
3639
|
+
const fromHash = from.hashcode();
|
|
3640
|
+
const canStartLookup =
|
|
3641
|
+
options?.skipCheck === true || this.canStartPendingSyncLookup(fromHash);
|
|
3642
|
+
const peerClaimCount =
|
|
3643
|
+
this.syncInFlightQueueInverted.get(fromHash)?.size ?? 0;
|
|
3644
|
+
let availableClaims = Math.max(
|
|
3645
|
+
0,
|
|
3646
|
+
Math.min(
|
|
3647
|
+
MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER -
|
|
3648
|
+
peerClaimCount -
|
|
3649
|
+
(this.pendingSyncAdmissionCountByPeer.get(fromHash) ?? 0),
|
|
3650
|
+
MAX_PENDING_SIMPLE_SYNC_KEYS_GLOBAL -
|
|
3651
|
+
this.pendingSyncClaimCount -
|
|
3652
|
+
this.pendingSyncAdmissionCount,
|
|
3653
|
+
),
|
|
3654
|
+
);
|
|
3655
|
+
const targetEpoch = this.getOrCreateSyncDispatchTargetEpoch(fromHash);
|
|
3656
|
+
const ownershipLifecycleController = this.syncDispatchLifecycleController;
|
|
3657
|
+
const isCapturedLifecycleActive = () =>
|
|
3658
|
+
this.closed !== true &&
|
|
3659
|
+
this.syncDispatchLifecycleController === ownershipLifecycleController &&
|
|
3660
|
+
!ownershipLifecycleController.signal.aborted &&
|
|
3661
|
+
this.syncDispatchTargetEpochs.get(fromHash) === targetEpoch;
|
|
956
3662
|
const requestHashes: SyncableKey[] = [];
|
|
3663
|
+
const existingRequestHashes: SyncableKey[] = [];
|
|
957
3664
|
const profile = this.syncOptions?.profile;
|
|
958
3665
|
const startedAt = syncProfileStart(profile);
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
options?.skipCheck === true
|
|
962
|
-
? undefined
|
|
963
|
-
: await this.resolveKnownSyncKeys(keys);
|
|
964
|
-
if (profile) {
|
|
965
|
-
emitSyncProfileDuration(profile, resolveKnownStartedAt, {
|
|
966
|
-
name: "simple.queueSync.resolveKnown",
|
|
967
|
-
entries: keys.length,
|
|
968
|
-
count: knownKeys?.keys.size ?? 0,
|
|
969
|
-
details: {
|
|
970
|
-
checkedCoordinates: knownKeys?.checkedCoordinates === true,
|
|
971
|
-
checkedHashes: knownKeys?.checkedHashes === true,
|
|
972
|
-
skipCheck: options?.skipCheck === true,
|
|
973
|
-
},
|
|
974
|
-
});
|
|
3666
|
+
if (!isCapturedLifecycleActive()) {
|
|
3667
|
+
return;
|
|
975
3668
|
}
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
3669
|
+
if (availableClaims === 0) {
|
|
3670
|
+
return;
|
|
3671
|
+
}
|
|
3672
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
3673
|
+
const keysToCheck: SyncableKey[] = [];
|
|
3674
|
+
const identitiesToCheck: SyncableKey[] = [];
|
|
3675
|
+
const seen = new Set<SyncableKey>();
|
|
3676
|
+
const pendingAdmissionIdentities =
|
|
3677
|
+
this.pendingSyncAdmissionIdentitiesByPeer.get(fromHash);
|
|
3678
|
+
// Default senders use 1,024-key messages. Capping examined input at the
|
|
3679
|
+
// entire per-peer allowance prevents a duplicate-filled oversized vector
|
|
3680
|
+
// from turning admission itself into unbounded work.
|
|
3681
|
+
const inspectionLimit = canStartLookup
|
|
3682
|
+
? MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER
|
|
3683
|
+
: Math.min(DEFAULT_MAX_HASHES_PER_MESSAGE, keys.length);
|
|
3684
|
+
for (
|
|
3685
|
+
let index = 0;
|
|
3686
|
+
index < keys.length && index < inspectionLimit;
|
|
3687
|
+
index += 1
|
|
3688
|
+
) {
|
|
3689
|
+
const key = keys[index]!;
|
|
3690
|
+
const queuedKeyResult = this.getQueuedSyncKeyForAdmission(key);
|
|
3691
|
+
if (queuedKeyResult === QUEUED_SYNC_ALIAS_REFRESH_PENDING) {
|
|
3692
|
+
continue;
|
|
3693
|
+
}
|
|
3694
|
+
const queuedKey = queuedKeyResult;
|
|
3695
|
+
const coordinateOrHash = queuedKey ?? key;
|
|
3696
|
+
const identity = this.getPendingSyncKeyIdentity(coordinateOrHash);
|
|
3697
|
+
if (seen.has(identity)) {
|
|
3698
|
+
continue;
|
|
3699
|
+
}
|
|
3700
|
+
seen.add(identity);
|
|
3701
|
+
|
|
3702
|
+
if (queuedKey != null) {
|
|
3703
|
+
let transferredDeadline: number | undefined;
|
|
3704
|
+
if (
|
|
3705
|
+
availableClaims > 0 &&
|
|
3706
|
+
pendingAdmissionIdentities?.has(identity) &&
|
|
3707
|
+
!this.hasPendingSyncClaim(queuedKey, fromHash)
|
|
3708
|
+
) {
|
|
3709
|
+
transferredDeadline = this.transferPendingSyncAdmissionIdentity(
|
|
3710
|
+
fromHash,
|
|
3711
|
+
identity,
|
|
3712
|
+
);
|
|
3713
|
+
if (transferredDeadline != null) {
|
|
3714
|
+
this.movePendingSyncKeyExpiryEarlier(
|
|
3715
|
+
queuedKey,
|
|
3716
|
+
transferredDeadline,
|
|
3717
|
+
);
|
|
993
3718
|
}
|
|
994
3719
|
}
|
|
995
|
-
|
|
3720
|
+
if (availableClaims === 0) {
|
|
3721
|
+
continue;
|
|
3722
|
+
}
|
|
3723
|
+
if (this.addPendingSyncClaim(queuedKey, from)) {
|
|
3724
|
+
availableClaims -= 1;
|
|
3725
|
+
existingRequestHashes.push(queuedKey);
|
|
3726
|
+
}
|
|
3727
|
+
continue;
|
|
996
3728
|
}
|
|
997
|
-
const hash = this.coordinateToHash.get(key);
|
|
998
|
-
return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
|
|
999
|
-
};
|
|
1000
3729
|
|
|
3730
|
+
if (
|
|
3731
|
+
pendingAdmissionIdentities?.has(identity) ||
|
|
3732
|
+
!canStartLookup ||
|
|
3733
|
+
availableClaims === 0
|
|
3734
|
+
) {
|
|
3735
|
+
continue;
|
|
3736
|
+
}
|
|
3737
|
+
keysToCheck.push(key);
|
|
3738
|
+
identitiesToCheck.push(identity);
|
|
3739
|
+
availableClaims -= 1;
|
|
3740
|
+
}
|
|
3741
|
+
const admission = this.reservePendingSyncAdmission(
|
|
3742
|
+
fromHash,
|
|
3743
|
+
identitiesToCheck,
|
|
3744
|
+
);
|
|
3745
|
+
const dispatchableExistingRequestHashes =
|
|
3746
|
+
this.filterDispatchablePendingSyncClaims(
|
|
3747
|
+
existingRequestHashes,
|
|
3748
|
+
fromHash,
|
|
3749
|
+
targetEpoch,
|
|
3750
|
+
);
|
|
3751
|
+
const existingRequest =
|
|
3752
|
+
dispatchableExistingRequestHashes.length > 0
|
|
3753
|
+
? this.requestSync(dispatchableExistingRequestHashes, [fromHash], {
|
|
3754
|
+
ownershipLifecycleController,
|
|
3755
|
+
targetEpochs: new Map([[fromHash, targetEpoch]]),
|
|
3756
|
+
createTargetEpochs: false,
|
|
3757
|
+
})
|
|
3758
|
+
: undefined;
|
|
3759
|
+
// Existing queued claims must not wait behind a potentially blocked storage
|
|
3760
|
+
// lookup for unrelated new keys. Observe this eagerly-started request even
|
|
3761
|
+
// when a later lifecycle check returns before joining it.
|
|
3762
|
+
void existingRequest?.catch(() => {});
|
|
3763
|
+
const resolveKnownStartedAt = syncProfileStart(profile);
|
|
1001
3764
|
try {
|
|
3765
|
+
const knownKeys =
|
|
3766
|
+
options?.skipCheck === true || keysToCheck.length === 0
|
|
3767
|
+
? undefined
|
|
3768
|
+
: await this.resolveKnownSyncKeys(keysToCheck);
|
|
3769
|
+
if (!isCapturedLifecycleActive()) {
|
|
3770
|
+
return;
|
|
3771
|
+
}
|
|
3772
|
+
if (profile) {
|
|
3773
|
+
emitSyncProfileDuration(profile, resolveKnownStartedAt, {
|
|
3774
|
+
name: "simple.queueSync.resolveKnown",
|
|
3775
|
+
entries: keysToCheck.length,
|
|
3776
|
+
count: knownKeys?.keys.size ?? 0,
|
|
3777
|
+
details: {
|
|
3778
|
+
checkedCoordinates: knownKeys?.checkedCoordinates === true,
|
|
3779
|
+
checkedHashes: knownKeys?.checkedHashes === true,
|
|
3780
|
+
skipCheck: options?.skipCheck === true,
|
|
3781
|
+
},
|
|
3782
|
+
});
|
|
3783
|
+
}
|
|
3784
|
+
|
|
3785
|
+
if (keysToCheck.length > 0) {
|
|
3786
|
+
// A resolver/index lookup may have populated coordinateToHash while
|
|
3787
|
+
// it yielded. Refresh another fixed-size slice, never the full queue.
|
|
3788
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
3789
|
+
}
|
|
1002
3790
|
const loopStartedAt = syncProfileStart(profile);
|
|
1003
|
-
for (
|
|
1004
|
-
const
|
|
3791
|
+
for (let index = 0; index < keysToCheck.length; index += 1) {
|
|
3792
|
+
const key = keysToCheck[index]!;
|
|
3793
|
+
const identity = identitiesToCheck[index]!;
|
|
3794
|
+
if (!isCapturedLifecycleActive()) {
|
|
3795
|
+
return;
|
|
3796
|
+
}
|
|
3797
|
+
const queuedKeyResult = this.getQueuedSyncKeyForAdmission(key);
|
|
3798
|
+
if (queuedKeyResult === QUEUED_SYNC_ALIAS_REFRESH_PENDING) {
|
|
3799
|
+
const consumption = this.consumePendingSyncAdmission(
|
|
3800
|
+
admission!,
|
|
3801
|
+
identity,
|
|
3802
|
+
);
|
|
3803
|
+
if (consumption === "invalid") {
|
|
3804
|
+
return;
|
|
3805
|
+
}
|
|
3806
|
+
continue;
|
|
3807
|
+
}
|
|
3808
|
+
const coordinateOrHash = queuedKeyResult ?? key;
|
|
1005
3809
|
const inFlight = this.syncInFlightQueue.get(coordinateOrHash);
|
|
1006
3810
|
if (inFlight) {
|
|
1007
|
-
if (!
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
3811
|
+
if (!this.hasPendingSyncClaim(coordinateOrHash, fromHash)) {
|
|
3812
|
+
const consumption = this.consumePendingSyncAdmission(
|
|
3813
|
+
admission!,
|
|
3814
|
+
identity,
|
|
3815
|
+
);
|
|
3816
|
+
if (consumption === "invalid") {
|
|
3817
|
+
return;
|
|
3818
|
+
}
|
|
3819
|
+
if (consumption === "settled") {
|
|
3820
|
+
continue;
|
|
3821
|
+
}
|
|
3822
|
+
this.movePendingSyncKeyExpiryEarlier(
|
|
3823
|
+
coordinateOrHash,
|
|
3824
|
+
admission!.expiresAt,
|
|
3825
|
+
);
|
|
3826
|
+
const added = this.addPendingSyncClaim(
|
|
3827
|
+
coordinateOrHash,
|
|
3828
|
+
from,
|
|
3829
|
+
admission!.expiresAt,
|
|
3830
|
+
);
|
|
3831
|
+
if (added) {
|
|
3832
|
+
requestHashes.push(coordinateOrHash);
|
|
1013
3833
|
}
|
|
1014
|
-
inverted.add(coordinateOrHash);
|
|
1015
3834
|
}
|
|
1016
|
-
} else
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
coordinateOrHash,
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
) {
|
|
1023
|
-
// Track the initial sender so we can retry if the first request is lost.
|
|
1024
|
-
this.syncInFlightQueue.set(coordinateOrHash, [from]);
|
|
1025
|
-
let inverted = this.syncInFlightQueueInverted.get(fromHash);
|
|
1026
|
-
if (!inverted) {
|
|
1027
|
-
inverted = new Set();
|
|
1028
|
-
this.syncInFlightQueueInverted.set(fromHash, inverted);
|
|
3835
|
+
} else {
|
|
3836
|
+
const has =
|
|
3837
|
+
options?.skipCheck !== true &&
|
|
3838
|
+
(await this.checkHasCoordinateOrHash(coordinateOrHash, knownKeys));
|
|
3839
|
+
if (!isCapturedLifecycleActive()) {
|
|
3840
|
+
return;
|
|
1029
3841
|
}
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
3842
|
+
if (has) {
|
|
3843
|
+
this.clearPendingSyncAdmissionIdentity(identity);
|
|
3844
|
+
continue;
|
|
3845
|
+
}
|
|
3846
|
+
const consumption = this.consumePendingSyncAdmission(
|
|
3847
|
+
admission!,
|
|
3848
|
+
identity,
|
|
3849
|
+
);
|
|
3850
|
+
if (consumption === "invalid") {
|
|
3851
|
+
return;
|
|
3852
|
+
}
|
|
3853
|
+
if (consumption === "settled") {
|
|
3854
|
+
continue;
|
|
1040
3855
|
}
|
|
3856
|
+
// Track the initial sender so we can retry if the first request is lost.
|
|
3857
|
+
this.addPendingSyncClaim(
|
|
3858
|
+
coordinateOrHash,
|
|
3859
|
+
from,
|
|
3860
|
+
admission!.expiresAt,
|
|
3861
|
+
);
|
|
3862
|
+
requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash)
|
|
1041
3863
|
}
|
|
1042
3864
|
}
|
|
1043
3865
|
if (profile) {
|
|
1044
3866
|
emitSyncProfileDuration(profile, loopStartedAt, {
|
|
1045
3867
|
name: "simple.queueSync.plan",
|
|
1046
|
-
entries:
|
|
3868
|
+
entries: keysToCheck.length,
|
|
1047
3869
|
count: requestHashes.length,
|
|
1048
3870
|
targets: 1,
|
|
1049
3871
|
});
|
|
1050
3872
|
}
|
|
1051
3873
|
|
|
1052
|
-
|
|
1053
|
-
|
|
3874
|
+
// Persistent admission work is complete. Do not let an unrelated
|
|
3875
|
+
// blocked transport send retain unused quota.
|
|
3876
|
+
this.releasePendingSyncAdmission(admission);
|
|
3877
|
+
const dispatchableRequestHashes =
|
|
3878
|
+
this.filterDispatchablePendingSyncClaims(
|
|
3879
|
+
requestHashes,
|
|
3880
|
+
fromHash,
|
|
3881
|
+
targetEpoch,
|
|
3882
|
+
);
|
|
3883
|
+
dispatchableRequestHashes.length > 0 &&
|
|
3884
|
+
(await this.requestSync(dispatchableRequestHashes, [fromHash], {
|
|
3885
|
+
ownershipLifecycleController,
|
|
3886
|
+
targetEpochs: new Map([[fromHash, targetEpoch]]),
|
|
3887
|
+
createTargetEpochs: false,
|
|
3888
|
+
}));
|
|
3889
|
+
await existingRequest;
|
|
1054
3890
|
} finally {
|
|
3891
|
+
this.releasePendingSyncAdmission(admission);
|
|
1055
3892
|
if (profile) {
|
|
1056
3893
|
emitSyncProfileDuration(profile, startedAt, {
|
|
1057
3894
|
name: "simple.queueSync",
|
|
1058
3895
|
entries: keys.length,
|
|
1059
3896
|
targets: 1,
|
|
1060
3897
|
details: {
|
|
3898
|
+
admitted: keysToCheck.length,
|
|
1061
3899
|
requested: requestHashes.length,
|
|
1062
3900
|
skipCheck: options?.skipCheck === true,
|
|
1063
3901
|
},
|
|
@@ -1066,10 +3904,24 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1066
3904
|
}
|
|
1067
3905
|
}
|
|
1068
3906
|
|
|
1069
|
-
private async requestSync(
|
|
1070
|
-
|
|
3907
|
+
private async requestSync(
|
|
3908
|
+
hashes: SyncableKey[],
|
|
3909
|
+
to: Set<string> | string[],
|
|
3910
|
+
options?: {
|
|
3911
|
+
ownershipLifecycleController?: AbortController;
|
|
3912
|
+
targetEpochs?: Map<string, SyncDispatchTargetEpoch>;
|
|
3913
|
+
createTargetEpochs?: boolean;
|
|
3914
|
+
},
|
|
3915
|
+
) {
|
|
3916
|
+
const targets = [...new Set(to)];
|
|
3917
|
+
if (hashes.length === 0 || targets.length === 0) {
|
|
1071
3918
|
return;
|
|
1072
3919
|
}
|
|
3920
|
+
const lifecycle = this.captureSyncDispatchLifecycle(targets, undefined, {
|
|
3921
|
+
ownershipLifecycleController: options?.ownershipLifecycleController,
|
|
3922
|
+
targetEpochs: options?.targetEpochs,
|
|
3923
|
+
createTargetEpochs: options?.createTargetEpochs,
|
|
3924
|
+
});
|
|
1073
3925
|
const profile = this.syncOptions?.profile;
|
|
1074
3926
|
const startedAt = syncProfileStart(profile);
|
|
1075
3927
|
let coordinateMessages = 0;
|
|
@@ -1079,14 +3931,12 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1079
3931
|
|
|
1080
3932
|
try {
|
|
1081
3933
|
const now = +new Date();
|
|
1082
|
-
for (const node of
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
map = new Map();
|
|
1086
|
-
this.syncInFlight.set(node, map);
|
|
3934
|
+
for (const node of targets) {
|
|
3935
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, node)) {
|
|
3936
|
+
continue;
|
|
1087
3937
|
}
|
|
1088
3938
|
for (const hash of hashes) {
|
|
1089
|
-
|
|
3939
|
+
this.setSyncInFlightTargetKey(node, hash, now);
|
|
1090
3940
|
}
|
|
1091
3941
|
}
|
|
1092
3942
|
|
|
@@ -1102,48 +3952,85 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1102
3952
|
coordinateHashCount = coordinateHashes.length;
|
|
1103
3953
|
stringHashCount = stringHashes.length;
|
|
1104
3954
|
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
3955
|
+
if (coordinateHashes.length > 0) {
|
|
3956
|
+
const chunks = this.chunk(
|
|
3957
|
+
coordinateHashes,
|
|
3958
|
+
this.maxCoordinatesPerMessage,
|
|
3959
|
+
);
|
|
3960
|
+
for (const target of targets) {
|
|
1111
3961
|
for (const chunk of chunks) {
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
3962
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
3963
|
+
break;
|
|
3964
|
+
}
|
|
3965
|
+
try {
|
|
3966
|
+
await this.rpc.send(
|
|
3967
|
+
this.syncOptions?.rawExchangeHeads
|
|
3968
|
+
? new RequestMaybeSyncCoordinateCapabilities({
|
|
3969
|
+
hashNumbers: chunk,
|
|
3970
|
+
})
|
|
3971
|
+
: new RequestMaybeSyncCoordinate({
|
|
3972
|
+
hashNumbers: chunk,
|
|
3973
|
+
}),
|
|
3974
|
+
{
|
|
3975
|
+
mode: new SilentDelivery({
|
|
3976
|
+
to: [target],
|
|
3977
|
+
redundancy: 1,
|
|
3978
|
+
}),
|
|
3979
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
3980
|
+
signal: this.getSyncDispatchSignal(lifecycle, target),
|
|
3981
|
+
},
|
|
3982
|
+
);
|
|
3983
|
+
coordinateMessages += 1;
|
|
3984
|
+
} catch (error) {
|
|
3985
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
3986
|
+
break;
|
|
3987
|
+
}
|
|
3988
|
+
throw error;
|
|
3989
|
+
}
|
|
1123
3990
|
}
|
|
1124
3991
|
}
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
3992
|
+
}
|
|
3993
|
+
if (stringHashes.length > 0) {
|
|
3994
|
+
const chunks = this.chunk(stringHashes, this.maxHashesPerMessage);
|
|
3995
|
+
for (const target of targets) {
|
|
1128
3996
|
for (const chunk of chunks) {
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
3997
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
3998
|
+
break;
|
|
3999
|
+
}
|
|
4000
|
+
try {
|
|
4001
|
+
await this.rpc.send(
|
|
4002
|
+
this.syncOptions?.rawExchangeHeads
|
|
4003
|
+
? new ResponseMaybeSyncCapabilities({
|
|
4004
|
+
hashes: chunk,
|
|
4005
|
+
})
|
|
4006
|
+
: new ResponseMaybeSync({ hashes: chunk }),
|
|
4007
|
+
{
|
|
4008
|
+
mode: new SilentDelivery({
|
|
4009
|
+
to: [target],
|
|
4010
|
+
redundancy: 1,
|
|
4011
|
+
}),
|
|
4012
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
4013
|
+
signal: this.getSyncDispatchSignal(lifecycle, target),
|
|
4014
|
+
},
|
|
4015
|
+
);
|
|
4016
|
+
stringMessages += 1;
|
|
4017
|
+
} catch (error) {
|
|
4018
|
+
if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
|
|
4019
|
+
break;
|
|
4020
|
+
}
|
|
4021
|
+
throw error;
|
|
4022
|
+
}
|
|
1138
4023
|
}
|
|
1139
4024
|
}
|
|
4025
|
+
}
|
|
1140
4026
|
} finally {
|
|
4027
|
+
this.finishSyncDispatchLifecycle(lifecycle);
|
|
1141
4028
|
if (profile) {
|
|
1142
4029
|
emitSyncProfileDuration(profile, startedAt, {
|
|
1143
4030
|
name: "simple.requestSync",
|
|
1144
4031
|
entries: hashes.length,
|
|
1145
4032
|
messages: coordinateMessages + stringMessages,
|
|
1146
|
-
targets:
|
|
4033
|
+
targets: targets.length,
|
|
1147
4034
|
details: {
|
|
1148
4035
|
coordinateHashes: coordinateHashCount,
|
|
1149
4036
|
stringHashes: stringHashCount,
|
|
@@ -1198,6 +4085,12 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1198
4085
|
key: string | bigint,
|
|
1199
4086
|
knownKeys?: KnownSyncKeys,
|
|
1200
4087
|
) {
|
|
4088
|
+
if (typeof key === "bigint") {
|
|
4089
|
+
const mappedHash = this.coordinateToHash.get(key);
|
|
4090
|
+
if (mappedHash != null && (await this.log.has(mappedHash))) {
|
|
4091
|
+
return true;
|
|
4092
|
+
}
|
|
4093
|
+
}
|
|
1201
4094
|
if (knownKeys) {
|
|
1202
4095
|
if (knownKeys.keys.has(key)) {
|
|
1203
4096
|
return true;
|
|
@@ -1214,8 +4107,31 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1214
4107
|
: this.log.has(key);
|
|
1215
4108
|
}
|
|
1216
4109
|
async open() {
|
|
4110
|
+
this.syncDispatchLifecycleController.abort();
|
|
4111
|
+
this.clearPendingMaybeSyncResponses();
|
|
4112
|
+
this.syncDispatchTargetEpochs.clear();
|
|
4113
|
+
this.syncDispatchLifecycleController = new AbortController();
|
|
4114
|
+
this.syncInFlightRetryIterator = undefined;
|
|
4115
|
+
this.syncInFlightRetryRemaining = 0;
|
|
4116
|
+
const openLifecycleController = this.syncDispatchLifecycleController;
|
|
1217
4117
|
this.closed = false;
|
|
1218
|
-
const
|
|
4118
|
+
const isOpenLifecycleActive = () =>
|
|
4119
|
+
this.closed !== true &&
|
|
4120
|
+
this.syncDispatchLifecycleController === openLifecycleController &&
|
|
4121
|
+
!openLifecycleController.signal.aborted;
|
|
4122
|
+
let requestSyncLoop!: () => Promise<void>;
|
|
4123
|
+
const scheduleRequestSyncLoop = () => {
|
|
4124
|
+
if (!isOpenLifecycleActive()) {
|
|
4125
|
+
return;
|
|
4126
|
+
}
|
|
4127
|
+
this.syncMoreInterval = setTimeout(() => {
|
|
4128
|
+
void requestSyncLoop().catch(() => {
|
|
4129
|
+
// The loop is best-effort. Observe all failures so a transport or
|
|
4130
|
+
// storage rejection cannot become an unhandled process rejection.
|
|
4131
|
+
});
|
|
4132
|
+
}, 3e3);
|
|
4133
|
+
};
|
|
4134
|
+
requestSyncLoop = async () => {
|
|
1219
4135
|
/**
|
|
1220
4136
|
* This method fetches entries that we potentially want.
|
|
1221
4137
|
* In a case in which we become replicator of a segment,
|
|
@@ -1224,75 +4140,185 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1224
4140
|
* so we don't get flooded with the same entry
|
|
1225
4141
|
*/
|
|
1226
4142
|
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
4143
|
+
if (!isOpenLifecycleActive()) {
|
|
4144
|
+
return;
|
|
4145
|
+
}
|
|
4146
|
+
try {
|
|
4147
|
+
const requestHashesByEpoch = new Map<
|
|
4148
|
+
SyncDispatchTargetEpoch,
|
|
4149
|
+
{ target: string; hashes: SyncableKey[] }
|
|
4150
|
+
>();
|
|
4151
|
+
const now = Date.now();
|
|
4152
|
+
if (!this.syncInFlightRetryIterator) {
|
|
4153
|
+
this.syncInFlightRetryIterator = this.syncInFlightQueue.entries();
|
|
4154
|
+
this.syncInFlightRetryRemaining = this.syncInFlightQueue.size;
|
|
1233
4155
|
}
|
|
4156
|
+
for (
|
|
4157
|
+
let inspected = 0;
|
|
4158
|
+
inspected < MAX_SIMPLE_SYNC_RETRY_KEYS_PER_TICK &&
|
|
4159
|
+
this.syncInFlightRetryRemaining > 0;
|
|
4160
|
+
inspected += 1
|
|
4161
|
+
) {
|
|
4162
|
+
const next = this.syncInFlightRetryIterator.next();
|
|
4163
|
+
if (next.done) {
|
|
4164
|
+
this.syncInFlightRetryIterator = undefined;
|
|
4165
|
+
this.syncInFlightRetryRemaining = 0;
|
|
4166
|
+
break;
|
|
4167
|
+
}
|
|
4168
|
+
this.syncInFlightRetryRemaining -= 1;
|
|
4169
|
+
const [key] = next.value;
|
|
4170
|
+
if (!isOpenLifecycleActive()) {
|
|
4171
|
+
return;
|
|
4172
|
+
}
|
|
4173
|
+
const expiresAt = this.syncInFlightQueueExpiresAt.get(key);
|
|
4174
|
+
if (expiresAt != null && expiresAt <= Date.now()) {
|
|
4175
|
+
this.clearSyncProcessKey(key);
|
|
4176
|
+
continue;
|
|
4177
|
+
}
|
|
1234
4178
|
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
4179
|
+
const has = await this.checkHasCoordinateOrHash(key);
|
|
4180
|
+
if (!isOpenLifecycleActive()) {
|
|
4181
|
+
return;
|
|
4182
|
+
}
|
|
4183
|
+
const value = this.syncInFlightQueue.get(key);
|
|
4184
|
+
if (!value) {
|
|
4185
|
+
continue;
|
|
4186
|
+
}
|
|
4187
|
+
const currentExpiresAt = this.syncInFlightQueueExpiresAt.get(key);
|
|
4188
|
+
if (currentExpiresAt != null && currentExpiresAt <= Date.now()) {
|
|
1240
4189
|
this.clearSyncProcessKey(key);
|
|
1241
4190
|
continue;
|
|
1242
4191
|
}
|
|
1243
4192
|
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
4193
|
+
if (!has) {
|
|
4194
|
+
if (value.length === 0) {
|
|
4195
|
+
this.clearSyncProcessKey(key);
|
|
4196
|
+
continue;
|
|
4197
|
+
}
|
|
4198
|
+
|
|
4199
|
+
const cursor =
|
|
4200
|
+
(this.syncInFlightQueueRoundRobinCursor.get(key) ?? 0) %
|
|
4201
|
+
value.length;
|
|
4202
|
+
const candidate = value[cursor]!;
|
|
4203
|
+
const publicKeyHash = candidate.hashcode();
|
|
4204
|
+
const inflightTimestamp = this.syncInFlight
|
|
4205
|
+
.get(publicKeyHash)
|
|
4206
|
+
?.get(key)?.timestamp;
|
|
4207
|
+
if (
|
|
4208
|
+
inflightTimestamp == null ||
|
|
4209
|
+
now - inflightTimestamp >= SIMPLE_SYNC_RETRY_AFTER_MS
|
|
4210
|
+
) {
|
|
4211
|
+
const epoch = this.syncDispatchTargetEpochs.get(publicKeyHash);
|
|
4212
|
+
if (!epoch) {
|
|
4213
|
+
continue;
|
|
4214
|
+
}
|
|
4215
|
+
let request = requestHashesByEpoch.get(epoch);
|
|
4216
|
+
if (!request) {
|
|
4217
|
+
request = { target: publicKeyHash, hashes: [] };
|
|
4218
|
+
requestHashesByEpoch.set(epoch, request);
|
|
4219
|
+
}
|
|
4220
|
+
request.hashes.push(key);
|
|
4221
|
+
if (value.length > 1) {
|
|
4222
|
+
this.syncInFlightQueueRoundRobinCursor.set(
|
|
4223
|
+
key,
|
|
4224
|
+
(cursor + 1) % value.length,
|
|
4225
|
+
);
|
|
4226
|
+
}
|
|
1260
4227
|
}
|
|
4228
|
+
} else {
|
|
4229
|
+
this.clearSyncProcessKey(key);
|
|
1261
4230
|
}
|
|
1262
|
-
} else {
|
|
1263
|
-
this.clearSyncProcessKey(key);
|
|
1264
4231
|
}
|
|
1265
|
-
|
|
4232
|
+
if (this.syncInFlightRetryRemaining === 0) {
|
|
4233
|
+
this.syncInFlightRetryIterator = undefined;
|
|
4234
|
+
}
|
|
1266
4235
|
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
4236
|
+
if (!isOpenLifecycleActive()) {
|
|
4237
|
+
return;
|
|
4238
|
+
}
|
|
4239
|
+
const nowMin10s = +new Date() - 2e4;
|
|
4240
|
+
for (const [target, map] of this.syncInFlight) {
|
|
4241
|
+
for (const [key, { timestamp }] of map) {
|
|
4242
|
+
if (timestamp < nowMin10s) {
|
|
4243
|
+
this.removeSyncInFlightTargetKey(target, key);
|
|
4244
|
+
}
|
|
1273
4245
|
}
|
|
1274
4246
|
}
|
|
1275
|
-
if (
|
|
1276
|
-
|
|
4247
|
+
if (!isOpenLifecycleActive()) {
|
|
4248
|
+
return;
|
|
1277
4249
|
}
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
4250
|
+
for (const [epoch, request] of requestHashesByEpoch) {
|
|
4251
|
+
if (!isOpenLifecycleActive()) {
|
|
4252
|
+
return;
|
|
4253
|
+
}
|
|
4254
|
+
const requestHashes = this.filterDispatchablePendingSyncClaims(
|
|
4255
|
+
request.hashes,
|
|
4256
|
+
request.target,
|
|
4257
|
+
epoch,
|
|
4258
|
+
);
|
|
4259
|
+
if (requestHashes.length === 0) {
|
|
4260
|
+
continue;
|
|
4261
|
+
}
|
|
4262
|
+
try {
|
|
4263
|
+
await this.requestSync(requestHashes, [request.target], {
|
|
4264
|
+
ownershipLifecycleController: openLifecycleController,
|
|
4265
|
+
targetEpochs: new Map([[request.target, epoch]]),
|
|
4266
|
+
createTargetEpochs: false,
|
|
4267
|
+
});
|
|
4268
|
+
} catch {
|
|
4269
|
+
if (!isOpenLifecycleActive()) {
|
|
4270
|
+
return;
|
|
4271
|
+
}
|
|
4272
|
+
// A failed target must not prevent bounded retries for unrelated
|
|
4273
|
+
// peers in this pass.
|
|
4274
|
+
}
|
|
4275
|
+
}
|
|
4276
|
+
if (!isOpenLifecycleActive()) {
|
|
1281
4277
|
return;
|
|
1282
4278
|
}
|
|
1283
|
-
|
|
1284
|
-
|
|
4279
|
+
} catch {
|
|
4280
|
+
if (!isOpenLifecycleActive()) {
|
|
4281
|
+
return;
|
|
4282
|
+
}
|
|
4283
|
+
// Retry on the next interval; the rejection is deliberately observed.
|
|
4284
|
+
}
|
|
4285
|
+
scheduleRequestSyncLoop();
|
|
1285
4286
|
};
|
|
1286
4287
|
|
|
1287
|
-
requestSyncLoop()
|
|
4288
|
+
void requestSyncLoop().catch(() => {
|
|
4289
|
+
// Defensive observation for failures outside the loop's best-effort body.
|
|
4290
|
+
});
|
|
1288
4291
|
}
|
|
1289
4292
|
|
|
1290
4293
|
async close() {
|
|
1291
4294
|
this.closed = true;
|
|
4295
|
+
this.syncDispatchLifecycleController.abort();
|
|
4296
|
+
this.syncDispatchTargetEpochs.clear();
|
|
4297
|
+
this.clearPendingSyncAdmissions();
|
|
1292
4298
|
this.syncInFlightQueue.clear();
|
|
1293
4299
|
this.syncInFlightQueueInverted.clear();
|
|
4300
|
+
this.syncInFlightRetryIterator = undefined;
|
|
4301
|
+
this.syncInFlightRetryRemaining = 0;
|
|
4302
|
+
this.syncInFlightQueueExpiresAt.clear();
|
|
4303
|
+
this.pendingSyncExpiryHeap.length = 0;
|
|
4304
|
+
this.pendingSyncKeyExpiryNodes.clear();
|
|
4305
|
+
this.pendingSyncAdmissionExpiryNodes.clear();
|
|
4306
|
+
this.syncInFlightQueueClaimants.clear();
|
|
4307
|
+
this.syncInFlightQueueClaimantIndexes.clear();
|
|
4308
|
+
this.syncInFlightQueueRoundRobinCursor.clear();
|
|
4309
|
+
this.syncInFlightQueuedCoordinates.clear();
|
|
4310
|
+
this.syncInFlightQueuedHashByCoordinate.clear();
|
|
4311
|
+
this.syncInFlightQueuedCoordinatesByHash.clear();
|
|
4312
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
4313
|
+
this.pendingSyncClaimCount = 0;
|
|
4314
|
+
if (this.syncInFlightQueueExpiryTimer != null) {
|
|
4315
|
+
clearTimeout(this.syncInFlightQueueExpiryTimer);
|
|
4316
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
4317
|
+
}
|
|
1294
4318
|
this.syncInFlight.clear();
|
|
4319
|
+
this.syncInFlightTargetsByKey.clear();
|
|
1295
4320
|
this.recentlySentExchangeHeads.clear();
|
|
4321
|
+
this.clearPendingMaybeSyncResponses();
|
|
1296
4322
|
for (const sessionId of [...this.repairSessions.keys()]) {
|
|
1297
4323
|
this.finalizeRepairSession(sessionId, false);
|
|
1298
4324
|
}
|
|
@@ -1306,6 +4332,9 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1306
4332
|
if (hashes.length === 0 || !this.hasEntryAddedState()) {
|
|
1307
4333
|
return;
|
|
1308
4334
|
}
|
|
4335
|
+
for (const hash of hashes) {
|
|
4336
|
+
this.clearPendingSyncAdmissionIdentity(hash);
|
|
4337
|
+
}
|
|
1309
4338
|
this.clearSyncProcesses(hashes);
|
|
1310
4339
|
this.markRepairSessionResolvedHashes(hashes);
|
|
1311
4340
|
}
|
|
@@ -1314,6 +4343,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1314
4343
|
if (!this.hasEntryAddedState()) {
|
|
1315
4344
|
return;
|
|
1316
4345
|
}
|
|
4346
|
+
this.clearPendingSyncAdmissionIdentity(hash);
|
|
1317
4347
|
this.clearSyncProcess(hash);
|
|
1318
4348
|
this.markRepairSessionResolvedHash(hash);
|
|
1319
4349
|
}
|
|
@@ -1340,6 +4370,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1340
4370
|
return (
|
|
1341
4371
|
this.syncInFlightQueue.size > 0 ||
|
|
1342
4372
|
this.syncInFlightQueueInverted.size > 0 ||
|
|
4373
|
+
this.pendingSyncAdmissionCount > 0 ||
|
|
1343
4374
|
this.syncInFlight.size > 0
|
|
1344
4375
|
);
|
|
1345
4376
|
}
|
|
@@ -1357,43 +4388,171 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1357
4388
|
}
|
|
1358
4389
|
}
|
|
1359
4390
|
|
|
4391
|
+
const trackedClaimants = this.syncInFlightQueueClaimants.get(key);
|
|
4392
|
+
this.pendingSyncClaimCount = Math.max(
|
|
4393
|
+
0,
|
|
4394
|
+
this.pendingSyncClaimCount -
|
|
4395
|
+
(trackedClaimants?.size ?? inflight.length),
|
|
4396
|
+
);
|
|
1360
4397
|
this.syncInFlightQueue.delete(key);
|
|
1361
4398
|
}
|
|
4399
|
+
this.syncInFlightQueueClaimants.delete(key);
|
|
4400
|
+
this.syncInFlightQueueClaimantIndexes.delete(key);
|
|
4401
|
+
this.syncInFlightQueueRoundRobinCursor.delete(key);
|
|
4402
|
+
if (typeof key === "bigint") {
|
|
4403
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
4404
|
+
}
|
|
4405
|
+
this.removePendingSyncKeyExpiry(key);
|
|
4406
|
+
this.syncInFlightQueueExpiresAt.delete(key);
|
|
4407
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
1362
4408
|
|
|
1363
4409
|
this.clearSyncInFlightKey(key);
|
|
1364
4410
|
}
|
|
1365
4411
|
|
|
1366
|
-
private
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
4412
|
+
private removePendingSyncClaim(key: SyncableKey, peer: string): void {
|
|
4413
|
+
const inflight = this.syncInFlightQueue.get(key);
|
|
4414
|
+
if (!inflight) {
|
|
4415
|
+
return;
|
|
4416
|
+
}
|
|
4417
|
+
let claimants = this.syncInFlightQueueClaimants.get(key);
|
|
4418
|
+
let claimantIndexes = this.syncInFlightQueueClaimantIndexes.get(key);
|
|
4419
|
+
if (!claimants || !claimantIndexes) {
|
|
4420
|
+
claimants = new Set();
|
|
4421
|
+
claimantIndexes = new Map();
|
|
4422
|
+
for (let index = 0; index < inflight.length; index += 1) {
|
|
4423
|
+
const claimant = inflight[index]!.hashcode();
|
|
4424
|
+
claimants.add(claimant);
|
|
4425
|
+
claimantIndexes.set(claimant, index);
|
|
4426
|
+
}
|
|
4427
|
+
if (!this.syncInFlightQueueClaimants.has(key)) {
|
|
4428
|
+
this.pendingSyncClaimCount += claimants.size;
|
|
1371
4429
|
}
|
|
4430
|
+
this.syncInFlightQueueClaimants.set(key, claimants);
|
|
4431
|
+
this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
|
|
4432
|
+
}
|
|
4433
|
+
const index = claimantIndexes.get(peer);
|
|
4434
|
+
if (index == null) {
|
|
4435
|
+
return;
|
|
4436
|
+
}
|
|
4437
|
+
|
|
4438
|
+
const lastIndex = inflight.length - 1;
|
|
4439
|
+
if (index !== lastIndex) {
|
|
4440
|
+
const lastClaimant = inflight[lastIndex]!;
|
|
4441
|
+
const lastClaimantHash = lastClaimant.hashcode();
|
|
4442
|
+
inflight[index] = lastClaimant;
|
|
4443
|
+
claimantIndexes.set(lastClaimantHash, index);
|
|
4444
|
+
}
|
|
4445
|
+
inflight.pop();
|
|
4446
|
+
claimantIndexes.delete(peer);
|
|
4447
|
+
claimants.delete(peer);
|
|
4448
|
+
this.pendingSyncClaimCount = Math.max(0, this.pendingSyncClaimCount - 1);
|
|
4449
|
+
const inverted = this.syncInFlightQueueInverted.get(peer);
|
|
4450
|
+
inverted?.delete(key);
|
|
4451
|
+
if (inverted?.size === 0) {
|
|
4452
|
+
this.syncInFlightQueueInverted.delete(peer);
|
|
4453
|
+
}
|
|
4454
|
+
if (inflight.length > 0) {
|
|
4455
|
+
const cursor = this.syncInFlightQueueRoundRobinCursor.get(key) ?? 0;
|
|
4456
|
+
this.syncInFlightQueueRoundRobinCursor.set(
|
|
4457
|
+
key,
|
|
4458
|
+
cursor === lastIndex
|
|
4459
|
+
? index % inflight.length
|
|
4460
|
+
: cursor % inflight.length,
|
|
4461
|
+
);
|
|
4462
|
+
return;
|
|
1372
4463
|
}
|
|
4464
|
+
|
|
4465
|
+
this.syncInFlightQueue.delete(key);
|
|
4466
|
+
this.syncInFlightQueueClaimants.delete(key);
|
|
4467
|
+
this.syncInFlightQueueClaimantIndexes.delete(key);
|
|
4468
|
+
this.syncInFlightQueueRoundRobinCursor.delete(key);
|
|
4469
|
+
if (typeof key === "bigint") {
|
|
4470
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
4471
|
+
}
|
|
4472
|
+
this.removePendingSyncKeyExpiry(key);
|
|
4473
|
+
this.syncInFlightQueueExpiresAt.delete(key);
|
|
4474
|
+
this.clearSyncInFlightKey(key);
|
|
4475
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
1373
4476
|
}
|
|
1374
4477
|
|
|
1375
|
-
private
|
|
1376
|
-
|
|
1377
|
-
|
|
4478
|
+
private removeSyncInFlightTargetKey(peer: string, key: SyncableKey): void {
|
|
4479
|
+
const map = this.syncInFlight.get(peer);
|
|
4480
|
+
if (!map?.delete(key)) {
|
|
4481
|
+
return;
|
|
4482
|
+
}
|
|
4483
|
+
if (map.size === 0) {
|
|
4484
|
+
this.syncInFlight.delete(peer);
|
|
4485
|
+
}
|
|
4486
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
4487
|
+
targets?.delete(peer);
|
|
4488
|
+
if (targets?.size === 0) {
|
|
4489
|
+
this.syncInFlightTargetsByKey.delete(key);
|
|
4490
|
+
}
|
|
4491
|
+
}
|
|
4492
|
+
|
|
4493
|
+
private setSyncInFlightTargetKey(
|
|
4494
|
+
peer: string,
|
|
4495
|
+
key: SyncableKey,
|
|
4496
|
+
timestamp: number,
|
|
1378
4497
|
): void {
|
|
1379
|
-
|
|
1380
|
-
if (
|
|
4498
|
+
let map = this.syncInFlight.get(peer);
|
|
4499
|
+
if (!map) {
|
|
4500
|
+
map = new Map();
|
|
4501
|
+
this.syncInFlight.set(peer, map);
|
|
4502
|
+
}
|
|
4503
|
+
const existing = map.get(key);
|
|
4504
|
+
if (!existing || existing.timestamp < timestamp) {
|
|
4505
|
+
map.set(key, { timestamp });
|
|
4506
|
+
}
|
|
4507
|
+
let targets = this.syncInFlightTargetsByKey.get(key);
|
|
4508
|
+
if (!targets) {
|
|
4509
|
+
targets = new Set();
|
|
4510
|
+
this.syncInFlightTargetsByKey.set(key, targets);
|
|
4511
|
+
}
|
|
4512
|
+
targets.add(peer);
|
|
4513
|
+
}
|
|
4514
|
+
|
|
4515
|
+
private clearSyncInFlightTarget(peer: string): void {
|
|
4516
|
+
const map = this.syncInFlight.get(peer);
|
|
4517
|
+
if (!map) {
|
|
1381
4518
|
return;
|
|
1382
4519
|
}
|
|
1383
|
-
for (const key of
|
|
1384
|
-
|
|
1385
|
-
|
|
4520
|
+
for (const key of map.keys()) {
|
|
4521
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
4522
|
+
targets?.delete(peer);
|
|
4523
|
+
if (targets?.size === 0) {
|
|
4524
|
+
this.syncInFlightTargetsByKey.delete(key);
|
|
1386
4525
|
}
|
|
1387
4526
|
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
4527
|
+
this.syncInFlight.delete(peer);
|
|
4528
|
+
}
|
|
4529
|
+
|
|
4530
|
+
private clearSyncInFlightKey(key: SyncableKey) {
|
|
4531
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
4532
|
+
if (!targets) {
|
|
4533
|
+
// Defensive compatibility for tests or integrations that seed the public
|
|
4534
|
+
// syncInFlight map directly. Internal writes always populate the index.
|
|
4535
|
+
for (const [peer, map] of this.syncInFlight) {
|
|
4536
|
+
if (map.has(key)) {
|
|
4537
|
+
this.removeSyncInFlightTargetKey(peer, key);
|
|
1395
4538
|
}
|
|
1396
4539
|
}
|
|
4540
|
+
return;
|
|
4541
|
+
}
|
|
4542
|
+
for (const peer of [...targets]) {
|
|
4543
|
+
this.removeSyncInFlightTargetKey(peer, key);
|
|
4544
|
+
}
|
|
4545
|
+
}
|
|
4546
|
+
|
|
4547
|
+
private forEachKnownAlias(
|
|
4548
|
+
hash: string,
|
|
4549
|
+
callback: (key: SyncableKey) => void,
|
|
4550
|
+
): void {
|
|
4551
|
+
callback(hash);
|
|
4552
|
+
for (const coordinate of [
|
|
4553
|
+
...(this.syncInFlightQueuedCoordinatesByHash.get(hash) ?? []),
|
|
4554
|
+
]) {
|
|
4555
|
+
callback(coordinate);
|
|
1397
4556
|
}
|
|
1398
4557
|
}
|
|
1399
4558
|
|
|
@@ -1402,10 +4561,10 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1402
4561
|
if (!map) {
|
|
1403
4562
|
return;
|
|
1404
4563
|
}
|
|
1405
|
-
this.
|
|
1406
|
-
|
|
1407
|
-
this.
|
|
1408
|
-
|
|
4564
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
4565
|
+
this.forEachKnownAlias(hash, (key) =>
|
|
4566
|
+
this.removeSyncInFlightTargetKey(publicKeyHash, key),
|
|
4567
|
+
);
|
|
1409
4568
|
}
|
|
1410
4569
|
|
|
1411
4570
|
private clearSyncInFlightForPeerHashes(
|
|
@@ -1416,26 +4575,16 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1416
4575
|
if (!map || hashes.length === 0) {
|
|
1417
4576
|
return;
|
|
1418
4577
|
}
|
|
1419
|
-
|
|
1420
|
-
const
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
}
|
|
1425
|
-
const hash = this.coordinateToHash.get(key);
|
|
1426
|
-
if (hash != null && hashSet.has(hash)) {
|
|
1427
|
-
keys.add(key);
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
for (const key of keys) {
|
|
1431
|
-
map.delete(key);
|
|
1432
|
-
}
|
|
1433
|
-
if (map.size === 0) {
|
|
1434
|
-
this.syncInFlight.delete(publicKeyHash);
|
|
4578
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
4579
|
+
for (const hash of hashes) {
|
|
4580
|
+
this.forEachKnownAlias(hash, (key) =>
|
|
4581
|
+
this.removeSyncInFlightTargetKey(publicKeyHash, key),
|
|
4582
|
+
);
|
|
1435
4583
|
}
|
|
1436
4584
|
}
|
|
1437
4585
|
|
|
1438
4586
|
private clearSyncProcess(hash: string) {
|
|
4587
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
1439
4588
|
this.forEachKnownAlias(hash, (key) => this.clearSyncProcessKey(key));
|
|
1440
4589
|
}
|
|
1441
4590
|
|
|
@@ -1443,24 +4592,10 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1443
4592
|
if (hashes.length === 0) {
|
|
1444
4593
|
return;
|
|
1445
4594
|
}
|
|
1446
|
-
|
|
1447
|
-
const
|
|
1448
|
-
const
|
|
1449
|
-
|
|
1450
|
-
return;
|
|
1451
|
-
}
|
|
1452
|
-
const hash = this.coordinateToHash.get(key);
|
|
1453
|
-
if (hash != null && hashSet.has(hash)) {
|
|
1454
|
-
keys.add(key);
|
|
1455
|
-
}
|
|
1456
|
-
};
|
|
1457
|
-
for (const key of this.syncInFlightQueue.keys()) {
|
|
1458
|
-
maybeAddAlias(key);
|
|
1459
|
-
}
|
|
1460
|
-
for (const map of this.syncInFlight.values()) {
|
|
1461
|
-
for (const key of map.keys()) {
|
|
1462
|
-
maybeAddAlias(key);
|
|
1463
|
-
}
|
|
4595
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
4596
|
+
const keys = new Set<SyncableKey>();
|
|
4597
|
+
for (const hash of hashes) {
|
|
4598
|
+
this.forEachKnownAlias(hash, (key) => keys.add(key));
|
|
1464
4599
|
}
|
|
1465
4600
|
for (const key of keys) {
|
|
1466
4601
|
this.clearSyncProcessKey(key);
|
|
@@ -1472,20 +4607,30 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
|
|
|
1472
4607
|
return this.clearSyncProcessPublicKeyHash(publicKeyHash);
|
|
1473
4608
|
}
|
|
1474
4609
|
private clearSyncProcessPublicKeyHash(publicKeyHash: string) {
|
|
1475
|
-
this.
|
|
4610
|
+
this.syncDispatchTargetEpochs.delete(publicKeyHash);
|
|
4611
|
+
this.clearPendingSyncAdmissions(publicKeyHash);
|
|
4612
|
+
for (const targetLifecycle of [
|
|
4613
|
+
...(this.syncDispatchTargets.get(publicKeyHash) ?? []),
|
|
4614
|
+
]) {
|
|
4615
|
+
if (targetLifecycle.lifecycle.abortAllOnTargetDisconnect) {
|
|
4616
|
+
this.abortSyncDispatchLifecycle(
|
|
4617
|
+
targetLifecycle.lifecycle,
|
|
4618
|
+
new Error("sync target disconnected"),
|
|
4619
|
+
);
|
|
4620
|
+
} else {
|
|
4621
|
+
this.abortSyncDispatchTarget(
|
|
4622
|
+
targetLifecycle,
|
|
4623
|
+
new Error("sync target disconnected"),
|
|
4624
|
+
);
|
|
4625
|
+
}
|
|
4626
|
+
}
|
|
4627
|
+
this.clearSyncInFlightTarget(publicKeyHash);
|
|
1476
4628
|
this.recentlySentExchangeHeads.delete(publicKeyHash);
|
|
4629
|
+
this.clearPendingMaybeSyncResponses(publicKeyHash);
|
|
1477
4630
|
const map = this.syncInFlightQueueInverted.get(publicKeyHash);
|
|
1478
4631
|
if (map) {
|
|
1479
|
-
for (const hash of map) {
|
|
1480
|
-
|
|
1481
|
-
if (arr) {
|
|
1482
|
-
const filtered = arr.filter((x) => x.hashcode() !== publicKeyHash);
|
|
1483
|
-
if (filtered.length > 0) {
|
|
1484
|
-
this.syncInFlightQueue.set(hash, filtered);
|
|
1485
|
-
} else {
|
|
1486
|
-
this.syncInFlightQueue.delete(hash);
|
|
1487
|
-
}
|
|
1488
|
-
}
|
|
4632
|
+
for (const hash of [...map]) {
|
|
4633
|
+
this.removePendingSyncClaim(hash, publicKeyHash);
|
|
1489
4634
|
}
|
|
1490
4635
|
this.syncInFlightQueueInverted.delete(publicKeyHash);
|
|
1491
4636
|
}
|