@peerbit/shared-log 13.2.12 → 13.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,8 +21,8 @@ import {
21
21
  import { TransportMessage } from "../message.js";
22
22
  import type { EntryReplicated } from "../ranges.js";
23
23
  import type {
24
- HashSymbolResolver,
25
24
  HashSymbolHashListResolver,
25
+ HashSymbolResolver,
26
26
  RawExchangeHeadsSender,
27
27
  RepairSession,
28
28
  RepairSessionMode,
@@ -135,15 +135,26 @@ const getHashesFromSymbols = async (
135
135
  coordinateToHash: Cache<string>,
136
136
  resolveHashesForSymbols?: HashSymbolResolver,
137
137
  resolveHashListForSymbols?: HashSymbolHashListResolver,
138
+ maxHashes = 10_000,
138
139
  ): Promise<Set<string> | string[]> => {
139
140
  let queries: IntegerCompare[] = [];
140
141
  let batchSize = 128; // TODO arg
141
142
  let results = new Set<string>();
142
143
  let missingSymbols: bigint[] = [];
144
+ const addHash = (hash: string) => {
145
+ if (results.has(hash)) {
146
+ return true;
147
+ }
148
+ if (results.size >= maxHashes) {
149
+ return false;
150
+ }
151
+ results.add(hash);
152
+ return true;
153
+ };
143
154
  const addMissingUnlessCached = (symbol: bigint) => {
144
155
  const fromCache = coordinateToHash.get(symbol);
145
156
  if (fromCache) {
146
- results.add(fromCache);
157
+ addHash(fromCache);
147
158
  return;
148
159
  }
149
160
  missingSymbols.push(symbol);
@@ -159,8 +170,13 @@ const getHashesFromSymbols = async (
159
170
  queries = [];
160
171
 
161
172
  for (const entry of entries) {
162
- results.add(entry.value.hash);
173
+ if (!addHash(entry.value.hash)) {
174
+ break;
175
+ }
163
176
  coordinateToHash.add(entry.value.hashNumber, entry.value.hash);
177
+ if (results.size >= maxHashes) {
178
+ break;
179
+ }
164
180
  }
165
181
  }
166
182
  };
@@ -168,15 +184,31 @@ const getHashesFromSymbols = async (
168
184
  if (resolveHashListForSymbols) {
169
185
  const resolvedHashes = await resolveHashListForSymbols(symbols);
170
186
  if (resolvedHashes) {
171
- const resolvedHashList = Array.isArray(resolvedHashes)
172
- ? resolvedHashes
173
- : [...resolvedHashes];
187
+ const resolvedHashList: string[] = [];
188
+ const iterator = resolvedHashes[Symbol.iterator]();
189
+ let exhausted = false;
190
+ try {
191
+ while (resolvedHashList.length < maxHashes) {
192
+ const next = iterator.next();
193
+ if (next.done) {
194
+ exhausted = true;
195
+ break;
196
+ }
197
+ resolvedHashList.push(next.value);
198
+ }
199
+ } finally {
200
+ if (!exhausted) {
201
+ iterator.return?.();
202
+ }
203
+ }
174
204
  let mergedHashes: Set<string> | undefined;
175
205
  for (const symbol of symbols) {
176
206
  const fromCache = coordinateToHash.get(symbol);
177
207
  if (fromCache) {
178
208
  mergedHashes ??= new Set(resolvedHashList);
179
- mergedHashes.add(fromCache);
209
+ if (mergedHashes.size < maxHashes) {
210
+ mergedHashes.add(fromCache);
211
+ }
180
212
  }
181
213
  }
182
214
  return mergedHashes ?? resolvedHashList;
@@ -186,7 +218,11 @@ const getHashesFromSymbols = async (
186
218
  if (resolveHashesForSymbols) {
187
219
  const resolved = await resolveHashesForSymbols(symbols);
188
220
  if (resolved) {
221
+ let resolvedItemCount = 0;
189
222
  for (const symbol of symbols) {
223
+ if (resolvedItemCount >= maxHashes) {
224
+ break;
225
+ }
190
226
  const hashes = resolved.get(symbol);
191
227
  if (!hashes) {
192
228
  addMissingUnlessCached(symbol);
@@ -194,14 +230,37 @@ const getHashesFromSymbols = async (
194
230
  }
195
231
  let singleHash: string | undefined;
196
232
  let count = 0;
197
- for (const hash of hashes) {
198
- results.add(hash);
199
- singleHash = hash;
200
- count += 1;
233
+ let truncated = false;
234
+ const iterator = hashes[Symbol.iterator]();
235
+ let exhausted = false;
236
+ try {
237
+ while (resolvedItemCount < maxHashes) {
238
+ const next = iterator.next();
239
+ if (next.done) {
240
+ exhausted = true;
241
+ break;
242
+ }
243
+ resolvedItemCount += 1;
244
+ if (!addHash(next.value)) {
245
+ truncated = true;
246
+ break;
247
+ }
248
+ singleHash = next.value;
249
+ count += 1;
250
+ if (results.size >= maxHashes) {
251
+ truncated = true;
252
+ break;
253
+ }
254
+ }
255
+ } finally {
256
+ if (!exhausted) {
257
+ truncated = true;
258
+ iterator.return?.();
259
+ }
201
260
  }
202
261
  if (count === 0) {
203
262
  addMissingUnlessCached(symbol);
204
- } else if (count === 1) {
263
+ } else if (count === 1 && !truncated) {
205
264
  coordinateToHash.add(symbol, singleHash!);
206
265
  }
207
266
  }
@@ -217,6 +276,9 @@ const getHashesFromSymbols = async (
217
276
  }
218
277
 
219
278
  for (const symbol of missingSymbols) {
279
+ if (results.size >= maxHashes) {
280
+ break;
281
+ }
220
282
  const matchQuery = new IntegerCompare({
221
283
  key: "hashNumber",
222
284
  compare: Compare.Equal,
@@ -241,6 +303,10 @@ const SESSION_POLL_INTERVAL_MS = 100;
241
303
  const DEFAULT_MAX_HASHES_PER_MESSAGE = 1_024;
242
304
  const DEFAULT_MAX_COORDINATES_PER_MESSAGE = 1_024;
243
305
  const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
306
+ export const MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS = 1_024;
307
+ export const MAX_SIMPLE_COORDINATE_RESPONSE_HASHES = 10_000;
308
+ export const MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_PER_PEER = 4;
309
+ export const MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_GLOBAL = 32;
244
310
  // Keep convergence sync above the default/background lane. Dropping it to the
245
311
  // background priority lets repair traffic starve behind foreground work.
246
312
  export const SYNC_MESSAGE_PRIORITY = CONVERGENCE_MESSAGE_PRIORITY;
@@ -250,6 +316,153 @@ export const SYNC_MESSAGE_PRIORITY = CONVERGENCE_MESSAGE_PRIORITY;
250
316
  const SIMPLE_SYNC_RETRY_AFTER_MS = 10_000;
251
317
  const EXCHANGE_HEAD_RESPONSE_DEDUPE_TTL_MS = SIMPLE_SYNC_RETRY_AFTER_MS - 1_000;
252
318
  const RECENT_KNOWN_EXCHANGE_HEAD_SUPPRESSION_MS = 30_000;
319
+ const PENDING_MAYBE_SYNC_RESPONSE_TTL_MS = 30_000;
320
+ // An incoming maybe-sync claim keeps one retry candidate in both
321
+ // syncInFlightQueue and syncInFlightQueueInverted. Bound associations rather
322
+ // than only unique keys: otherwise many peers can grow the claimant array for
323
+ // the same keys without changing syncInFlightQueue.size.
324
+ // The per-peer allowance matches one full 10,000-hash response-authorization
325
+ // window; the global allowance lets four peers make full bounded progress.
326
+ export const MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER = 10_000;
327
+ export const MAX_PENDING_SIMPLE_SYNC_KEYS_GLOBAL = 40_000;
328
+ // Storage presence resolution is not universally abortable. Keep the number of
329
+ // live resolver calls small even when requests use only one key each.
330
+ export const MAX_PENDING_SIMPLE_SYNC_LOOKUPS_PER_PEER = 4;
331
+ export const MAX_PENDING_SIMPLE_SYNC_LOOKUPS_GLOBAL = 32;
332
+ // Retry scanning can touch persistent indexes. Bound each pass so a full
333
+ // adversarial queue cannot force 40,000 lookups in one event-loop turn.
334
+ export const MAX_SIMPLE_SYNC_RETRY_KEYS_PER_TICK = 4_096;
335
+ // Late coordinate-to-hash cache fills are discovered incrementally. Keep this
336
+ // independent of retained queue size so an empty or repeated request cannot
337
+ // force an O(global pending keys) reverse-alias rebuild.
338
+ const MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE = 128;
339
+ const QUEUED_SYNC_ALIAS_REFRESH_PENDING = Symbol(
340
+ "queued-sync-alias-refresh-pending",
341
+ );
342
+ // This is an absolute first-seen lifetime. Repeated claims and additional peers
343
+ // deliberately do not slide the deadline.
344
+ export const PENDING_SIMPLE_SYNC_KEY_TTL_MS = 60_000;
345
+ // Bound retained request/response associations globally. Ten thousand hashes
346
+ // covers several full default-size request batches while keeping adversarial or
347
+ // abandoned requests to a small, predictable amount of heap.
348
+ const MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES = 10_000;
349
+ export const MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_PER_PEER = 4;
350
+ export const MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_GLOBAL = 32;
351
+ const MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITER_BYPASSES = 32;
352
+ const MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITERS = 10_000;
353
+
354
+ type PendingSyncAdmissionReservation = {
355
+ peer: string;
356
+ remaining: number;
357
+ active: boolean;
358
+ released: boolean;
359
+ expiresAt: number;
360
+ identities: Set<SyncableKey>;
361
+ retainedSettled: number;
362
+ };
363
+
364
+ type PendingSyncExpiryNode =
365
+ | {
366
+ kind: "key";
367
+ key: SyncableKey;
368
+ expiresAt: number;
369
+ heapIndex: number;
370
+ }
371
+ | {
372
+ kind: "admission";
373
+ reservation: PendingSyncAdmissionReservation;
374
+ expiresAt: number;
375
+ heapIndex: number;
376
+ };
377
+
378
+ type PendingMaybeSyncResponse = {
379
+ hashes: Set<string>;
380
+ target: string;
381
+ targetLifecycle: SyncDispatchTargetLifecycle;
382
+ expiresAt: number;
383
+ heapIndex: number;
384
+ };
385
+
386
+ type PendingMaybeSyncResponseAuthorizationEvent =
387
+ | "delivered"
388
+ | "fulfilled"
389
+ | "released";
390
+
391
+ type PendingMaybeSyncResponseAuthorization = {
392
+ batch: PendingMaybeSyncResponse;
393
+ hash: string;
394
+ waiters: Set<(event: PendingMaybeSyncResponseAuthorizationEvent) => void>;
395
+ requestDelivered?: boolean;
396
+ deliveryInFlight?: boolean;
397
+ settled?: "fulfilled" | "released";
398
+ active?: boolean;
399
+ };
400
+
401
+ type PendingMaybeSyncResponseReservation = {
402
+ release: () => void;
403
+ beginDelivery: () => void;
404
+ finishDelivery: () => void;
405
+ markDelivered: () => void;
406
+ newlyAuthorizedByTarget: Map<string, string[]>;
407
+ retained: () => boolean;
408
+ signal: AbortSignal;
409
+ };
410
+
411
+ type PendingMaybeSyncResponseReservationAttempt =
412
+ | {
413
+ kind: "reserved";
414
+ reservation: PendingMaybeSyncResponseReservation;
415
+ conflicts: PendingMaybeSyncResponseAuthorization[];
416
+ }
417
+ | {
418
+ kind: "capacity";
419
+ required: number;
420
+ }
421
+ | {
422
+ kind: "inactive";
423
+ };
424
+
425
+ type PendingMaybeSyncResponseWaiter = {
426
+ required: number;
427
+ associations: number;
428
+ order: number;
429
+ bypasses: number;
430
+ heapIndex: number;
431
+ fitHeapIndex: number;
432
+ wake: () => void;
433
+ };
434
+
435
+ export type AuthorizedMaybeSyncResponseLease = {
436
+ hashes: string[];
437
+ signal: AbortSignal;
438
+ release: (options?: { fulfilled?: boolean }) => void;
439
+ };
440
+
441
+ type SyncDispatchLifecycle = {
442
+ ownershipLifecycleController: AbortController;
443
+ callerSignal?: AbortSignal;
444
+ controller: AbortController;
445
+ targets: Map<string, SyncDispatchTargetLifecycle>;
446
+ retainedWork: number;
447
+ onOwnerOrCallerAbort: () => void;
448
+ dispatchFinished: boolean;
449
+ disposed: boolean;
450
+ abortAllOnTargetDisconnect: boolean;
451
+ };
452
+
453
+ type SyncDispatchTargetEpoch = {
454
+ id: number;
455
+ };
456
+
457
+ type SyncDispatchTargetLifecycle = {
458
+ lifecycle: SyncDispatchLifecycle;
459
+ target: string;
460
+ epoch: SyncDispatchTargetEpoch;
461
+ controller: AbortController;
462
+ batches: Set<PendingMaybeSyncResponse>;
463
+ responseLeases: number;
464
+ activeWaiters: number;
465
+ };
253
466
 
254
467
  const createDeferred = <T>() => {
255
468
  let resolve!: (value: T | PromiseLike<T>) => void;
@@ -266,6 +479,7 @@ type RepairSessionTargetState = {
266
479
  requestedCount: number;
267
480
  requestedTotalCount: number;
268
481
  attempts: number;
482
+ targetEpoch: SyncDispatchTargetEpoch;
269
483
  };
270
484
 
271
485
  type RepairSessionState = {
@@ -287,9 +501,50 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
287
501
  // map of hash to public keys that we can ask for entries
288
502
  syncInFlightQueue: Map<SyncableKey, PublicSignKey[]>;
289
503
  syncInFlightQueueInverted: Map<string, Set<SyncableKey>>;
504
+ private syncInFlightQueueExpiresAt: Map<SyncableKey, number>;
505
+ private syncInFlightQueueExpiryTimer?: ReturnType<typeof setTimeout>;
506
+ private pendingSyncExpiryHeap: PendingSyncExpiryNode[];
507
+ private pendingSyncKeyExpiryNodes: Map<SyncableKey, PendingSyncExpiryNode>;
508
+ private pendingSyncAdmissionExpiryNodes: Map<
509
+ PendingSyncAdmissionReservation,
510
+ PendingSyncExpiryNode
511
+ >;
512
+ private syncInFlightRetryIterator?: IterableIterator<
513
+ [SyncableKey, PublicSignKey[]]
514
+ >;
515
+ private syncInFlightRetryRemaining = 0;
516
+ private syncInFlightQueueClaimants: Map<SyncableKey, Set<string>>;
517
+ private syncInFlightQueueClaimantIndexes: Map<
518
+ SyncableKey,
519
+ Map<string, number>
520
+ >;
521
+ private syncInFlightQueueRoundRobinCursor: Map<SyncableKey, number>;
522
+ private syncInFlightQueuedCoordinates: Set<bigint>;
523
+ private syncInFlightQueuedHashByCoordinate: Map<bigint, string>;
524
+ private syncInFlightQueuedCoordinatesByHash: Map<string, Set<bigint>>;
525
+ private syncInFlightQueuedCoordinateRefreshIterator?: IterableIterator<bigint>;
526
+ private pendingSyncClaimCount: number;
527
+ private pendingSyncAdmissionCount: number;
528
+ private pendingSyncActiveAdmissionReservations: number;
529
+ private pendingSyncAdmissionCountByPeer: Map<string, number>;
530
+ private pendingSyncAdmissionIdentitiesByPeer: Map<string, Set<SyncableKey>>;
531
+ private pendingSyncAdmissionReservations: Set<PendingSyncAdmissionReservation>;
532
+ private pendingSyncAdmissionReservationsByPeer: Map<
533
+ string,
534
+ Set<PendingSyncAdmissionReservation>
535
+ >;
536
+ private pendingSyncAdmissionReservationsByIdentity: Map<
537
+ SyncableKey,
538
+ Set<PendingSyncAdmissionReservation>
539
+ >;
540
+ private pendingCoordinateLookupCount: number;
541
+ private pendingCoordinateLookupCountByPeer: Map<string, number>;
542
+ private pendingCoordinateResponseCount: number;
543
+ private pendingCoordinateResponseCountByPeer: Map<string, number>;
290
544
 
291
545
  // map of hash to public keys that we have asked for entries
292
546
  syncInFlight!: Map<string, Map<SyncableKey, { timestamp: number }>>;
547
+ private syncInFlightTargetsByKey: Map<SyncableKey, Set<string>>;
293
548
 
294
549
  rpc: RPC<TransportMessage, TransportMessage>;
295
550
  log: Log<any>;
@@ -305,6 +560,27 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
305
560
  ) => boolean;
306
561
  private sendRawExchangeHeads?: RawExchangeHeadsSender;
307
562
  private recentlySentExchangeHeads: Map<string, Map<string, number>>;
563
+ private pendingMaybeSyncResponses: Map<
564
+ string,
565
+ Map<string, PendingMaybeSyncResponseAuthorization>
566
+ >;
567
+ private pendingMaybeSyncResponseCount: number;
568
+ private pendingMaybeSyncResponseWaiters: Set<PendingMaybeSyncResponseWaiter>;
569
+ private pendingMaybeSyncResponseWaiterHeap: PendingMaybeSyncResponseWaiter[];
570
+ private pendingMaybeSyncResponseWaiterFitHeap: PendingMaybeSyncResponseWaiter[];
571
+ private pendingMaybeSyncResponseWaiterOrder: number;
572
+ private pendingMaybeSyncResponseWaiterAssociationCount: number;
573
+ private pendingMaybeSyncResponseWakeScheduled: boolean;
574
+ private pendingMaybeSyncResponseConflictWaiterCount: number;
575
+ private pendingMaybeSyncResponseBatches: Set<PendingMaybeSyncResponse>;
576
+ private pendingMaybeSyncResponseExpiryTimer?: ReturnType<typeof setTimeout>;
577
+ private pendingMaybeSyncResponseExpiryHeap: PendingMaybeSyncResponse[];
578
+ private activeMaybeSyncResponseCount: number;
579
+ private activeMaybeSyncResponseCountByPeer: Map<string, number>;
580
+ private syncDispatchLifecycleController: AbortController;
581
+ private syncDispatchTargetEpochCounter: number;
582
+ private syncDispatchTargetEpochs: Map<string, SyncDispatchTargetEpoch>;
583
+ private syncDispatchTargets: Map<string, Set<SyncDispatchTargetLifecycle>>;
308
584
  private repairSessionCounter: number;
309
585
  private repairSessions: Map<string, RepairSessionState>;
310
586
 
@@ -330,7 +606,30 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
330
606
  }) {
331
607
  this.syncInFlightQueue = new Map();
332
608
  this.syncInFlightQueueInverted = new Map();
609
+ this.syncInFlightQueueExpiresAt = new Map();
610
+ this.pendingSyncExpiryHeap = [];
611
+ this.pendingSyncKeyExpiryNodes = new Map();
612
+ this.pendingSyncAdmissionExpiryNodes = new Map();
613
+ this.syncInFlightQueueClaimants = new Map();
614
+ this.syncInFlightQueueClaimantIndexes = new Map();
615
+ this.syncInFlightQueueRoundRobinCursor = new Map();
616
+ this.syncInFlightQueuedCoordinates = new Set();
617
+ this.syncInFlightQueuedHashByCoordinate = new Map();
618
+ this.syncInFlightQueuedCoordinatesByHash = new Map();
619
+ this.pendingSyncClaimCount = 0;
620
+ this.pendingSyncAdmissionCount = 0;
621
+ this.pendingSyncActiveAdmissionReservations = 0;
622
+ this.pendingSyncAdmissionCountByPeer = new Map();
623
+ this.pendingSyncAdmissionIdentitiesByPeer = new Map();
624
+ this.pendingSyncAdmissionReservations = new Set();
625
+ this.pendingSyncAdmissionReservationsByPeer = new Map();
626
+ this.pendingSyncAdmissionReservationsByIdentity = new Map();
627
+ this.pendingCoordinateLookupCount = 0;
628
+ this.pendingCoordinateLookupCountByPeer = new Map();
629
+ this.pendingCoordinateResponseCount = 0;
630
+ this.pendingCoordinateResponseCountByPeer = new Map();
333
631
  this.syncInFlight = new Map();
632
+ this.syncInFlightTargetsByKey = new Map();
334
633
  this.rpc = properties.rpc;
335
634
  this.log = properties.log;
336
635
  this.entryIndex = properties.entryIndex;
@@ -341,6 +640,23 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
341
640
  this.isEntryRecentlyKnownByPeer = properties.isEntryRecentlyKnownByPeer;
342
641
  this.sendRawExchangeHeads = properties.sendRawExchangeHeads;
343
642
  this.recentlySentExchangeHeads = new Map();
643
+ this.pendingMaybeSyncResponses = new Map();
644
+ this.pendingMaybeSyncResponseCount = 0;
645
+ this.pendingMaybeSyncResponseWaiters = new Set();
646
+ this.pendingMaybeSyncResponseWaiterHeap = [];
647
+ this.pendingMaybeSyncResponseWaiterFitHeap = [];
648
+ this.pendingMaybeSyncResponseWaiterOrder = 0;
649
+ this.pendingMaybeSyncResponseWaiterAssociationCount = 0;
650
+ this.pendingMaybeSyncResponseWakeScheduled = false;
651
+ this.pendingMaybeSyncResponseConflictWaiterCount = 0;
652
+ this.pendingMaybeSyncResponseBatches = new Set();
653
+ this.pendingMaybeSyncResponseExpiryHeap = [];
654
+ this.activeMaybeSyncResponseCount = 0;
655
+ this.activeMaybeSyncResponseCountByPeer = new Map();
656
+ this.syncDispatchLifecycleController = new AbortController();
657
+ this.syncDispatchTargetEpochCounter = 0;
658
+ this.syncDispatchTargetEpochs = new Map();
659
+ this.syncDispatchTargets = new Map();
344
660
  this.repairSessionCounter = 0;
345
661
  this.repairSessions = new Map();
346
662
  }
@@ -389,21 +705,24 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
389
705
  private get maxHashesPerMessage() {
390
706
  const value = this.syncOptions?.maxSimpleHashesPerMessage;
391
707
  return value && Number.isFinite(value) && value > 0
392
- ? Math.floor(value)
708
+ ? Math.max(1, Math.floor(value))
393
709
  : DEFAULT_MAX_HASHES_PER_MESSAGE;
394
710
  }
395
711
 
396
712
  private get maxCoordinatesPerMessage() {
397
713
  const value = this.syncOptions?.maxSimpleCoordinatesPerMessage;
398
714
  return value && Number.isFinite(value) && value > 0
399
- ? Math.floor(value)
715
+ ? Math.min(
716
+ MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS,
717
+ Math.max(1, Math.floor(value)),
718
+ )
400
719
  : DEFAULT_MAX_COORDINATES_PER_MESSAGE;
401
720
  }
402
721
 
403
722
  private get maxConvergentTrackedHashes() {
404
723
  const value = this.syncOptions?.maxConvergentTrackedHashes;
405
724
  return value && Number.isFinite(value) && value > 0
406
- ? Math.floor(value)
725
+ ? Math.max(1, Math.floor(value))
407
726
  : DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
408
727
  }
409
728
 
@@ -459,493 +778,2850 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
459
778
  return out;
460
779
  }
461
780
 
462
- private isRepairSessionComplete(session: RepairSessionState): boolean {
463
- for (const state of session.targets.values()) {
464
- if (state.unresolved.size > 0) {
465
- return false;
466
- }
781
+ private forgetRecentlySentExchangeHeads(
782
+ hashes: Iterable<string>,
783
+ peer: PublicSignKey,
784
+ ): void {
785
+ const recentlySent = this.recentlySentExchangeHeads.get(peer.hashcode());
786
+ if (!recentlySent) {
787
+ return;
788
+ }
789
+ for (const hash of hashes) {
790
+ recentlySent.delete(hash);
791
+ }
792
+ if (recentlySent.size === 0) {
793
+ this.recentlySentExchangeHeads.delete(peer.hashcode());
467
794
  }
468
- return true;
469
795
  }
470
796
 
471
- private buildRepairSessionResult(
472
- session: RepairSessionState,
473
- completed: boolean,
474
- ): RepairSessionResult[] {
475
- const durationMs = Date.now() - session.startedAt;
476
- const out: RepairSessionResult[] = [];
477
- for (const [target, state] of session.targets) {
478
- const unresolved = [...state.unresolved];
479
- out.push({
797
+ private getOrCreateSyncDispatchTargetEpoch(
798
+ target: string,
799
+ ): SyncDispatchTargetEpoch {
800
+ let epoch = this.syncDispatchTargetEpochs.get(target);
801
+ if (!epoch) {
802
+ epoch = { id: ++this.syncDispatchTargetEpochCounter };
803
+ this.syncDispatchTargetEpochs.set(target, epoch);
804
+ }
805
+ return epoch;
806
+ }
807
+
808
+ private captureSyncDispatchLifecycle(
809
+ targets: string[],
810
+ callerSignal?: AbortSignal,
811
+ options?: {
812
+ abortAllOnTargetDisconnect?: boolean;
813
+ ownershipLifecycleController?: AbortController;
814
+ targetEpochs?: Map<string, SyncDispatchTargetEpoch>;
815
+ createTargetEpochs?: boolean;
816
+ },
817
+ ): SyncDispatchLifecycle {
818
+ const ownershipLifecycleController =
819
+ options?.ownershipLifecycleController ??
820
+ this.syncDispatchLifecycleController;
821
+ const lifecycle = {
822
+ ownershipLifecycleController,
823
+ callerSignal,
824
+ controller: new AbortController(),
825
+ targets: new Map<string, SyncDispatchTargetLifecycle>(),
826
+ retainedWork: 0,
827
+ onOwnerOrCallerAbort: () => {
828
+ const reason =
829
+ callerSignal?.aborted === true
830
+ ? callerSignal.reason
831
+ : ownershipLifecycleController.signal.reason;
832
+ this.abortSyncDispatchLifecycle(lifecycle, reason);
833
+ },
834
+ dispatchFinished: false,
835
+ disposed: false,
836
+ abortAllOnTargetDisconnect: options?.abortAllOnTargetDisconnect === true,
837
+ } satisfies SyncDispatchLifecycle;
838
+
839
+ for (const target of [...new Set(targets)]) {
840
+ const expectedEpoch = options?.targetEpochs?.get(target);
841
+ const currentEpoch = this.syncDispatchTargetEpochs.get(target);
842
+ const epoch =
843
+ expectedEpoch ??
844
+ currentEpoch ??
845
+ (options?.createTargetEpochs === false
846
+ ? undefined
847
+ : this.getOrCreateSyncDispatchTargetEpoch(target));
848
+ if (!epoch) {
849
+ continue;
850
+ }
851
+ const targetLifecycle: SyncDispatchTargetLifecycle = {
852
+ lifecycle,
480
853
  target,
481
- requested: state.requestedCount,
482
- resolved: state.requestedCount - unresolved.length,
483
- unresolved,
484
- attempts: state.attempts,
485
- durationMs,
486
- completed,
487
- requestedTotal: state.requestedTotalCount,
488
- truncated: session.truncated,
854
+ epoch,
855
+ controller: new AbortController(),
856
+ batches: new Set(),
857
+ responseLeases: 0,
858
+ activeWaiters: 0,
859
+ };
860
+ lifecycle.targets.set(target, targetLifecycle);
861
+ let activeForTarget = this.syncDispatchTargets.get(target);
862
+ if (!activeForTarget) {
863
+ activeForTarget = new Set();
864
+ this.syncDispatchTargets.set(target, activeForTarget);
865
+ }
866
+ activeForTarget.add(targetLifecycle);
867
+ if (this.syncDispatchTargetEpochs.get(target) !== epoch) {
868
+ this.abortSyncDispatchTarget(
869
+ targetLifecycle,
870
+ new Error("sync target lifecycle is stale"),
871
+ );
872
+ }
873
+ }
874
+
875
+ ownershipLifecycleController.signal.addEventListener(
876
+ "abort",
877
+ lifecycle.onOwnerOrCallerAbort,
878
+ { once: true },
879
+ );
880
+ if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
881
+ callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
882
+ once: true,
489
883
  });
490
884
  }
491
- return out;
885
+ if (
886
+ this.closed === true ||
887
+ ownershipLifecycleController !== this.syncDispatchLifecycleController ||
888
+ ownershipLifecycleController.signal.aborted ||
889
+ callerSignal?.aborted
890
+ ) {
891
+ lifecycle.onOwnerOrCallerAbort();
892
+ }
893
+ return lifecycle;
492
894
  }
493
895
 
494
- private finalizeRepairSession(sessionId: string, completed: boolean): void {
495
- const session = this.repairSessions.get(sessionId);
496
- if (!session) {
497
- return;
896
+ private abortSyncDispatchTarget(
897
+ targetLifecycle: SyncDispatchTargetLifecycle,
898
+ reason?: unknown,
899
+ ): void {
900
+ if (!targetLifecycle.controller.signal.aborted) {
901
+ targetLifecycle.controller.abort(reason);
498
902
  }
499
- this.repairSessions.delete(sessionId);
500
- session.cancelled = true;
501
- if (session.timer) {
502
- clearTimeout(session.timer);
903
+ for (const batch of [...targetLifecycle.batches]) {
904
+ this.removePendingMaybeSyncResponseBatch(batch);
503
905
  }
504
- session.deferred.resolve(this.buildRepairSessionResult(session, completed));
906
+ this.maybeDisposeSyncDispatchLifecycle(targetLifecycle.lifecycle);
505
907
  }
506
908
 
507
- private async refreshRepairSessionState(sessionId: string): Promise<void> {
508
- const session = this.repairSessions.get(sessionId);
509
- if (!session) {
510
- return;
909
+ private abortSyncDispatchLifecycle(
910
+ lifecycle: SyncDispatchLifecycle,
911
+ reason?: unknown,
912
+ ): void {
913
+ if (!lifecycle.controller.signal.aborted) {
914
+ lifecycle.controller.abort(reason);
511
915
  }
512
- for (const state of session.targets.values()) {
513
- const resolved =
514
- typeof this.log.hasMany === "function"
515
- ? await this.log.hasMany(state.unresolved)
516
- : await this.getExistingRepairHashes(state.unresolved);
517
- for (const hash of resolved) {
518
- state.unresolved.delete(hash);
519
- }
916
+ for (const targetLifecycle of lifecycle.targets.values()) {
917
+ this.abortSyncDispatchTarget(targetLifecycle, reason);
520
918
  }
919
+ this.maybeDisposeSyncDispatchLifecycle(lifecycle);
521
920
  }
522
921
 
523
- private async getExistingRepairHashes(
524
- hashes: Iterable<string>,
525
- ): Promise<Set<string>> {
526
- const resolved = new Set<string>();
527
- for (const hash of hashes) {
528
- if (await this.log.has(hash)) {
529
- resolved.add(hash);
530
- }
531
- }
532
- return resolved;
922
+ private finishSyncDispatchLifecycle(lifecycle: SyncDispatchLifecycle): void {
923
+ lifecycle.dispatchFinished = true;
924
+ this.maybeDisposeSyncDispatchLifecycle(lifecycle);
533
925
  }
534
926
 
535
- private markRepairSessionResolvedHashes(hashes: string[]): void {
536
- if (hashes.length === 0 || this.repairSessions.size === 0) {
927
+ private maybeDisposeSyncDispatchLifecycle(
928
+ lifecycle: SyncDispatchLifecycle,
929
+ ): void {
930
+ if (
931
+ lifecycle.disposed ||
932
+ !lifecycle.dispatchFinished ||
933
+ lifecycle.retainedWork > 0
934
+ ) {
537
935
  return;
538
936
  }
539
- for (const [sessionId, session] of this.repairSessions) {
540
- for (const state of session.targets.values()) {
541
- for (const hash of hashes) {
542
- state.unresolved.delete(hash);
543
- }
544
- }
545
- if (this.isRepairSessionComplete(session)) {
546
- this.finalizeRepairSession(sessionId, true);
937
+ lifecycle.disposed = true;
938
+ lifecycle.ownershipLifecycleController.signal.removeEventListener(
939
+ "abort",
940
+ lifecycle.onOwnerOrCallerAbort,
941
+ );
942
+ if (
943
+ lifecycle.callerSignal &&
944
+ lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
945
+ ) {
946
+ lifecycle.callerSignal.removeEventListener(
947
+ "abort",
948
+ lifecycle.onOwnerOrCallerAbort,
949
+ );
950
+ }
951
+ for (const targetLifecycle of lifecycle.targets.values()) {
952
+ const activeForTarget = this.syncDispatchTargets.get(
953
+ targetLifecycle.target,
954
+ );
955
+ activeForTarget?.delete(targetLifecycle);
956
+ if (activeForTarget?.size === 0) {
957
+ this.syncDispatchTargets.delete(targetLifecycle.target);
547
958
  }
548
959
  }
549
960
  }
550
961
 
551
- private markRepairSessionResolvedHash(hash: string): void {
552
- if (this.repairSessions.size === 0) {
553
- return;
962
+ private isSyncDispatchLifecycleActive(
963
+ lifecycle: SyncDispatchLifecycle,
964
+ target?: string,
965
+ ): boolean {
966
+ if (
967
+ this.closed === true ||
968
+ lifecycle.disposed ||
969
+ lifecycle.ownershipLifecycleController !==
970
+ this.syncDispatchLifecycleController ||
971
+ lifecycle.ownershipLifecycleController.signal.aborted ||
972
+ lifecycle.callerSignal?.aborted ||
973
+ lifecycle.controller.signal.aborted
974
+ ) {
975
+ return false;
554
976
  }
555
- for (const [sessionId, session] of this.repairSessions) {
556
- for (const state of session.targets.values()) {
557
- state.unresolved.delete(hash);
558
- }
559
- if (this.isRepairSessionComplete(session)) {
560
- this.finalizeRepairSession(sessionId, true);
561
- }
977
+ if (target === undefined) {
978
+ return true;
562
979
  }
980
+ const targetLifecycle = lifecycle.targets.get(target);
981
+ return (
982
+ targetLifecycle !== undefined &&
983
+ !targetLifecycle.controller.signal.aborted &&
984
+ this.syncDispatchTargetEpochs.get(target) === targetLifecycle.epoch
985
+ );
563
986
  }
564
987
 
565
- private async runRepairSession(sessionId: string): Promise<void> {
566
- const session = this.repairSessions.get(sessionId);
567
- if (!session) {
568
- return;
569
- }
570
-
571
- let previousDelay = 0;
572
- for (const delayMs of session.retryIntervalsMs) {
573
- if (!this.repairSessions.has(sessionId) || this.closed) {
574
- return;
575
- }
576
-
577
- const waitMs = Math.max(0, delayMs - previousDelay);
578
- previousDelay = delayMs;
579
- if (waitMs > 0) {
580
- await new Promise<void>((resolve) => {
581
- const timer = setTimeout(resolve, waitMs);
582
- timer.unref?.();
583
- });
584
- }
585
- if (!this.repairSessions.has(sessionId) || this.closed) {
586
- return;
587
- }
988
+ private getSyncDispatchSignal(
989
+ lifecycle: SyncDispatchLifecycle,
990
+ target: string,
991
+ ): AbortSignal {
992
+ return (
993
+ lifecycle.targets.get(target)?.controller.signal ??
994
+ lifecycle.controller.signal
995
+ );
996
+ }
588
997
 
589
- await this.refreshRepairSessionState(sessionId);
590
- const current = this.repairSessions.get(sessionId);
591
- if (!current) {
592
- return;
593
- }
594
- if (this.isRepairSessionComplete(current)) {
595
- this.finalizeRepairSession(sessionId, true);
596
- return;
597
- }
998
+ private pendingMaybeSyncResponseWaiterBefore(
999
+ left: PendingMaybeSyncResponseWaiter,
1000
+ right: PendingMaybeSyncResponseWaiter,
1001
+ ): boolean {
1002
+ return left.order < right.order;
1003
+ }
598
1004
 
599
- for (const [target, state] of current.targets) {
600
- if (state.unresolved.size === 0) {
601
- continue;
602
- }
603
- state.attempts += 1;
604
- try {
605
- await this.requestSync([...state.unresolved], [target]);
606
- } catch {
607
- // Best-effort: keep unresolved and let retries/timeout determine outcome.
608
- }
609
- }
1005
+ private swapPendingMaybeSyncResponseWaiters(
1006
+ left: number,
1007
+ right: number,
1008
+ ): void {
1009
+ const leftWaiter = this.pendingMaybeSyncResponseWaiterHeap[left]!;
1010
+ const rightWaiter = this.pendingMaybeSyncResponseWaiterHeap[right]!;
1011
+ this.pendingMaybeSyncResponseWaiterHeap[left] = rightWaiter;
1012
+ this.pendingMaybeSyncResponseWaiterHeap[right] = leftWaiter;
1013
+ rightWaiter.heapIndex = left;
1014
+ leftWaiter.heapIndex = right;
1015
+ }
610
1016
 
611
- await this.refreshRepairSessionState(sessionId);
612
- const afterSend = this.repairSessions.get(sessionId);
613
- if (!afterSend) {
614
- return;
615
- }
616
- if (this.isRepairSessionComplete(afterSend)) {
617
- this.finalizeRepairSession(sessionId, true);
618
- return;
1017
+ private pushPendingMaybeSyncResponseWaiter(
1018
+ waiter: PendingMaybeSyncResponseWaiter,
1019
+ ): void {
1020
+ waiter.heapIndex = this.pendingMaybeSyncResponseWaiterHeap.length;
1021
+ this.pendingMaybeSyncResponseWaiterHeap.push(waiter);
1022
+ let index = waiter.heapIndex;
1023
+ while (index > 0) {
1024
+ const parent = Math.floor((index - 1) / 2);
1025
+ if (
1026
+ this.pendingMaybeSyncResponseWaiterBefore(
1027
+ this.pendingMaybeSyncResponseWaiterHeap[parent]!,
1028
+ this.pendingMaybeSyncResponseWaiterHeap[index]!,
1029
+ )
1030
+ ) {
1031
+ break;
619
1032
  }
1033
+ this.swapPendingMaybeSyncResponseWaiters(parent, index);
1034
+ index = parent;
1035
+ }
1036
+ this.pushPendingMaybeSyncResponseFitWaiter(waiter);
1037
+ }
620
1038
 
621
- if (afterSend.mode === "best-effort") {
622
- this.finalizeRepairSession(sessionId, false);
623
- return;
1039
+ private removePendingMaybeSyncResponseWaiter(
1040
+ waiter: PendingMaybeSyncResponseWaiter,
1041
+ ): void {
1042
+ this.removePendingMaybeSyncResponseFitWaiter(waiter);
1043
+ const index = waiter.heapIndex;
1044
+ if (
1045
+ index < 0 ||
1046
+ index >= this.pendingMaybeSyncResponseWaiterHeap.length ||
1047
+ this.pendingMaybeSyncResponseWaiterHeap[index] !== waiter
1048
+ ) {
1049
+ return;
1050
+ }
1051
+ const last = this.pendingMaybeSyncResponseWaiterHeap.pop()!;
1052
+ waiter.heapIndex = -1;
1053
+ if (index >= this.pendingMaybeSyncResponseWaiterHeap.length) {
1054
+ return;
1055
+ }
1056
+ this.pendingMaybeSyncResponseWaiterHeap[index] = last;
1057
+ last.heapIndex = index;
1058
+ let current = index;
1059
+ while (current > 0) {
1060
+ const parent = Math.floor((current - 1) / 2);
1061
+ if (
1062
+ this.pendingMaybeSyncResponseWaiterBefore(
1063
+ this.pendingMaybeSyncResponseWaiterHeap[parent]!,
1064
+ this.pendingMaybeSyncResponseWaiterHeap[current]!,
1065
+ )
1066
+ ) {
1067
+ break;
624
1068
  }
1069
+ this.swapPendingMaybeSyncResponseWaiters(parent, current);
1070
+ current = parent;
625
1071
  }
626
-
627
1072
  for (;;) {
628
- if (!this.repairSessions.has(sessionId) || this.closed) {
629
- return;
1073
+ const left = current * 2 + 1;
1074
+ const right = left + 1;
1075
+ let smallest = current;
1076
+ if (
1077
+ left < this.pendingMaybeSyncResponseWaiterHeap.length &&
1078
+ this.pendingMaybeSyncResponseWaiterBefore(
1079
+ this.pendingMaybeSyncResponseWaiterHeap[left]!,
1080
+ this.pendingMaybeSyncResponseWaiterHeap[smallest]!,
1081
+ )
1082
+ ) {
1083
+ smallest = left;
630
1084
  }
631
- await this.refreshRepairSessionState(sessionId);
632
- const current = this.repairSessions.get(sessionId);
633
- if (!current) {
634
- return;
1085
+ if (
1086
+ right < this.pendingMaybeSyncResponseWaiterHeap.length &&
1087
+ this.pendingMaybeSyncResponseWaiterBefore(
1088
+ this.pendingMaybeSyncResponseWaiterHeap[right]!,
1089
+ this.pendingMaybeSyncResponseWaiterHeap[smallest]!,
1090
+ )
1091
+ ) {
1092
+ smallest = right;
635
1093
  }
636
- if (this.isRepairSessionComplete(current)) {
637
- this.finalizeRepairSession(sessionId, true);
638
- return;
1094
+ if (smallest === current) {
1095
+ break;
639
1096
  }
640
- await new Promise<void>((resolve) => {
641
- const timer = setTimeout(resolve, SESSION_POLL_INTERVAL_MS);
642
- timer.unref?.();
643
- });
1097
+ this.swapPendingMaybeSyncResponseWaiters(current, smallest);
1098
+ current = smallest;
644
1099
  }
645
1100
  }
646
1101
 
647
- private getQueuedSyncKey(key: SyncableKey): SyncableKey | undefined {
648
- if (this.syncInFlightQueue.has(key)) {
1102
+ private pendingMaybeSyncResponseFitWaiterBefore(
1103
+ left: PendingMaybeSyncResponseWaiter,
1104
+ right: PendingMaybeSyncResponseWaiter,
1105
+ ): boolean {
1106
+ return (
1107
+ left.required < right.required ||
1108
+ (left.required === right.required && left.order < right.order)
1109
+ );
1110
+ }
1111
+
1112
+ private swapPendingMaybeSyncResponseFitWaiters(
1113
+ left: number,
1114
+ right: number,
1115
+ ): void {
1116
+ const leftWaiter = this.pendingMaybeSyncResponseWaiterFitHeap[left]!;
1117
+ const rightWaiter = this.pendingMaybeSyncResponseWaiterFitHeap[right]!;
1118
+ this.pendingMaybeSyncResponseWaiterFitHeap[left] = rightWaiter;
1119
+ this.pendingMaybeSyncResponseWaiterFitHeap[right] = leftWaiter;
1120
+ rightWaiter.fitHeapIndex = left;
1121
+ leftWaiter.fitHeapIndex = right;
1122
+ }
1123
+
1124
+ private pushPendingMaybeSyncResponseFitWaiter(
1125
+ waiter: PendingMaybeSyncResponseWaiter,
1126
+ ): void {
1127
+ waiter.fitHeapIndex = this.pendingMaybeSyncResponseWaiterFitHeap.length;
1128
+ this.pendingMaybeSyncResponseWaiterFitHeap.push(waiter);
1129
+ let index = waiter.fitHeapIndex;
1130
+ while (index > 0) {
1131
+ const parent = Math.floor((index - 1) / 2);
1132
+ if (
1133
+ this.pendingMaybeSyncResponseFitWaiterBefore(
1134
+ this.pendingMaybeSyncResponseWaiterFitHeap[parent]!,
1135
+ this.pendingMaybeSyncResponseWaiterFitHeap[index]!,
1136
+ )
1137
+ ) {
1138
+ break;
1139
+ }
1140
+ this.swapPendingMaybeSyncResponseFitWaiters(parent, index);
1141
+ index = parent;
1142
+ }
1143
+ }
1144
+
1145
+ private removePendingMaybeSyncResponseFitWaiter(
1146
+ waiter: PendingMaybeSyncResponseWaiter,
1147
+ ): void {
1148
+ const index = waiter.fitHeapIndex;
1149
+ if (
1150
+ index < 0 ||
1151
+ index >= this.pendingMaybeSyncResponseWaiterFitHeap.length ||
1152
+ this.pendingMaybeSyncResponseWaiterFitHeap[index] !== waiter
1153
+ ) {
1154
+ return;
1155
+ }
1156
+ const last = this.pendingMaybeSyncResponseWaiterFitHeap.pop()!;
1157
+ waiter.fitHeapIndex = -1;
1158
+ if (index >= this.pendingMaybeSyncResponseWaiterFitHeap.length) {
1159
+ return;
1160
+ }
1161
+ this.pendingMaybeSyncResponseWaiterFitHeap[index] = last;
1162
+ last.fitHeapIndex = index;
1163
+ let current = index;
1164
+ while (current > 0) {
1165
+ const parent = Math.floor((current - 1) / 2);
1166
+ if (
1167
+ this.pendingMaybeSyncResponseFitWaiterBefore(
1168
+ this.pendingMaybeSyncResponseWaiterFitHeap[parent]!,
1169
+ this.pendingMaybeSyncResponseWaiterFitHeap[current]!,
1170
+ )
1171
+ ) {
1172
+ break;
1173
+ }
1174
+ this.swapPendingMaybeSyncResponseFitWaiters(parent, current);
1175
+ current = parent;
1176
+ }
1177
+ for (;;) {
1178
+ const left = current * 2 + 1;
1179
+ const right = left + 1;
1180
+ let smallest = current;
1181
+ if (
1182
+ left < this.pendingMaybeSyncResponseWaiterFitHeap.length &&
1183
+ this.pendingMaybeSyncResponseFitWaiterBefore(
1184
+ this.pendingMaybeSyncResponseWaiterFitHeap[left]!,
1185
+ this.pendingMaybeSyncResponseWaiterFitHeap[smallest]!,
1186
+ )
1187
+ ) {
1188
+ smallest = left;
1189
+ }
1190
+ if (
1191
+ right < this.pendingMaybeSyncResponseWaiterFitHeap.length &&
1192
+ this.pendingMaybeSyncResponseFitWaiterBefore(
1193
+ this.pendingMaybeSyncResponseWaiterFitHeap[right]!,
1194
+ this.pendingMaybeSyncResponseWaiterFitHeap[smallest]!,
1195
+ )
1196
+ ) {
1197
+ smallest = right;
1198
+ }
1199
+ if (smallest === current) {
1200
+ break;
1201
+ }
1202
+ this.swapPendingMaybeSyncResponseFitWaiters(current, smallest);
1203
+ current = smallest;
1204
+ }
1205
+ }
1206
+
1207
+ private notifyPendingMaybeSyncResponseWaiter(): void {
1208
+ const waiter = this.pendingMaybeSyncResponseWaiterHeap[0];
1209
+ const available =
1210
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
1211
+ this.pendingMaybeSyncResponseCount;
1212
+ if (!waiter) {
1213
+ return;
1214
+ }
1215
+ if (waiter.required <= available) {
1216
+ waiter.wake();
1217
+ return;
1218
+ }
1219
+ if (waiter.bypasses >= MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITER_BYPASSES) {
1220
+ // Reserve newly freed capacity for the oldest large request after a
1221
+ // bounded number of smaller requests have bypassed it.
1222
+ return;
1223
+ }
1224
+ const candidate = this.pendingMaybeSyncResponseWaiterFitHeap[0];
1225
+ if (candidate && candidate.required <= available) {
1226
+ waiter.bypasses += 1;
1227
+ candidate.wake();
1228
+ }
1229
+ }
1230
+
1231
+ private schedulePendingMaybeSyncResponseWaiter(): void {
1232
+ if (this.pendingMaybeSyncResponseWakeScheduled) {
1233
+ return;
1234
+ }
1235
+ this.pendingMaybeSyncResponseWakeScheduled = true;
1236
+ queueMicrotask(() => {
1237
+ this.pendingMaybeSyncResponseWakeScheduled = false;
1238
+ this.notifyPendingMaybeSyncResponseWaiter();
1239
+ });
1240
+ }
1241
+
1242
+ private swapPendingMaybeSyncResponseExpiry(
1243
+ left: number,
1244
+ right: number,
1245
+ ): void {
1246
+ const leftBatch = this.pendingMaybeSyncResponseExpiryHeap[left]!;
1247
+ const rightBatch = this.pendingMaybeSyncResponseExpiryHeap[right]!;
1248
+ this.pendingMaybeSyncResponseExpiryHeap[left] = rightBatch;
1249
+ this.pendingMaybeSyncResponseExpiryHeap[right] = leftBatch;
1250
+ rightBatch.heapIndex = left;
1251
+ leftBatch.heapIndex = right;
1252
+ }
1253
+
1254
+ private pushPendingMaybeSyncResponseExpiry(
1255
+ batch: PendingMaybeSyncResponse,
1256
+ ): void {
1257
+ batch.heapIndex = this.pendingMaybeSyncResponseExpiryHeap.length;
1258
+ this.pendingMaybeSyncResponseExpiryHeap.push(batch);
1259
+ let index = batch.heapIndex;
1260
+ while (index > 0) {
1261
+ const parent = Math.floor((index - 1) / 2);
1262
+ if (
1263
+ this.pendingMaybeSyncResponseExpiryHeap[parent]!.expiresAt <=
1264
+ this.pendingMaybeSyncResponseExpiryHeap[index]!.expiresAt
1265
+ ) {
1266
+ break;
1267
+ }
1268
+ this.swapPendingMaybeSyncResponseExpiry(parent, index);
1269
+ index = parent;
1270
+ }
1271
+ }
1272
+
1273
+ private removePendingMaybeSyncResponseExpiry(
1274
+ batch: PendingMaybeSyncResponse,
1275
+ ): void {
1276
+ const index = batch.heapIndex;
1277
+ if (
1278
+ index < 0 ||
1279
+ index >= this.pendingMaybeSyncResponseExpiryHeap.length ||
1280
+ this.pendingMaybeSyncResponseExpiryHeap[index] !== batch
1281
+ ) {
1282
+ return;
1283
+ }
1284
+ const last = this.pendingMaybeSyncResponseExpiryHeap.pop()!;
1285
+ batch.heapIndex = -1;
1286
+ if (index >= this.pendingMaybeSyncResponseExpiryHeap.length) {
1287
+ return;
1288
+ }
1289
+ this.pendingMaybeSyncResponseExpiryHeap[index] = last;
1290
+ last.heapIndex = index;
1291
+ let current = index;
1292
+ while (current > 0) {
1293
+ const parent = Math.floor((current - 1) / 2);
1294
+ if (
1295
+ this.pendingMaybeSyncResponseExpiryHeap[parent]!.expiresAt <=
1296
+ this.pendingMaybeSyncResponseExpiryHeap[current]!.expiresAt
1297
+ ) {
1298
+ break;
1299
+ }
1300
+ this.swapPendingMaybeSyncResponseExpiry(parent, current);
1301
+ current = parent;
1302
+ }
1303
+ for (;;) {
1304
+ const left = current * 2 + 1;
1305
+ const right = left + 1;
1306
+ let smallest = current;
1307
+ if (
1308
+ left < this.pendingMaybeSyncResponseExpiryHeap.length &&
1309
+ this.pendingMaybeSyncResponseExpiryHeap[left]!.expiresAt <
1310
+ this.pendingMaybeSyncResponseExpiryHeap[smallest]!.expiresAt
1311
+ ) {
1312
+ smallest = left;
1313
+ }
1314
+ if (
1315
+ right < this.pendingMaybeSyncResponseExpiryHeap.length &&
1316
+ this.pendingMaybeSyncResponseExpiryHeap[right]!.expiresAt <
1317
+ this.pendingMaybeSyncResponseExpiryHeap[smallest]!.expiresAt
1318
+ ) {
1319
+ smallest = right;
1320
+ }
1321
+ if (smallest === current) {
1322
+ break;
1323
+ }
1324
+ this.swapPendingMaybeSyncResponseExpiry(current, smallest);
1325
+ current = smallest;
1326
+ }
1327
+ }
1328
+
1329
+ private schedulePendingMaybeSyncResponseExpiry(): void {
1330
+ if (
1331
+ this.pendingMaybeSyncResponseExpiryTimer ||
1332
+ this.pendingMaybeSyncResponseExpiryHeap.length === 0
1333
+ ) {
1334
+ return;
1335
+ }
1336
+ const earliest = this.pendingMaybeSyncResponseExpiryHeap[0]!.expiresAt;
1337
+ this.pendingMaybeSyncResponseExpiryTimer = setTimeout(
1338
+ () => {
1339
+ this.pendingMaybeSyncResponseExpiryTimer = undefined;
1340
+ this.expirePendingMaybeSyncResponses();
1341
+ this.schedulePendingMaybeSyncResponseExpiry();
1342
+ },
1343
+ Math.max(0, earliest - Date.now()),
1344
+ );
1345
+ this.pendingMaybeSyncResponseExpiryTimer.unref?.();
1346
+ }
1347
+
1348
+ private expirePendingMaybeSyncResponses(now = Date.now()): void {
1349
+ for (;;) {
1350
+ const batch = this.pendingMaybeSyncResponseExpiryHeap[0];
1351
+ if (!batch || batch.expiresAt > now) {
1352
+ break;
1353
+ }
1354
+ this.removePendingMaybeSyncResponseBatch(batch);
1355
+ }
1356
+ }
1357
+
1358
+ private settlePendingMaybeSyncResponseAuthorization(
1359
+ authorization: PendingMaybeSyncResponseAuthorization,
1360
+ fulfilled: boolean,
1361
+ ): void {
1362
+ if (authorization.settled) {
1363
+ return;
1364
+ }
1365
+ authorization.settled = fulfilled ? "fulfilled" : "released";
1366
+ const waiters = [...authorization.waiters];
1367
+ authorization.waiters.clear();
1368
+ for (const waiter of waiters) {
1369
+ waiter(fulfilled ? "fulfilled" : "released");
1370
+ }
1371
+ }
1372
+
1373
+ private doesPendingMaybeSyncResponseScopeMatch(
1374
+ authorization: PendingMaybeSyncResponseAuthorization,
1375
+ lifecycle: SyncDispatchLifecycle,
1376
+ target: string,
1377
+ ): boolean {
1378
+ const owner = authorization.batch.targetLifecycle;
1379
+ return (
1380
+ owner.epoch === lifecycle.targets.get(target)?.epoch &&
1381
+ owner.lifecycle.ownershipLifecycleController ===
1382
+ lifecycle.ownershipLifecycleController &&
1383
+ owner.lifecycle.callerSignal === lifecycle.callerSignal
1384
+ );
1385
+ }
1386
+
1387
+ private waitForPendingMaybeSyncResponseConflicts(
1388
+ conflicts: PendingMaybeSyncResponseAuthorization[],
1389
+ lifecycle: SyncDispatchLifecycle,
1390
+ target: string,
1391
+ retainedAssociations: number,
1392
+ ): Promise<string[]> {
1393
+ const unique = [...new Set(conflicts)];
1394
+ if (
1395
+ this.pendingMaybeSyncResponseWaiterAssociationCount +
1396
+ retainedAssociations >
1397
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES
1398
+ ) {
1399
+ return Promise.resolve([]);
1400
+ }
1401
+ const available =
1402
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
1403
+ this.pendingMaybeSyncResponseConflictWaiterCount;
1404
+ const admitted = unique.slice(0, Math.max(0, available));
1405
+ if (admitted.length === 0) {
1406
+ return Promise.resolve([]);
1407
+ }
1408
+ this.pendingMaybeSyncResponseConflictWaiterCount += admitted.length;
1409
+ this.pendingMaybeSyncResponseWaiterAssociationCount += retainedAssociations;
1410
+ const targetSignal =
1411
+ lifecycle.targets.get(target)?.controller.signal ??
1412
+ lifecycle.controller.signal;
1413
+ return new Promise<string[]>((resolve) => {
1414
+ const retry = new Set<string>();
1415
+ const callbacks = new Map<
1416
+ PendingMaybeSyncResponseAuthorization,
1417
+ (event: PendingMaybeSyncResponseAuthorizationEvent) => void
1418
+ >();
1419
+ let remaining = admitted.length;
1420
+ let groupSettled = false;
1421
+ const finishGroup = () => {
1422
+ if (groupSettled || remaining !== 0) {
1423
+ return;
1424
+ }
1425
+ groupSettled = true;
1426
+ lifecycle.controller.signal.removeEventListener("abort", abort);
1427
+ if (targetSignal !== lifecycle.controller.signal) {
1428
+ targetSignal.removeEventListener("abort", abort);
1429
+ }
1430
+ this.pendingMaybeSyncResponseWaiterAssociationCount -=
1431
+ retainedAssociations;
1432
+ resolve([...retry]);
1433
+ };
1434
+ const finishOne = (
1435
+ authorization: PendingMaybeSyncResponseAuthorization,
1436
+ event: PendingMaybeSyncResponseAuthorizationEvent,
1437
+ ) => {
1438
+ if (
1439
+ event === "delivered" &&
1440
+ (authorization.active === true ||
1441
+ !this.doesPendingMaybeSyncResponseScopeMatch(
1442
+ authorization,
1443
+ lifecycle,
1444
+ target,
1445
+ ))
1446
+ ) {
1447
+ return;
1448
+ }
1449
+ const callback = callbacks.get(authorization);
1450
+ if (!callback) {
1451
+ return;
1452
+ }
1453
+ callbacks.delete(authorization);
1454
+ authorization.waiters.delete(callback);
1455
+ this.pendingMaybeSyncResponseConflictWaiterCount -= 1;
1456
+ if (
1457
+ event === "released" &&
1458
+ this.isSyncDispatchLifecycleActive(lifecycle, target)
1459
+ ) {
1460
+ retry.add(authorization.hash);
1461
+ }
1462
+ remaining -= 1;
1463
+ finishGroup();
1464
+ };
1465
+ const abort = () => {
1466
+ for (const authorization of [...callbacks.keys()]) {
1467
+ finishOne(authorization, "fulfilled");
1468
+ }
1469
+ };
1470
+ lifecycle.controller.signal.addEventListener("abort", abort, {
1471
+ once: true,
1472
+ });
1473
+ if (targetSignal !== lifecycle.controller.signal) {
1474
+ targetSignal.addEventListener("abort", abort, { once: true });
1475
+ }
1476
+ for (const authorization of admitted) {
1477
+ const callback = (event: PendingMaybeSyncResponseAuthorizationEvent) =>
1478
+ finishOne(authorization, event);
1479
+ callbacks.set(authorization, callback);
1480
+ authorization.waiters.add(callback);
1481
+ if (authorization.settled) {
1482
+ callback(authorization.settled);
1483
+ } else if (
1484
+ authorization.requestDelivered === true &&
1485
+ authorization.active !== true
1486
+ ) {
1487
+ callback("delivered");
1488
+ }
1489
+ }
1490
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
1491
+ abort();
1492
+ }
1493
+ });
1494
+ }
1495
+
1496
+ private removePendingMaybeSyncResponseBatch(
1497
+ batch: PendingMaybeSyncResponse,
1498
+ ): void {
1499
+ const pendingForTarget = this.pendingMaybeSyncResponses.get(batch.target);
1500
+ let removed = 0;
1501
+ if (pendingForTarget) {
1502
+ for (const hash of batch.hashes) {
1503
+ const authorization = pendingForTarget.get(hash);
1504
+ if (authorization?.batch !== batch) {
1505
+ continue;
1506
+ }
1507
+ this.settlePendingMaybeSyncResponseAuthorization(authorization, false);
1508
+ pendingForTarget.delete(hash);
1509
+ if (authorization.deliveryInFlight !== true) {
1510
+ removed += 1;
1511
+ }
1512
+ }
1513
+ if (pendingForTarget.size === 0) {
1514
+ this.pendingMaybeSyncResponses.delete(batch.target);
1515
+ }
1516
+ }
1517
+ this.pendingMaybeSyncResponseCount -= removed;
1518
+ this.pendingMaybeSyncResponseBatches.delete(batch);
1519
+ this.removePendingMaybeSyncResponseExpiry(batch);
1520
+ if (batch.targetLifecycle.batches.delete(batch)) {
1521
+ batch.targetLifecycle.lifecycle.retainedWork -= 1;
1522
+ }
1523
+ batch.hashes.clear();
1524
+ if (
1525
+ this.pendingMaybeSyncResponseExpiryHeap.length === 0 &&
1526
+ this.pendingMaybeSyncResponseExpiryTimer
1527
+ ) {
1528
+ clearTimeout(this.pendingMaybeSyncResponseExpiryTimer);
1529
+ this.pendingMaybeSyncResponseExpiryTimer = undefined;
1530
+ }
1531
+ if (removed > 0) {
1532
+ this.schedulePendingMaybeSyncResponseWaiter();
1533
+ }
1534
+ this.maybeDisposeSyncDispatchLifecycle(batch.targetLifecycle.lifecycle);
1535
+ }
1536
+
1537
+ private clearPendingMaybeSyncResponses(target?: string): void {
1538
+ const batches =
1539
+ target === undefined
1540
+ ? [...this.pendingMaybeSyncResponseBatches]
1541
+ : [
1542
+ ...new Set(
1543
+ [
1544
+ ...(this.pendingMaybeSyncResponses.get(target)?.values() ?? []),
1545
+ ].map((authorization) => authorization.batch),
1546
+ ),
1547
+ ];
1548
+ for (const batch of batches) {
1549
+ this.removePendingMaybeSyncResponseBatch(batch);
1550
+ }
1551
+ }
1552
+
1553
+ private tryReservePendingMaybeSyncResponse(properties: {
1554
+ hashes: Iterable<string>;
1555
+ targets: string[];
1556
+ lifecycle: SyncDispatchLifecycle;
1557
+ }): PendingMaybeSyncResponseReservationAttempt {
1558
+ // Timers are only a cleanup aid. Enforce absolute deadlines at the
1559
+ // admission boundary as well, including for unrelated fresh hashes.
1560
+ this.expirePendingMaybeSyncResponses();
1561
+ const hashes = [...new Set(properties.hashes)];
1562
+ const targets = [...new Set(properties.targets)];
1563
+ if (
1564
+ !this.isSyncDispatchLifecycleActive(properties.lifecycle) ||
1565
+ targets.some(
1566
+ (target) =>
1567
+ !this.isSyncDispatchLifecycleActive(properties.lifecycle, target),
1568
+ )
1569
+ ) {
1570
+ return { kind: "inactive" };
1571
+ }
1572
+
1573
+ const hashesToAddByTarget = new Map<string, string[]>();
1574
+ const conflicts: PendingMaybeSyncResponseAuthorization[] = [];
1575
+ let required = 0;
1576
+ for (const target of targets) {
1577
+ const hashesToAdd: string[] = [];
1578
+ for (const hash of hashes) {
1579
+ let existing = this.pendingMaybeSyncResponses.get(target)?.get(hash);
1580
+ if (
1581
+ existing &&
1582
+ existing.active !== true &&
1583
+ (existing.batch.expiresAt <= Date.now() ||
1584
+ !this.isSyncDispatchLifecycleActive(
1585
+ existing.batch.targetLifecycle.lifecycle,
1586
+ target,
1587
+ ))
1588
+ ) {
1589
+ this.removePendingMaybeSyncResponseBatch(existing.batch);
1590
+ existing = undefined;
1591
+ }
1592
+ if (existing) {
1593
+ const existingTarget = existing.batch.targetLifecycle;
1594
+ if (
1595
+ existingTarget.lifecycle === properties.lifecycle ||
1596
+ (existing.active !== true &&
1597
+ existing.requestDelivered === true &&
1598
+ this.doesPendingMaybeSyncResponseScopeMatch(
1599
+ existing,
1600
+ properties.lifecycle,
1601
+ target,
1602
+ ))
1603
+ ) {
1604
+ continue;
1605
+ }
1606
+ // Another live caller already owns the authorization for this
1607
+ // exact target/hash. Send unrelated hashes now, then wait for this
1608
+ // authorization to be fulfilled or released before deciding
1609
+ // whether this caller must retry it.
1610
+ conflicts.push(existing);
1611
+ continue;
1612
+ }
1613
+ hashesToAdd.push(hash);
1614
+ }
1615
+ if (hashesToAdd.length > 0) {
1616
+ hashesToAddByTarget.set(target, hashesToAdd);
1617
+ required += hashesToAdd.length;
1618
+ }
1619
+ }
1620
+
1621
+ if (
1622
+ required >
1623
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES -
1624
+ this.pendingMaybeSyncResponseCount
1625
+ ) {
1626
+ return { kind: "capacity", required };
1627
+ }
1628
+
1629
+ const addedBatches: PendingMaybeSyncResponse[] = [];
1630
+ const addedAuthorizations: PendingMaybeSyncResponseAuthorization[] = [];
1631
+ for (const [target, hashesToAdd] of hashesToAddByTarget) {
1632
+ const targetLifecycle = properties.lifecycle.targets.get(target)!;
1633
+ const batch: PendingMaybeSyncResponse = {
1634
+ hashes: new Set(hashesToAdd),
1635
+ target,
1636
+ targetLifecycle,
1637
+ // Start the response deadline only after rpc.send succeeds. Keeping
1638
+ // pre-delivery work charged prevents a slow/non-abortable transport
1639
+ // from rolling over an unbounded number of sends.
1640
+ expiresAt: Infinity,
1641
+ heapIndex: -1,
1642
+ };
1643
+ let pendingForTarget = this.pendingMaybeSyncResponses.get(target);
1644
+ if (!pendingForTarget) {
1645
+ pendingForTarget = new Map();
1646
+ this.pendingMaybeSyncResponses.set(target, pendingForTarget);
1647
+ }
1648
+ for (const hash of hashesToAdd) {
1649
+ const authorization: PendingMaybeSyncResponseAuthorization = {
1650
+ batch,
1651
+ hash,
1652
+ waiters: new Set(),
1653
+ };
1654
+ pendingForTarget.set(hash, authorization);
1655
+ addedAuthorizations.push(authorization);
1656
+ }
1657
+ this.pendingMaybeSyncResponseCount += hashesToAdd.length;
1658
+ this.pendingMaybeSyncResponseBatches.add(batch);
1659
+ targetLifecycle.batches.add(batch);
1660
+ targetLifecycle.lifecycle.retainedWork += 1;
1661
+ addedBatches.push(batch);
1662
+ }
1663
+ this.schedulePendingMaybeSyncResponseExpiry();
1664
+ if (this.pendingMaybeSyncResponseWaiters.size > 0) {
1665
+ this.schedulePendingMaybeSyncResponseWaiter();
1666
+ }
1667
+
1668
+ let released = false;
1669
+ const release = () => {
1670
+ if (released) {
1671
+ return;
1672
+ }
1673
+ released = true;
1674
+ for (const batch of addedBatches) {
1675
+ this.removePendingMaybeSyncResponseBatch(batch);
1676
+ }
1677
+ };
1678
+ const retained = () =>
1679
+ this.isSyncDispatchLifecycleActive(properties.lifecycle) &&
1680
+ addedBatches.every(
1681
+ (batch) =>
1682
+ this.pendingMaybeSyncResponseBatches.has(batch) &&
1683
+ batch.expiresAt > Date.now() &&
1684
+ batch.hashes.size > 0,
1685
+ );
1686
+ if (!this.isSyncDispatchLifecycleActive(properties.lifecycle)) {
1687
+ release();
1688
+ return { kind: "inactive" };
1689
+ }
1690
+ return {
1691
+ kind: "reserved",
1692
+ reservation: {
1693
+ release,
1694
+ beginDelivery: () => {
1695
+ for (const authorization of addedAuthorizations) {
1696
+ if (!authorization.settled) {
1697
+ authorization.deliveryInFlight = true;
1698
+ }
1699
+ }
1700
+ },
1701
+ finishDelivery: () => {
1702
+ let releasedCount = 0;
1703
+ for (const authorization of addedAuthorizations) {
1704
+ if (authorization.deliveryInFlight !== true) {
1705
+ continue;
1706
+ }
1707
+ authorization.deliveryInFlight = false;
1708
+ if (
1709
+ authorization.settled ||
1710
+ this.pendingMaybeSyncResponses
1711
+ .get(authorization.batch.target)
1712
+ ?.get(authorization.hash) !== authorization
1713
+ ) {
1714
+ releasedCount += 1;
1715
+ }
1716
+ }
1717
+ if (releasedCount > 0) {
1718
+ this.pendingMaybeSyncResponseCount -= releasedCount;
1719
+ this.schedulePendingMaybeSyncResponseWaiter();
1720
+ }
1721
+ },
1722
+ markDelivered: () => {
1723
+ const delivered: PendingMaybeSyncResponseAuthorization[] = [];
1724
+ const deliveredBatches = new Set<PendingMaybeSyncResponse>();
1725
+ for (const authorization of addedAuthorizations) {
1726
+ if (
1727
+ authorization.settled ||
1728
+ authorization.active === true ||
1729
+ this.pendingMaybeSyncResponses
1730
+ .get(authorization.batch.target)
1731
+ ?.get(authorization.hash) !== authorization ||
1732
+ !this.pendingMaybeSyncResponseBatches.has(authorization.batch)
1733
+ ) {
1734
+ continue;
1735
+ }
1736
+ authorization.requestDelivered = true;
1737
+ delivered.push(authorization);
1738
+ deliveredBatches.add(authorization.batch);
1739
+ }
1740
+ const expiresAt = Date.now() + PENDING_MAYBE_SYNC_RESPONSE_TTL_MS;
1741
+ for (const batch of deliveredBatches) {
1742
+ if (batch.heapIndex >= 0) {
1743
+ this.removePendingMaybeSyncResponseExpiry(batch);
1744
+ }
1745
+ batch.expiresAt = expiresAt;
1746
+ this.pushPendingMaybeSyncResponseExpiry(batch);
1747
+ }
1748
+ this.schedulePendingMaybeSyncResponseExpiry();
1749
+ for (const authorization of delivered) {
1750
+ if (authorization.settled || authorization.active === true) {
1751
+ continue;
1752
+ }
1753
+ for (const waiter of [...authorization.waiters]) {
1754
+ waiter("delivered");
1755
+ }
1756
+ }
1757
+ },
1758
+ newlyAuthorizedByTarget: hashesToAddByTarget,
1759
+ retained,
1760
+ signal: properties.lifecycle.controller.signal,
1761
+ },
1762
+ conflicts,
1763
+ };
1764
+ }
1765
+
1766
+ private waitForPendingMaybeSyncResponseChange(
1767
+ lifecycle: SyncDispatchLifecycle,
1768
+ targets: string[],
1769
+ required: number,
1770
+ associations: number,
1771
+ ): Promise<void> {
1772
+ if (
1773
+ !this.isSyncDispatchLifecycleActive(lifecycle) ||
1774
+ targets.some(
1775
+ (target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
1776
+ )
1777
+ ) {
1778
+ return Promise.resolve();
1779
+ }
1780
+ const targetLifecycles = targets
1781
+ .map((target) => lifecycle.targets.get(target))
1782
+ .filter(
1783
+ (target): target is SyncDispatchTargetLifecycle => target !== undefined,
1784
+ );
1785
+ for (const targetLifecycle of targetLifecycles) {
1786
+ targetLifecycle.activeWaiters += 1;
1787
+ targetLifecycle.lifecycle.retainedWork += 1;
1788
+ }
1789
+ return new Promise<void>((resolve) => {
1790
+ let settled = false;
1791
+ let waiter!: PendingMaybeSyncResponseWaiter;
1792
+ const abortSignals = [
1793
+ lifecycle.controller.signal,
1794
+ ...targetLifecycles.map(
1795
+ (targetLifecycle) => targetLifecycle.controller.signal,
1796
+ ),
1797
+ ];
1798
+ const wake = () => {
1799
+ if (settled) {
1800
+ return;
1801
+ }
1802
+ const advanceWaiters =
1803
+ !this.isSyncDispatchLifecycleActive(lifecycle) ||
1804
+ targets.some(
1805
+ (target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
1806
+ );
1807
+ settled = true;
1808
+ this.pendingMaybeSyncResponseWaiters.delete(waiter);
1809
+ this.removePendingMaybeSyncResponseWaiter(waiter);
1810
+ this.pendingMaybeSyncResponseWaiterAssociationCount -=
1811
+ waiter.associations;
1812
+ for (const signal of abortSignals) {
1813
+ signal.removeEventListener("abort", wake);
1814
+ }
1815
+ for (const targetLifecycle of targetLifecycles) {
1816
+ targetLifecycle.activeWaiters -= 1;
1817
+ targetLifecycle.lifecycle.retainedWork -= 1;
1818
+ }
1819
+ resolve();
1820
+ this.maybeDisposeSyncDispatchLifecycle(lifecycle);
1821
+ if (advanceWaiters) {
1822
+ this.schedulePendingMaybeSyncResponseWaiter();
1823
+ }
1824
+ };
1825
+ waiter = {
1826
+ required,
1827
+ associations,
1828
+ order: ++this.pendingMaybeSyncResponseWaiterOrder,
1829
+ bypasses: 0,
1830
+ heapIndex: -1,
1831
+ fitHeapIndex: -1,
1832
+ wake,
1833
+ };
1834
+ this.pendingMaybeSyncResponseWaiters.add(waiter);
1835
+ this.pendingMaybeSyncResponseWaiterAssociationCount += associations;
1836
+ this.pushPendingMaybeSyncResponseWaiter(waiter);
1837
+ // This exact request does not fit, but an older/stale requirement may
1838
+ // have changed and another smaller waiter can still use the capacity.
1839
+ this.schedulePendingMaybeSyncResponseWaiter();
1840
+ for (const signal of abortSignals) {
1841
+ signal.addEventListener("abort", wake, { once: true });
1842
+ }
1843
+ if (
1844
+ !this.isSyncDispatchLifecycleActive(lifecycle) ||
1845
+ targets.some(
1846
+ (target) => !this.isSyncDispatchLifecycleActive(lifecycle, target),
1847
+ )
1848
+ ) {
1849
+ wake();
1850
+ }
1851
+ });
1852
+ }
1853
+
1854
+ private async reservePendingMaybeSyncResponse(properties: {
1855
+ hashes: Iterable<string>;
1856
+ targets: string[];
1857
+ lifecycle: SyncDispatchLifecycle;
1858
+ }): Promise<
1859
+ | {
1860
+ reservation: PendingMaybeSyncResponseReservation;
1861
+ conflicts: PendingMaybeSyncResponseAuthorization[];
1862
+ }
1863
+ | undefined
1864
+ > {
1865
+ const hashes = [...new Set(properties.hashes)];
1866
+ const targets = [...new Set(properties.targets)];
1867
+ const associations = Math.max(
1868
+ 1,
1869
+ Math.min(MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES, hashes.length) *
1870
+ Math.min(MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES, targets.length),
1871
+ );
1872
+ while (
1873
+ this.isSyncDispatchLifecycleActive(properties.lifecycle) &&
1874
+ targets.every((target) =>
1875
+ this.isSyncDispatchLifecycleActive(properties.lifecycle, target),
1876
+ )
1877
+ ) {
1878
+ const attempt = this.tryReservePendingMaybeSyncResponse({
1879
+ hashes,
1880
+ targets,
1881
+ lifecycle: properties.lifecycle,
1882
+ });
1883
+ if (attempt.kind === "reserved") {
1884
+ return {
1885
+ reservation: attempt.reservation,
1886
+ conflicts: attempt.conflicts,
1887
+ };
1888
+ }
1889
+ if (attempt.kind !== "capacity") {
1890
+ this.schedulePendingMaybeSyncResponseWaiter();
1891
+ return undefined;
1892
+ }
1893
+ if (
1894
+ this.pendingMaybeSyncResponseWaiters.size >=
1895
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_WAITERS ||
1896
+ this.pendingMaybeSyncResponseWaiterAssociationCount + associations >
1897
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES
1898
+ ) {
1899
+ return undefined;
1900
+ }
1901
+ await this.waitForPendingMaybeSyncResponseChange(
1902
+ properties.lifecycle,
1903
+ targets,
1904
+ attempt.required,
1905
+ associations,
1906
+ );
1907
+ }
1908
+ return undefined;
1909
+ }
1910
+
1911
+ expectMaybeSyncResponse(properties: {
1912
+ hashes: Iterable<string>;
1913
+ targets: string[];
1914
+ signal?: AbortSignal;
1915
+ }): PendingMaybeSyncResponseReservation | undefined {
1916
+ const targets = [...new Set(properties.targets)];
1917
+ const lifecycle = this.captureSyncDispatchLifecycle(
1918
+ targets,
1919
+ properties.signal,
1920
+ { abortAllOnTargetDisconnect: true },
1921
+ );
1922
+ const attempt = this.tryReservePendingMaybeSyncResponse({
1923
+ hashes: properties.hashes,
1924
+ targets,
1925
+ lifecycle,
1926
+ });
1927
+ const reservation =
1928
+ attempt.kind === "reserved" ? attempt.reservation : undefined;
1929
+ let retainedReservation: PendingMaybeSyncResponseReservation | undefined;
1930
+ if (reservation) {
1931
+ // This synchronous helper represents an already-issued request.
1932
+ reservation.markDelivered();
1933
+ const leasedTargets = [...lifecycle.targets.values()];
1934
+ for (const targetLifecycle of leasedTargets) {
1935
+ targetLifecycle.responseLeases += 1;
1936
+ targetLifecycle.lifecycle.retainedWork += 1;
1937
+ }
1938
+ let released = false;
1939
+ retainedReservation = {
1940
+ ...reservation,
1941
+ release: () => {
1942
+ if (released) {
1943
+ return;
1944
+ }
1945
+ released = true;
1946
+ reservation.release();
1947
+ for (const targetLifecycle of leasedTargets) {
1948
+ targetLifecycle.responseLeases -= 1;
1949
+ targetLifecycle.lifecycle.retainedWork -= 1;
1950
+ }
1951
+ this.maybeDisposeSyncDispatchLifecycle(lifecycle);
1952
+ },
1953
+ };
1954
+ }
1955
+ this.finishSyncDispatchLifecycle(lifecycle);
1956
+ return retainedReservation;
1957
+ }
1958
+
1959
+ consumeAuthorizedMaybeSyncResponse(
1960
+ hashes: Iterable<string>,
1961
+ from: PublicSignKey,
1962
+ ): AuthorizedMaybeSyncResponseLease[] {
1963
+ const fromHash = from.hashcode();
1964
+ const pendingForTarget = this.pendingMaybeSyncResponses.get(fromHash);
1965
+ if (!pendingForTarget) {
1966
+ return [];
1967
+ }
1968
+ if (
1969
+ this.activeMaybeSyncResponseCount >=
1970
+ MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_GLOBAL ||
1971
+ (this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 0) >=
1972
+ MAX_ACTIVE_SIMPLE_SYNC_RESPONSES_PER_PEER
1973
+ ) {
1974
+ return [];
1975
+ }
1976
+ const acceptedByLifecycle = new Map<
1977
+ SyncDispatchTargetLifecycle,
1978
+ {
1979
+ hashes: string[];
1980
+ authorizations: PendingMaybeSyncResponseAuthorization[];
1981
+ }
1982
+ >();
1983
+ const seen = new Set<string>();
1984
+ let inspected = 0;
1985
+ const iterator = hashes[Symbol.iterator]();
1986
+ let exhausted = false;
1987
+ try {
1988
+ while (
1989
+ inspected < MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES &&
1990
+ pendingForTarget.size > 0
1991
+ ) {
1992
+ const next = iterator.next();
1993
+ if (next.done) {
1994
+ exhausted = true;
1995
+ break;
1996
+ }
1997
+ inspected += 1;
1998
+ const hash = next.value;
1999
+ if (seen.has(hash)) {
2000
+ continue;
2001
+ }
2002
+ seen.add(hash);
2003
+ const authorization = pendingForTarget.get(hash);
2004
+ if (!authorization || authorization.active === true) {
2005
+ continue;
2006
+ }
2007
+ const batch = authorization.batch;
2008
+ const targetLifecycle = batch.targetLifecycle;
2009
+ if (
2010
+ batch.expiresAt <= Date.now() ||
2011
+ !this.isSyncDispatchLifecycleActive(
2012
+ targetLifecycle.lifecycle,
2013
+ fromHash,
2014
+ )
2015
+ ) {
2016
+ this.removePendingMaybeSyncResponseBatch(batch);
2017
+ continue;
2018
+ }
2019
+ if (pendingForTarget.get(hash)?.batch !== batch) {
2020
+ continue;
2021
+ }
2022
+ let accepted = acceptedByLifecycle.get(targetLifecycle);
2023
+ if (!accepted) {
2024
+ accepted = { hashes: [], authorizations: [] };
2025
+ acceptedByLifecycle.set(targetLifecycle, accepted);
2026
+ targetLifecycle.responseLeases += 1;
2027
+ targetLifecycle.lifecycle.retainedWork += 1;
2028
+ }
2029
+ authorization.active = true;
2030
+ batch.hashes.delete(hash);
2031
+ accepted.hashes.push(hash);
2032
+ accepted.authorizations.push(authorization);
2033
+ if (batch.hashes.size === 0) {
2034
+ this.removePendingMaybeSyncResponseBatch(batch);
2035
+ }
2036
+ }
2037
+ } finally {
2038
+ if (!exhausted) {
2039
+ iterator.return?.();
2040
+ }
2041
+ }
2042
+ if (pendingForTarget.size === 0) {
2043
+ this.pendingMaybeSyncResponses.delete(fromHash);
2044
+ }
2045
+ if (acceptedByLifecycle.size === 0) {
2046
+ return [];
2047
+ }
2048
+ this.activeMaybeSyncResponseCount += 1;
2049
+ this.activeMaybeSyncResponseCountByPeer.set(
2050
+ fromHash,
2051
+ (this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 0) + 1,
2052
+ );
2053
+ let remainingLeases = acceptedByLifecycle.size;
2054
+ return [...acceptedByLifecycle].map(
2055
+ ([targetLifecycle, { hashes: acceptedHashes, authorizations }]) => {
2056
+ let released = false;
2057
+ return {
2058
+ hashes: acceptedHashes,
2059
+ signal: targetLifecycle.controller.signal,
2060
+ release: (options?: { fulfilled?: boolean }) => {
2061
+ if (released) {
2062
+ return;
2063
+ }
2064
+ released = true;
2065
+ const pendingForTarget =
2066
+ this.pendingMaybeSyncResponses.get(fromHash);
2067
+ for (const authorization of authorizations) {
2068
+ this.settlePendingMaybeSyncResponseAuthorization(
2069
+ authorization,
2070
+ options?.fulfilled === true,
2071
+ );
2072
+ if (pendingForTarget?.get(authorization.hash) === authorization) {
2073
+ pendingForTarget.delete(authorization.hash);
2074
+ }
2075
+ }
2076
+ if (pendingForTarget?.size === 0) {
2077
+ this.pendingMaybeSyncResponses.delete(fromHash);
2078
+ }
2079
+ this.pendingMaybeSyncResponseCount -= authorizations.filter(
2080
+ (authorization) => authorization.deliveryInFlight !== true,
2081
+ ).length;
2082
+ targetLifecycle.responseLeases -= 1;
2083
+ targetLifecycle.lifecycle.retainedWork -= 1;
2084
+ remainingLeases -= 1;
2085
+ if (remainingLeases === 0) {
2086
+ this.activeMaybeSyncResponseCount -= 1;
2087
+ const activeForPeer =
2088
+ (this.activeMaybeSyncResponseCountByPeer.get(fromHash) ?? 1) -
2089
+ 1;
2090
+ if (activeForPeer === 0) {
2091
+ this.activeMaybeSyncResponseCountByPeer.delete(fromHash);
2092
+ } else {
2093
+ this.activeMaybeSyncResponseCountByPeer.set(
2094
+ fromHash,
2095
+ activeForPeer,
2096
+ );
2097
+ }
2098
+ }
2099
+ this.schedulePendingMaybeSyncResponseWaiter();
2100
+ this.maybeDisposeSyncDispatchLifecycle(targetLifecycle.lifecycle);
2101
+ },
2102
+ };
2103
+ },
2104
+ );
2105
+ }
2106
+
2107
+ private isRepairSessionComplete(session: RepairSessionState): boolean {
2108
+ for (const state of session.targets.values()) {
2109
+ if (state.unresolved.size > 0) {
2110
+ return false;
2111
+ }
2112
+ }
2113
+ return true;
2114
+ }
2115
+
2116
+ private buildRepairSessionResult(
2117
+ session: RepairSessionState,
2118
+ completed: boolean,
2119
+ ): RepairSessionResult[] {
2120
+ const durationMs = Date.now() - session.startedAt;
2121
+ const out: RepairSessionResult[] = [];
2122
+ for (const [target, state] of session.targets) {
2123
+ const unresolved = [...state.unresolved];
2124
+ out.push({
2125
+ target,
2126
+ requested: state.requestedCount,
2127
+ resolved: state.requestedCount - unresolved.length,
2128
+ unresolved,
2129
+ attempts: state.attempts,
2130
+ durationMs,
2131
+ completed,
2132
+ requestedTotal: state.requestedTotalCount,
2133
+ truncated: session.truncated,
2134
+ });
2135
+ }
2136
+ return out;
2137
+ }
2138
+
2139
+ private finalizeRepairSession(sessionId: string, completed: boolean): void {
2140
+ const session = this.repairSessions.get(sessionId);
2141
+ if (!session) {
2142
+ return;
2143
+ }
2144
+ this.repairSessions.delete(sessionId);
2145
+ session.cancelled = true;
2146
+ if (session.timer) {
2147
+ clearTimeout(session.timer);
2148
+ }
2149
+ session.deferred.resolve(this.buildRepairSessionResult(session, completed));
2150
+ }
2151
+
2152
+ private async refreshRepairSessionState(sessionId: string): Promise<void> {
2153
+ const session = this.repairSessions.get(sessionId);
2154
+ if (!session) {
2155
+ return;
2156
+ }
2157
+ for (const state of session.targets.values()) {
2158
+ const resolved =
2159
+ typeof this.log.hasMany === "function"
2160
+ ? await this.log.hasMany(state.unresolved)
2161
+ : await this.getExistingRepairHashes(state.unresolved);
2162
+ for (const hash of resolved) {
2163
+ state.unresolved.delete(hash);
2164
+ }
2165
+ }
2166
+ }
2167
+
2168
+ private async getExistingRepairHashes(
2169
+ hashes: Iterable<string>,
2170
+ ): Promise<Set<string>> {
2171
+ const resolved = new Set<string>();
2172
+ for (const hash of hashes) {
2173
+ if (await this.log.has(hash)) {
2174
+ resolved.add(hash);
2175
+ }
2176
+ }
2177
+ return resolved;
2178
+ }
2179
+
2180
+ private markRepairSessionResolvedHashes(hashes: string[]): void {
2181
+ if (hashes.length === 0 || this.repairSessions.size === 0) {
2182
+ return;
2183
+ }
2184
+ for (const [sessionId, session] of this.repairSessions) {
2185
+ for (const state of session.targets.values()) {
2186
+ for (const hash of hashes) {
2187
+ state.unresolved.delete(hash);
2188
+ }
2189
+ }
2190
+ if (this.isRepairSessionComplete(session)) {
2191
+ this.finalizeRepairSession(sessionId, true);
2192
+ }
2193
+ }
2194
+ }
2195
+
2196
+ private markRepairSessionResolvedHash(hash: string): void {
2197
+ if (this.repairSessions.size === 0) {
2198
+ return;
2199
+ }
2200
+ for (const [sessionId, session] of this.repairSessions) {
2201
+ for (const state of session.targets.values()) {
2202
+ state.unresolved.delete(hash);
2203
+ }
2204
+ if (this.isRepairSessionComplete(session)) {
2205
+ this.finalizeRepairSession(sessionId, true);
2206
+ }
2207
+ }
2208
+ }
2209
+
2210
+ private async runRepairSession(sessionId: string): Promise<void> {
2211
+ const session = this.repairSessions.get(sessionId);
2212
+ if (!session) {
2213
+ return;
2214
+ }
2215
+
2216
+ let previousDelay = 0;
2217
+ for (const delayMs of session.retryIntervalsMs) {
2218
+ if (!this.repairSessions.has(sessionId) || this.closed) {
2219
+ return;
2220
+ }
2221
+
2222
+ const waitMs = Math.max(0, delayMs - previousDelay);
2223
+ previousDelay = delayMs;
2224
+ if (waitMs > 0) {
2225
+ await new Promise<void>((resolve) => {
2226
+ const timer = setTimeout(resolve, waitMs);
2227
+ timer.unref?.();
2228
+ });
2229
+ }
2230
+ if (!this.repairSessions.has(sessionId) || this.closed) {
2231
+ return;
2232
+ }
2233
+
2234
+ await this.refreshRepairSessionState(sessionId);
2235
+ const current = this.repairSessions.get(sessionId);
2236
+ if (!current) {
2237
+ return;
2238
+ }
2239
+ if (this.isRepairSessionComplete(current)) {
2240
+ this.finalizeRepairSession(sessionId, true);
2241
+ return;
2242
+ }
2243
+
2244
+ for (const [target, state] of current.targets) {
2245
+ if (state.unresolved.size === 0) {
2246
+ continue;
2247
+ }
2248
+ state.attempts += 1;
2249
+ try {
2250
+ await this.requestSync([...state.unresolved], [target], {
2251
+ targetEpochs: new Map([[target, state.targetEpoch]]),
2252
+ createTargetEpochs: false,
2253
+ });
2254
+ } catch {
2255
+ // Best-effort: keep unresolved and let retries/timeout determine outcome.
2256
+ }
2257
+ }
2258
+
2259
+ await this.refreshRepairSessionState(sessionId);
2260
+ const afterSend = this.repairSessions.get(sessionId);
2261
+ if (!afterSend) {
2262
+ return;
2263
+ }
2264
+ if (this.isRepairSessionComplete(afterSend)) {
2265
+ this.finalizeRepairSession(sessionId, true);
2266
+ return;
2267
+ }
2268
+
2269
+ if (afterSend.mode === "best-effort") {
2270
+ this.finalizeRepairSession(sessionId, false);
2271
+ return;
2272
+ }
2273
+ }
2274
+
2275
+ for (;;) {
2276
+ if (!this.repairSessions.has(sessionId) || this.closed) {
2277
+ return;
2278
+ }
2279
+ await this.refreshRepairSessionState(sessionId);
2280
+ const current = this.repairSessions.get(sessionId);
2281
+ if (!current) {
2282
+ return;
2283
+ }
2284
+ if (this.isRepairSessionComplete(current)) {
2285
+ this.finalizeRepairSession(sessionId, true);
2286
+ return;
2287
+ }
2288
+ await new Promise<void>((resolve) => {
2289
+ const timer = setTimeout(resolve, SESSION_POLL_INTERVAL_MS);
2290
+ timer.unref?.();
2291
+ });
2292
+ }
2293
+ }
2294
+
2295
+ private getQueuedSyncKey(key: SyncableKey): SyncableKey | undefined {
2296
+ if (this.syncInFlightQueue.has(key)) {
2297
+ return key;
2298
+ }
2299
+ if (typeof key === "string") {
2300
+ for (const queuedKey of this.syncInFlightQueue.keys()) {
2301
+ if (
2302
+ typeof queuedKey === "bigint" &&
2303
+ this.coordinateToHash.get(queuedKey) === key
2304
+ ) {
2305
+ return queuedKey;
2306
+ }
2307
+ }
2308
+ return undefined;
2309
+ }
2310
+ const hash = this.coordinateToHash.get(key);
2311
+ return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
2312
+ }
2313
+
2314
+ startRepairSession(properties: {
2315
+ entries: Map<string, SyncEntryCoordinates<R>>;
2316
+ targets: string[];
2317
+ mode?: RepairSessionMode;
2318
+ timeoutMs?: number;
2319
+ retryIntervalsMs?: number[];
2320
+ }): RepairSession {
2321
+ const mode = properties.mode ?? "best-effort";
2322
+ const startedAt = Date.now();
2323
+ const timeoutMs = Math.max(
2324
+ 1,
2325
+ Math.floor(
2326
+ properties.timeoutMs ??
2327
+ (mode === "convergent"
2328
+ ? DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS
2329
+ : DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS),
2330
+ ),
2331
+ );
2332
+ const retryIntervalsMs = this.normalizeRetryIntervals(
2333
+ mode,
2334
+ properties.retryIntervalsMs,
2335
+ );
2336
+ const allHashes = this.getPrioritizedHashes(properties.entries);
2337
+ const trackedHashes =
2338
+ mode === "convergent" &&
2339
+ allHashes.length > this.maxConvergentTrackedHashes
2340
+ ? allHashes.slice(0, this.maxConvergentTrackedHashes)
2341
+ : allHashes;
2342
+ const truncated = trackedHashes.length < allHashes.length;
2343
+ const targets = [...new Set(properties.targets)];
2344
+ const id = `repair-${++this.repairSessionCounter}`;
2345
+ const deferred = createDeferred<RepairSessionResult[]>();
2346
+
2347
+ const targetStates = new Map<string, RepairSessionTargetState>();
2348
+ for (const target of targets) {
2349
+ targetStates.set(target, {
2350
+ unresolved: new Set(trackedHashes),
2351
+ requestedCount: trackedHashes.length,
2352
+ requestedTotalCount: allHashes.length,
2353
+ attempts: 0,
2354
+ targetEpoch: this.getOrCreateSyncDispatchTargetEpoch(target),
2355
+ });
2356
+ }
2357
+
2358
+ const session: RepairSessionState = {
2359
+ id,
2360
+ mode,
2361
+ startedAt,
2362
+ timeoutMs,
2363
+ retryIntervalsMs,
2364
+ targets: targetStates,
2365
+ truncated,
2366
+ deferred,
2367
+ cancelled: false,
2368
+ };
2369
+
2370
+ if (allHashes.length === 0 || targets.length === 0) {
2371
+ deferred.resolve(this.buildRepairSessionResult(session, true));
2372
+ return {
2373
+ id,
2374
+ done: deferred.promise,
2375
+ cancel: () => {
2376
+ // no-op
2377
+ },
2378
+ };
2379
+ }
2380
+
2381
+ // For capped convergent sessions, still dispatch the full set once so large
2382
+ // repairs are not limited to tracked hashes.
2383
+ if (mode === "convergent" && truncated) {
2384
+ void this.onMaybeMissingEntries({
2385
+ entries: properties.entries,
2386
+ targets,
2387
+ }).catch(() => {
2388
+ // Best-effort: retries on tracked hashes continue via runRepairSession.
2389
+ });
2390
+ }
2391
+
2392
+ session.timer = setTimeout(() => {
2393
+ this.finalizeRepairSession(id, false);
2394
+ }, timeoutMs);
2395
+ session.timer.unref?.();
2396
+
2397
+ this.repairSessions.set(id, session);
2398
+ void this.runRepairSession(id).catch(() => {
2399
+ this.finalizeRepairSession(id, false);
2400
+ });
2401
+
2402
+ return {
2403
+ id,
2404
+ done: deferred.promise,
2405
+ cancel: () => {
2406
+ this.finalizeRepairSession(id, false);
2407
+ },
2408
+ };
2409
+ }
2410
+
2411
+ async onMaybeMissingEntries(properties: {
2412
+ entries: Map<string, SyncEntryCoordinates<R>>;
2413
+ targets: string[];
2414
+ signal?: AbortSignal;
2415
+ }): Promise<void> {
2416
+ if (properties.signal?.aborted) {
2417
+ return;
2418
+ }
2419
+ await this.onMaybeMissingHashes({
2420
+ hashes: this.getPrioritizedHashes(properties.entries),
2421
+ targets: properties.targets,
2422
+ signal: properties.signal,
2423
+ });
2424
+ }
2425
+
2426
+ async onMaybeMissingHashes(properties: {
2427
+ hashes: Iterable<string>;
2428
+ targets: string[];
2429
+ signal?: AbortSignal;
2430
+ }): Promise<void> {
2431
+ const targets = [...new Set(properties.targets)];
2432
+ const lifecycle = this.captureSyncDispatchLifecycle(
2433
+ targets,
2434
+ properties.signal,
2435
+ );
2436
+ if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
2437
+ this.finishSyncDispatchLifecycle(lifecycle);
2438
+ return;
2439
+ }
2440
+ const profile = this.syncOptions?.profile;
2441
+ const startedAt = syncProfileStart(profile);
2442
+ const hashes = [...new Set(properties.hashes)];
2443
+ const targetsPerAuthorizationWindow = Math.max(
2444
+ 1,
2445
+ Math.min(targets.length, MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES),
2446
+ );
2447
+ const chunks = this.chunk(
2448
+ hashes,
2449
+ Math.max(
2450
+ 1,
2451
+ Math.min(
2452
+ this.maxHashesPerMessage,
2453
+ Math.floor(
2454
+ MAX_PENDING_MAYBE_SYNC_RESPONSE_HASHES /
2455
+ targetsPerAuthorizationWindow,
2456
+ ),
2457
+ ),
2458
+ ),
2459
+ );
2460
+ let messages = 0;
2461
+ try {
2462
+ for (const chunk of chunks) {
2463
+ if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
2464
+ break;
2465
+ }
2466
+ for (const target of targets) {
2467
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
2468
+ continue;
2469
+ }
2470
+ let hashesToAuthorize = chunk;
2471
+ while (
2472
+ hashesToAuthorize.length > 0 &&
2473
+ this.isSyncDispatchLifecycleActive(lifecycle, target)
2474
+ ) {
2475
+ const reserved = await this.reservePendingMaybeSyncResponse({
2476
+ hashes: hashesToAuthorize,
2477
+ targets: [target],
2478
+ lifecycle,
2479
+ });
2480
+ if (
2481
+ !reserved ||
2482
+ !this.isSyncDispatchLifecycleActive(lifecycle, target)
2483
+ ) {
2484
+ break;
2485
+ }
2486
+ const { reservation, conflicts } = reserved;
2487
+ const hashesToSend =
2488
+ reservation.newlyAuthorizedByTarget.get(target) ?? [];
2489
+ if (!reservation.retained()) {
2490
+ reservation.release();
2491
+ break;
2492
+ }
2493
+ if (hashesToSend.length > 0) {
2494
+ reservation.beginDelivery();
2495
+ try {
2496
+ await this.rpc.send(
2497
+ new RequestMaybeSync({ hashes: hashesToSend }),
2498
+ {
2499
+ priority: SYNC_MESSAGE_PRIORITY,
2500
+ mode: new SilentDelivery({
2501
+ to: [target],
2502
+ redundancy: 1,
2503
+ }),
2504
+ signal: this.getSyncDispatchSignal(lifecycle, target),
2505
+ },
2506
+ );
2507
+ reservation.markDelivered();
2508
+ messages += 1;
2509
+ } catch (error) {
2510
+ reservation.release();
2511
+ if (
2512
+ !this.isSyncDispatchLifecycleActive(lifecycle) ||
2513
+ !this.isSyncDispatchLifecycleActive(lifecycle, target)
2514
+ ) {
2515
+ break;
2516
+ }
2517
+ throw error;
2518
+ } finally {
2519
+ reservation.finishDelivery();
2520
+ }
2521
+ }
2522
+ hashesToAuthorize =
2523
+ await this.waitForPendingMaybeSyncResponseConflicts(
2524
+ conflicts,
2525
+ lifecycle,
2526
+ target,
2527
+ hashesToAuthorize.length,
2528
+ );
2529
+ }
2530
+ }
2531
+ if (!this.isSyncDispatchLifecycleActive(lifecycle)) {
2532
+ break;
2533
+ }
2534
+ }
2535
+ } finally {
2536
+ this.finishSyncDispatchLifecycle(lifecycle);
2537
+ if (profile) {
2538
+ emitSyncProfileDuration(profile, startedAt, {
2539
+ name: "simple.onMaybeMissingEntries",
2540
+ entries: hashes.length,
2541
+ messages,
2542
+ targets: targets.length,
2543
+ details: !this.isSyncDispatchLifecycleActive(lifecycle)
2544
+ ? { cancelled: true }
2545
+ : undefined,
2546
+ });
2547
+ }
2548
+ }
2549
+ }
2550
+
2551
+ /**
2552
+ * Ship exchange heads to one peer: the fused native raw path when the
2553
+ * peer advertised raw capability and the shared log provided a fused
2554
+ * sender, otherwise the TS message path (raw or plain by capability).
2555
+ * Returns the number of messages sent and whether the fused path ran.
2556
+ */
2557
+ private async shipExchangeHeads(
2558
+ hashes: string[],
2559
+ to: PublicSignKey,
2560
+ canReceiveRaw: boolean,
2561
+ signal?: AbortSignal,
2562
+ ): Promise<{ messages: number; fused: boolean }> {
2563
+ if (signal?.aborted) {
2564
+ return { messages: 0, fused: false };
2565
+ }
2566
+ if (canReceiveRaw && this.sendRawExchangeHeads) {
2567
+ let sentMessages: number | undefined;
2568
+ try {
2569
+ sentMessages = await this.sendRawExchangeHeads(
2570
+ hashes,
2571
+ [to.hashcode()],
2572
+ { signal },
2573
+ );
2574
+ } catch (error) {
2575
+ if (signal?.aborted) {
2576
+ return { messages: 0, fused: true };
2577
+ }
2578
+ throw error;
2579
+ }
2580
+ if (sentMessages !== undefined) {
2581
+ return { messages: sentMessages, fused: true };
2582
+ }
2583
+ }
2584
+ let messages = 0;
2585
+ const messageGenerator = canReceiveRaw
2586
+ ? createRawExchangeHeadsMessages(
2587
+ this.log,
2588
+ hashes,
2589
+ this.syncOptions?.profile,
2590
+ )
2591
+ : createExchangeHeadsMessages(this.log, hashes);
2592
+ for await (const message of messageGenerator) {
2593
+ if (signal?.aborted) {
2594
+ break;
2595
+ }
2596
+ try {
2597
+ await this.rpc.send(message, {
2598
+ mode: new SilentDelivery({ to: [to], redundancy: 1 }),
2599
+ signal,
2600
+ });
2601
+ messages += 1;
2602
+ } catch (error) {
2603
+ if (signal?.aborted) {
2604
+ break;
2605
+ }
2606
+ throw error;
2607
+ }
2608
+ }
2609
+ return { messages, fused: false };
2610
+ }
2611
+
2612
+ /**
2613
+ * Ships hashes that the caller has already authorized for this exact sender.
2614
+ *
2615
+ * This deliberately does not consult or extend Simple's bounded pending-response
2616
+ * window. Rateless sync owns a separate, target-scoped authorization lifecycle
2617
+ * and uses this helper only after intersecting the response with that process's
2618
+ * exact advertised hash set.
2619
+ */
2620
+ async shipAuthorizedMaybeSyncResponse(properties: {
2621
+ hashes: string[];
2622
+ from: PublicSignKey;
2623
+ response: ResponseMaybeSync | ResponseMaybeSyncCapabilities;
2624
+ signal: AbortSignal;
2625
+ }): Promise<{ messages: number; fused: boolean; entries: number }> {
2626
+ const hashes = this.filterRecentlySentExchangeHeads(
2627
+ properties.hashes,
2628
+ properties.from,
2629
+ );
2630
+ try {
2631
+ const shipped = await this.shipExchangeHeads(
2632
+ hashes,
2633
+ properties.from,
2634
+ canReceiveRawExchangeHeads(properties.response),
2635
+ properties.signal,
2636
+ );
2637
+ return {
2638
+ ...shipped,
2639
+ entries: hashes.length,
2640
+ };
2641
+ } catch (error) {
2642
+ this.forgetRecentlySentExchangeHeads(hashes, properties.from);
2643
+ throw error;
2644
+ } finally {
2645
+ if (properties.signal.aborted) {
2646
+ this.forgetRecentlySentExchangeHeads(hashes, properties.from);
2647
+ }
2648
+ }
2649
+ }
2650
+
2651
+ async shipAuthorizedMaybeSyncResponseLeases(properties: {
2652
+ leases: AuthorizedMaybeSyncResponseLease[];
2653
+ from: PublicSignKey;
2654
+ response: ResponseMaybeSync | ResponseMaybeSyncCapabilities;
2655
+ source?: string;
2656
+ }): Promise<{ messages: number; fused: boolean; entries: number }> {
2657
+ if (properties.leases.length === 0) {
2658
+ return { messages: 0, fused: false, entries: 0 };
2659
+ }
2660
+ const profile = this.syncOptions?.profile;
2661
+ const startedAt = syncProfileStart(profile);
2662
+ let messages = 0;
2663
+ let fused = false;
2664
+ let entries = 0;
2665
+ let firstError: unknown;
2666
+ for (const lease of properties.leases) {
2667
+ let fulfilled = false;
2668
+ try {
2669
+ const shipped = await this.shipAuthorizedMaybeSyncResponse({
2670
+ hashes: lease.hashes,
2671
+ from: properties.from,
2672
+ response: properties.response,
2673
+ signal: lease.signal,
2674
+ });
2675
+ messages += shipped.messages;
2676
+ fused ||= shipped.fused;
2677
+ entries += shipped.entries;
2678
+ fulfilled = !lease.signal.aborted;
2679
+ } catch (error) {
2680
+ firstError ??= error;
2681
+ } finally {
2682
+ lease.release({ fulfilled });
2683
+ }
2684
+ }
2685
+ if (profile) {
2686
+ emitSyncProfileDuration(profile, startedAt, {
2687
+ name: "simple.exchangeHeads",
2688
+ entries,
2689
+ messages,
2690
+ targets: 1,
2691
+ details: {
2692
+ source: properties.source ?? "responseMaybeSync",
2693
+ fused,
2694
+ },
2695
+ });
2696
+ }
2697
+ if (firstError !== undefined) {
2698
+ throw firstError;
2699
+ }
2700
+ return { messages, fused, entries };
2701
+ }
2702
+
2703
+ async onMessage(
2704
+ msg: TransportMessage,
2705
+ context: RequestContext,
2706
+ ): Promise<boolean> {
2707
+ const from = context.from!;
2708
+ if (msg instanceof RequestMaybeSync) {
2709
+ await this.queueSync(msg.hashes, from);
2710
+ return true;
2711
+ } else if (
2712
+ msg instanceof ResponseMaybeSync ||
2713
+ msg instanceof ResponseMaybeSyncCapabilities
2714
+ ) {
2715
+ // TODO perhaps send less messages to more receivers for performance reasons?
2716
+ // TODO wait for previous send to target before trying to send more?
2717
+
2718
+ const pending = this.consumeAuthorizedMaybeSyncResponse(msg.hashes, from);
2719
+ if (pending.length === 0) {
2720
+ return true;
2721
+ }
2722
+ await this.shipAuthorizedMaybeSyncResponseLeases({
2723
+ leases: pending,
2724
+ from,
2725
+ response: msg,
2726
+ });
2727
+ return true;
2728
+ } else if (
2729
+ msg instanceof RequestMaybeSyncCoordinate ||
2730
+ msg instanceof RequestMaybeSyncCoordinateCapabilities
2731
+ ) {
2732
+ if (
2733
+ msg.hashNumbers.length === 0 ||
2734
+ msg.hashNumbers.length > MAX_SIMPLE_COORDINATE_REQUEST_SYMBOLS
2735
+ ) {
2736
+ return true;
2737
+ }
2738
+ const target = from.hashcode();
2739
+ const releaseLookup = this.tryAcquireCoordinateLookup(target);
2740
+ if (!releaseLookup) {
2741
+ return true;
2742
+ }
2743
+ const lifecycle = this.captureSyncDispatchLifecycle([target]);
2744
+ let lookupReleased = false;
2745
+ const finishLookup = () => {
2746
+ if (lookupReleased) {
2747
+ return;
2748
+ }
2749
+ lookupReleased = true;
2750
+ releaseLookup();
2751
+ };
2752
+ try {
2753
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
2754
+ return true;
2755
+ }
2756
+ const symbols = [...new Set(msg.hashNumbers)];
2757
+ const profile = this.syncOptions?.profile;
2758
+ const lookupStartedAt = syncProfileStart(profile);
2759
+ const hashes = await getHashesFromSymbols(
2760
+ symbols,
2761
+ this.entryIndex,
2762
+ this.coordinateToHash,
2763
+ this.resolveHashesForSymbols,
2764
+ this.resolveHashListForSymbols,
2765
+ MAX_SIMPLE_COORDINATE_RESPONSE_HASHES,
2766
+ );
2767
+ finishLookup();
2768
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
2769
+ return true;
2770
+ }
2771
+ if (profile) {
2772
+ emitSyncProfileDuration(profile, lookupStartedAt, {
2773
+ name: "simple.coordinateLookup",
2774
+ entries: hashLookupResultSize(hashes),
2775
+ symbols: symbols.length,
2776
+ });
2777
+ }
2778
+
2779
+ const releaseResponse = this.tryAcquireCoordinateResponse(target);
2780
+ if (!releaseResponse) {
2781
+ return true;
2782
+ }
2783
+ const exchangeStartedAt = syncProfileStart(profile);
2784
+ let hashesToSend: string[] = [];
2785
+ let messages = 0;
2786
+ let fused = false;
2787
+ try {
2788
+ hashesToSend = this.filterRecentlySentExchangeHeads(hashes, from);
2789
+ // dont set priority 1 here because this will block other messages that should higher priority
2790
+ ({ messages, fused } = await this.shipExchangeHeads(
2791
+ hashesToSend,
2792
+ context.from!,
2793
+ canReceiveRawExchangeHeads(msg),
2794
+ this.getSyncDispatchSignal(lifecycle, target),
2795
+ ));
2796
+ } finally {
2797
+ releaseResponse();
2798
+ if (profile) {
2799
+ emitSyncProfileDuration(profile, exchangeStartedAt, {
2800
+ name: "simple.exchangeHeads",
2801
+ entries: hashesToSend.length,
2802
+ messages,
2803
+ targets: 1,
2804
+ details: {
2805
+ source: "requestMaybeSyncCoordinate",
2806
+ fused,
2807
+ },
2808
+ });
2809
+ }
2810
+ }
2811
+
2812
+ return true;
2813
+ } finally {
2814
+ finishLookup();
2815
+ this.finishSyncDispatchLifecycle(lifecycle);
2816
+ }
2817
+ } else {
2818
+ return false; // no message was consumed
2819
+ }
2820
+ }
2821
+
2822
+ onReceivedEntries(properties: {
2823
+ entries: EntryWithRefs<any>[];
2824
+ from: PublicSignKey;
2825
+ }): Promise<void> | void {
2826
+ return this.onReceivedEntryHashes({
2827
+ hashes: properties.entries.map((entry) => entry.entry.hash),
2828
+ from: properties.from,
2829
+ });
2830
+ }
2831
+
2832
+ onReceivedEntryHashes(properties: {
2833
+ hashes: string[];
2834
+ from: PublicSignKey;
2835
+ }): Promise<void> | void {
2836
+ this.clearSyncInFlightForPeerHashes(
2837
+ properties.from.hashcode(),
2838
+ properties.hashes,
2839
+ );
2840
+ this.markRepairSessionResolvedHashes(properties.hashes);
2841
+ }
2842
+
2843
+ private getPendingSyncKeyIdentity(key: SyncableKey): SyncableKey {
2844
+ if (typeof key === "string") {
2845
+ return key;
2846
+ }
2847
+ const hash = this.coordinateToHash.get(key);
2848
+ return hash ?? key;
2849
+ }
2850
+
2851
+ private removeQueuedSyncCoordinateAlias(key: bigint): void {
2852
+ this.syncInFlightQueuedCoordinates.delete(key);
2853
+ if (this.syncInFlightQueuedCoordinates.size === 0) {
2854
+ this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
2855
+ }
2856
+ const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
2857
+ this.syncInFlightQueuedHashByCoordinate.delete(key);
2858
+ if (previousHash != null) {
2859
+ const coordinates =
2860
+ this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
2861
+ coordinates?.delete(key);
2862
+ if (coordinates?.size === 0) {
2863
+ this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
2864
+ }
2865
+ }
2866
+ }
2867
+
2868
+ private refreshQueuedSyncCoordinateAlias(key: bigint): void {
2869
+ if (!this.syncInFlightQueue.has(key)) {
2870
+ this.removeQueuedSyncCoordinateAlias(key);
2871
+ return;
2872
+ }
2873
+ this.syncInFlightQueuedCoordinates.add(key);
2874
+ const hash = this.coordinateToHash.get(key) ?? undefined;
2875
+ const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
2876
+ if (previousHash !== hash) {
2877
+ if (previousHash != null) {
2878
+ const coordinates =
2879
+ this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
2880
+ coordinates?.delete(key);
2881
+ if (coordinates?.size === 0) {
2882
+ this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
2883
+ }
2884
+ }
2885
+ if (hash == null) {
2886
+ this.syncInFlightQueuedHashByCoordinate.delete(key);
2887
+ } else {
2888
+ this.syncInFlightQueuedHashByCoordinate.set(key, hash);
2889
+ }
2890
+ }
2891
+ if (hash != null) {
2892
+ let coordinates = this.syncInFlightQueuedCoordinatesByHash.get(hash);
2893
+ if (!coordinates) {
2894
+ coordinates = new Set();
2895
+ this.syncInFlightQueuedCoordinatesByHash.set(hash, coordinates);
2896
+ }
2897
+ coordinates.add(key);
2898
+ this.reconcileQueuedSyncCoordinateAlias(key, hash);
2899
+ }
2900
+ }
2901
+
2902
+ private reconcileQueuedSyncCoordinateAlias(
2903
+ coordinate: bigint,
2904
+ hash: string,
2905
+ ): void {
2906
+ const now = Date.now();
2907
+ const coordinateExpiresAt = this.syncInFlightQueueExpiresAt.get(coordinate);
2908
+ if (coordinateExpiresAt != null && coordinateExpiresAt <= now) {
2909
+ this.clearSyncProcessKey(coordinate);
2910
+ return;
2911
+ }
2912
+ const hashExpiresAt = this.syncInFlightQueueExpiresAt.get(hash);
2913
+ if (hashExpiresAt != null && hashExpiresAt <= now) {
2914
+ this.clearSyncProcessKey(hash);
2915
+ return;
2916
+ }
2917
+ const hashClaimants = this.syncInFlightQueue.get(hash);
2918
+ if (!hashClaimants || !this.syncInFlightQueue.has(coordinate)) {
2919
+ return;
2920
+ }
2921
+ const expiresAt = Math.min(
2922
+ this.syncInFlightQueueExpiresAt.get(coordinate) ?? Infinity,
2923
+ this.syncInFlightQueueExpiresAt.get(hash) ?? Infinity,
2924
+ );
2925
+ for (const claimant of [...hashClaimants]) {
2926
+ this.addPendingSyncClaim(coordinate, claimant, expiresAt);
2927
+ }
2928
+ if (Number.isFinite(expiresAt)) {
2929
+ this.movePendingSyncKeyExpiryEarlier(coordinate, expiresAt);
2930
+ }
2931
+ for (const target of [...(this.syncInFlightTargetsByKey.get(hash) ?? [])]) {
2932
+ const state = this.syncInFlight.get(target)?.get(hash);
2933
+ if (state) {
2934
+ this.setSyncInFlightTargetKey(target, coordinate, state.timestamp);
2935
+ }
2936
+ }
2937
+ this.clearSyncProcessKey(hash);
2938
+ }
2939
+
2940
+ private refreshQueuedSyncCoordinateAliases(): void {
2941
+ if (
2942
+ this.syncInFlightQueuedCoordinates.size === 0 &&
2943
+ this.syncInFlightQueueClaimants.size < this.syncInFlightQueue.size
2944
+ ) {
2945
+ // Defensive compatibility for callers/tests that seed the public queue
2946
+ // directly. Keep hydration bounded; internal writes register coordinates
2947
+ // when the key is first admitted.
2948
+ let inspected = 0;
2949
+ for (const key of this.syncInFlightQueue.keys()) {
2950
+ if (typeof key === "bigint") {
2951
+ this.syncInFlightQueuedCoordinates.add(key);
2952
+ }
2953
+ inspected += 1;
2954
+ if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
2955
+ break;
2956
+ }
2957
+ }
2958
+ }
2959
+ if (this.syncInFlightQueuedCoordinates.size === 0) {
2960
+ this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
2961
+ return;
2962
+ }
2963
+ this.syncInFlightQueuedCoordinateRefreshIterator ??=
2964
+ this.syncInFlightQueuedCoordinates.values();
2965
+ for (
2966
+ let refreshed = 0;
2967
+ refreshed < MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE;
2968
+ refreshed += 1
2969
+ ) {
2970
+ const next = this.syncInFlightQueuedCoordinateRefreshIterator.next();
2971
+ if (next.done) {
2972
+ this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
2973
+ break;
2974
+ }
2975
+ this.refreshQueuedSyncCoordinateAlias(next.value);
2976
+ }
2977
+ }
2978
+
2979
+ private getQueuedSyncKeyForAdmission(
2980
+ key: SyncableKey,
2981
+ ): SyncableKey | typeof QUEUED_SYNC_ALIAS_REFRESH_PENDING | undefined {
2982
+ const getValidQueuedKey = (
2983
+ candidate: SyncableKey,
2984
+ ): SyncableKey | undefined => {
2985
+ if (!this.syncInFlightQueue.has(candidate)) {
2986
+ return undefined;
2987
+ }
2988
+ const expiresAt = this.syncInFlightQueueExpiresAt.get(candidate);
2989
+ if (expiresAt != null && expiresAt <= Date.now()) {
2990
+ this.clearSyncProcessKey(candidate);
2991
+ return undefined;
2992
+ }
2993
+ return candidate;
2994
+ };
2995
+ if (getValidQueuedKey(key) != null) {
2996
+ if (typeof key === "bigint") {
2997
+ this.refreshQueuedSyncCoordinateAlias(key);
2998
+ return getValidQueuedKey(key);
2999
+ }
649
3000
  return key;
650
3001
  }
651
3002
  if (typeof key === "string") {
652
- for (const queuedKey of this.syncInFlightQueue.keys()) {
3003
+ const aliases = this.syncInFlightQueuedCoordinatesByHash.get(key);
3004
+ if (aliases) {
3005
+ let inspected = 0;
3006
+ for (const alias of aliases) {
3007
+ if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
3008
+ return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
3009
+ }
3010
+ inspected += 1;
3011
+ this.refreshQueuedSyncCoordinateAlias(alias);
3012
+ if (
3013
+ this.syncInFlightQueuedCoordinatesByHash.get(key)?.has(alias) ===
3014
+ true
3015
+ ) {
3016
+ const validAlias = getValidQueuedKey(alias);
3017
+ if (validAlias != null) {
3018
+ return validAlias;
3019
+ }
3020
+ }
3021
+ }
653
3022
  if (
654
- typeof queuedKey === "bigint" &&
655
- this.coordinateToHash.get(queuedKey) === key
3023
+ (this.syncInFlightQueuedCoordinatesByHash.get(key)?.size ?? 0) > 0
656
3024
  ) {
657
- return queuedKey;
3025
+ return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
658
3026
  }
659
3027
  }
660
3028
  return undefined;
661
3029
  }
662
3030
  const hash = this.coordinateToHash.get(key);
663
- return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
3031
+ return hash != null ? getValidQueuedKey(hash) : undefined;
664
3032
  }
665
3033
 
666
- startRepairSession(properties: {
667
- entries: Map<string, SyncEntryCoordinates<R>>;
668
- targets: string[];
669
- mode?: RepairSessionMode;
670
- timeoutMs?: number;
671
- retryIntervalsMs?: number[];
672
- }): RepairSession {
673
- const mode = properties.mode ?? "best-effort";
674
- const startedAt = Date.now();
675
- const timeoutMs = Math.max(
676
- 1,
677
- Math.floor(
678
- properties.timeoutMs ??
679
- (mode === "convergent"
680
- ? DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS
681
- : DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS),
682
- ),
3034
+ private addPendingSyncClaim(
3035
+ key: SyncableKey,
3036
+ from: PublicSignKey,
3037
+ expiresAt?: number,
3038
+ ): boolean {
3039
+ const fromHash = from.hashcode();
3040
+ let peers = this.syncInFlightQueue.get(key);
3041
+ let claimants = this.syncInFlightQueueClaimants.get(key);
3042
+ let claimantIndexes = this.syncInFlightQueueClaimantIndexes.get(key);
3043
+ if (!peers) {
3044
+ peers = [];
3045
+ this.syncInFlightQueue.set(key, peers);
3046
+ claimants = new Set();
3047
+ this.syncInFlightQueueClaimants.set(key, claimants);
3048
+ claimantIndexes = new Map();
3049
+ this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
3050
+ const deadline = expiresAt ?? Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS;
3051
+ this.syncInFlightQueueExpiresAt.set(key, deadline);
3052
+ const expiryNode: PendingSyncExpiryNode = {
3053
+ kind: "key",
3054
+ key,
3055
+ expiresAt: deadline,
3056
+ heapIndex: -1,
3057
+ };
3058
+ this.pendingSyncKeyExpiryNodes.set(key, expiryNode);
3059
+ this.pushPendingSyncExpiry(expiryNode);
3060
+ if (typeof key === "bigint") {
3061
+ this.refreshQueuedSyncCoordinateAlias(key);
3062
+ }
3063
+ } else if (!claimants) {
3064
+ // Defensive compatibility for callers/tests that seed the public queue
3065
+ // maps directly. Internally every retained key gets this set at creation.
3066
+ claimants = new Set(peers.map((peer) => peer.hashcode()));
3067
+ this.syncInFlightQueueClaimants.set(key, claimants);
3068
+ this.pendingSyncClaimCount += claimants.size;
3069
+ }
3070
+ if (!claimantIndexes) {
3071
+ claimantIndexes = new Map();
3072
+ for (let index = 0; index < peers.length; index += 1) {
3073
+ claimantIndexes.set(peers[index]!.hashcode(), index);
3074
+ }
3075
+ this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
3076
+ }
3077
+ if (claimants.has(fromHash)) {
3078
+ return false;
3079
+ }
3080
+
3081
+ claimantIndexes.set(fromHash, peers.length);
3082
+ peers.push(from);
3083
+ claimants.add(fromHash);
3084
+ let inverted = this.syncInFlightQueueInverted.get(fromHash);
3085
+ if (!inverted) {
3086
+ inverted = new Set();
3087
+ this.syncInFlightQueueInverted.set(fromHash, inverted);
3088
+ }
3089
+ inverted.add(key);
3090
+ this.pendingSyncClaimCount += 1;
3091
+ this.schedulePendingSyncKeyExpiry();
3092
+ return true;
3093
+ }
3094
+
3095
+ private hasPendingSyncClaim(key: SyncableKey, peer: string): boolean {
3096
+ const claimants = this.syncInFlightQueueClaimants.get(key);
3097
+ if (claimants) {
3098
+ return claimants.has(peer);
3099
+ }
3100
+ const peers = this.syncInFlightQueue.get(key);
3101
+ if (!peers) {
3102
+ return false;
3103
+ }
3104
+ const hydrated = new Set(peers.map((candidate) => candidate.hashcode()));
3105
+ this.syncInFlightQueueClaimants.set(key, hydrated);
3106
+ const indexes = new Map<string, number>();
3107
+ for (let index = 0; index < peers.length; index += 1) {
3108
+ indexes.set(peers[index]!.hashcode(), index);
3109
+ }
3110
+ this.syncInFlightQueueClaimantIndexes.set(key, indexes);
3111
+ this.pendingSyncClaimCount += hydrated.size;
3112
+ return hydrated.has(peer);
3113
+ }
3114
+
3115
+ private filterDispatchablePendingSyncClaims(
3116
+ keys: SyncableKey[],
3117
+ peer: string,
3118
+ epoch: SyncDispatchTargetEpoch,
3119
+ ): SyncableKey[] {
3120
+ if (this.syncDispatchTargetEpochs.get(peer) !== epoch) {
3121
+ return [];
3122
+ }
3123
+ const now = Date.now();
3124
+ const dispatchable: SyncableKey[] = [];
3125
+ for (const key of keys) {
3126
+ const expiresAt = this.syncInFlightQueueExpiresAt.get(key);
3127
+ if (expiresAt != null && expiresAt <= now) {
3128
+ this.clearSyncProcessKey(key);
3129
+ continue;
3130
+ }
3131
+ if (
3132
+ this.syncInFlightQueue.has(key) &&
3133
+ this.hasPendingSyncClaim(key, peer)
3134
+ ) {
3135
+ dispatchable.push(key);
3136
+ }
3137
+ }
3138
+ return dispatchable;
3139
+ }
3140
+
3141
+ private canStartPendingSyncLookup(peer: string): boolean {
3142
+ return (
3143
+ this.pendingSyncAdmissionReservations.size +
3144
+ this.pendingCoordinateLookupCount <
3145
+ MAX_PENDING_SIMPLE_SYNC_LOOKUPS_GLOBAL &&
3146
+ (this.pendingSyncAdmissionReservationsByPeer.get(peer)?.size ?? 0) +
3147
+ (this.pendingCoordinateLookupCountByPeer.get(peer) ?? 0) <
3148
+ MAX_PENDING_SIMPLE_SYNC_LOOKUPS_PER_PEER
683
3149
  );
684
- const retryIntervalsMs = this.normalizeRetryIntervals(
685
- mode,
686
- properties.retryIntervalsMs,
3150
+ }
3151
+
3152
+ private tryAcquireCoordinateLookup(peer: string): (() => void) | undefined {
3153
+ if (!this.canStartPendingSyncLookup(peer)) {
3154
+ return undefined;
3155
+ }
3156
+ this.pendingCoordinateLookupCount += 1;
3157
+ this.pendingCoordinateLookupCountByPeer.set(
3158
+ peer,
3159
+ (this.pendingCoordinateLookupCountByPeer.get(peer) ?? 0) + 1,
687
3160
  );
688
- const allHashes = this.getPrioritizedHashes(properties.entries);
689
- const trackedHashes =
690
- mode === "convergent" &&
691
- allHashes.length > this.maxConvergentTrackedHashes
692
- ? allHashes.slice(0, this.maxConvergentTrackedHashes)
693
- : allHashes;
694
- const truncated = trackedHashes.length < allHashes.length;
695
- const targets = [...new Set(properties.targets)];
696
- const id = `repair-${++this.repairSessionCounter}`;
697
- const deferred = createDeferred<RepairSessionResult[]>();
3161
+ let released = false;
3162
+ return () => {
3163
+ if (released) {
3164
+ return;
3165
+ }
3166
+ released = true;
3167
+ this.pendingCoordinateLookupCount -= 1;
3168
+ const remaining =
3169
+ (this.pendingCoordinateLookupCountByPeer.get(peer) ?? 1) - 1;
3170
+ if (remaining === 0) {
3171
+ this.pendingCoordinateLookupCountByPeer.delete(peer);
3172
+ } else {
3173
+ this.pendingCoordinateLookupCountByPeer.set(peer, remaining);
3174
+ }
3175
+ };
3176
+ }
698
3177
 
699
- const targetStates = new Map<string, RepairSessionTargetState>();
700
- for (const target of targets) {
701
- targetStates.set(target, {
702
- unresolved: new Set(trackedHashes),
703
- requestedCount: trackedHashes.length,
704
- requestedTotalCount: allHashes.length,
705
- attempts: 0,
706
- });
3178
+ private tryAcquireCoordinateResponse(peer: string): (() => void) | undefined {
3179
+ if (
3180
+ this.pendingCoordinateResponseCount >=
3181
+ MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_GLOBAL ||
3182
+ (this.pendingCoordinateResponseCountByPeer.get(peer) ?? 0) >=
3183
+ MAX_PENDING_SIMPLE_COORDINATE_RESPONSES_PER_PEER
3184
+ ) {
3185
+ return undefined;
707
3186
  }
3187
+ this.pendingCoordinateResponseCount += 1;
3188
+ this.pendingCoordinateResponseCountByPeer.set(
3189
+ peer,
3190
+ (this.pendingCoordinateResponseCountByPeer.get(peer) ?? 0) + 1,
3191
+ );
3192
+ let released = false;
3193
+ return () => {
3194
+ if (released) {
3195
+ return;
3196
+ }
3197
+ released = true;
3198
+ this.pendingCoordinateResponseCount -= 1;
3199
+ const remaining =
3200
+ (this.pendingCoordinateResponseCountByPeer.get(peer) ?? 1) - 1;
3201
+ if (remaining === 0) {
3202
+ this.pendingCoordinateResponseCountByPeer.delete(peer);
3203
+ } else {
3204
+ this.pendingCoordinateResponseCountByPeer.set(peer, remaining);
3205
+ }
3206
+ };
3207
+ }
708
3208
 
709
- const session: RepairSessionState = {
710
- id,
711
- mode,
712
- startedAt,
713
- timeoutMs,
714
- retryIntervalsMs,
715
- targets: targetStates,
716
- truncated,
717
- deferred,
718
- cancelled: false,
3209
+ private reservePendingSyncAdmission(
3210
+ peer: string,
3211
+ identities: SyncableKey[],
3212
+ ): PendingSyncAdmissionReservation | undefined {
3213
+ const count = identities.length;
3214
+ if (count <= 0) {
3215
+ return undefined;
3216
+ }
3217
+ const reservation: PendingSyncAdmissionReservation = {
3218
+ peer,
3219
+ remaining: count,
3220
+ active: true,
3221
+ released: false,
3222
+ expiresAt: Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS,
3223
+ identities: new Set(identities),
3224
+ retainedSettled: 0,
3225
+ };
3226
+ this.pendingSyncAdmissionReservations.add(reservation);
3227
+ let peerReservations =
3228
+ this.pendingSyncAdmissionReservationsByPeer.get(peer);
3229
+ if (!peerReservations) {
3230
+ peerReservations = new Set();
3231
+ this.pendingSyncAdmissionReservationsByPeer.set(peer, peerReservations);
3232
+ }
3233
+ peerReservations.add(reservation);
3234
+ this.pendingSyncActiveAdmissionReservations += 1;
3235
+ this.pendingSyncAdmissionCount += count;
3236
+ this.pendingSyncAdmissionCountByPeer.set(
3237
+ peer,
3238
+ (this.pendingSyncAdmissionCountByPeer.get(peer) ?? 0) + count,
3239
+ );
3240
+ let reservedIdentities =
3241
+ this.pendingSyncAdmissionIdentitiesByPeer.get(peer);
3242
+ if (!reservedIdentities) {
3243
+ reservedIdentities = new Set();
3244
+ this.pendingSyncAdmissionIdentitiesByPeer.set(peer, reservedIdentities);
3245
+ }
3246
+ for (const identity of identities) {
3247
+ reservedIdentities.add(identity);
3248
+ let reservationsForIdentity =
3249
+ this.pendingSyncAdmissionReservationsByIdentity.get(identity);
3250
+ if (!reservationsForIdentity) {
3251
+ reservationsForIdentity = new Set();
3252
+ this.pendingSyncAdmissionReservationsByIdentity.set(
3253
+ identity,
3254
+ reservationsForIdentity,
3255
+ );
3256
+ }
3257
+ reservationsForIdentity.add(reservation);
3258
+ }
3259
+ const expiryNode: PendingSyncExpiryNode = {
3260
+ kind: "admission",
3261
+ reservation,
3262
+ expiresAt: reservation.expiresAt,
3263
+ heapIndex: -1,
719
3264
  };
3265
+ this.pendingSyncAdmissionExpiryNodes.set(reservation, expiryNode);
3266
+ this.pushPendingSyncExpiry(expiryNode);
3267
+ this.schedulePendingSyncKeyExpiry();
3268
+ return reservation;
3269
+ }
720
3270
 
721
- if (allHashes.length === 0 || targets.length === 0) {
722
- deferred.resolve(this.buildRepairSessionResult(session, true));
723
- return {
724
- id,
725
- done: deferred.promise,
726
- cancel: () => {
727
- // no-op
728
- },
729
- };
3271
+ private removePendingSyncAdmissionIdentity(
3272
+ reservation: PendingSyncAdmissionReservation,
3273
+ identity: SyncableKey,
3274
+ options?: { retainQuota?: boolean },
3275
+ ): boolean {
3276
+ if (!reservation.identities.delete(identity)) {
3277
+ return false;
3278
+ }
3279
+ if (options?.retainQuota === true) {
3280
+ reservation.retainedSettled += 1;
3281
+ } else {
3282
+ reservation.remaining -= 1;
3283
+ this.pendingSyncAdmissionCount -= 1;
3284
+ const peerCount =
3285
+ (this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) - 1;
3286
+ if (peerCount === 0) {
3287
+ this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
3288
+ } else {
3289
+ this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
3290
+ }
3291
+ }
3292
+ const reservedIdentities = this.pendingSyncAdmissionIdentitiesByPeer.get(
3293
+ reservation.peer,
3294
+ );
3295
+ reservedIdentities?.delete(identity);
3296
+ if (reservedIdentities?.size === 0) {
3297
+ this.pendingSyncAdmissionIdentitiesByPeer.delete(reservation.peer);
3298
+ }
3299
+ const reservationsForIdentity =
3300
+ this.pendingSyncAdmissionReservationsByIdentity.get(identity);
3301
+ reservationsForIdentity?.delete(reservation);
3302
+ if (reservationsForIdentity?.size === 0) {
3303
+ this.pendingSyncAdmissionReservationsByIdentity.delete(identity);
730
3304
  }
3305
+ return true;
3306
+ }
731
3307
 
732
- // For capped convergent sessions, still dispatch the full set once so large
733
- // repairs are not limited to tracked hashes.
734
- if (mode === "convergent" && truncated) {
735
- void this.onMaybeMissingEntries({
736
- entries: properties.entries,
737
- targets,
738
- }).catch(() => {
739
- // Best-effort: retries on tracked hashes continue via runRepairSession.
3308
+ private clearPendingSyncAdmissionIdentity(identity: SyncableKey): void {
3309
+ const reservations =
3310
+ this.pendingSyncAdmissionReservationsByIdentity.get(identity);
3311
+ if (!reservations) {
3312
+ return;
3313
+ }
3314
+ for (const reservation of [...reservations]) {
3315
+ this.removePendingSyncAdmissionIdentity(reservation, identity, {
3316
+ retainQuota: true,
740
3317
  });
741
3318
  }
3319
+ }
742
3320
 
743
- session.timer = setTimeout(() => {
744
- this.finalizeRepairSession(id, false);
745
- }, timeoutMs);
746
- session.timer.unref?.();
747
-
748
- this.repairSessions.set(id, session);
749
- void this.runRepairSession(id).catch(() => {
750
- this.finalizeRepairSession(id, false);
751
- });
3321
+ private consumePendingSyncAdmission(
3322
+ reservation: PendingSyncAdmissionReservation,
3323
+ identity: SyncableKey,
3324
+ ): "consumed" | "settled" | "invalid" {
3325
+ if (!reservation.active || reservation.expiresAt <= Date.now()) {
3326
+ if (reservation.expiresAt <= Date.now()) {
3327
+ this.invalidatePendingSyncAdmission(reservation);
3328
+ }
3329
+ return "invalid";
3330
+ }
3331
+ if (!reservation.identities.has(identity)) {
3332
+ return "settled";
3333
+ }
3334
+ this.removePendingSyncAdmissionIdentity(reservation, identity);
3335
+ if (reservation.remaining === 0) {
3336
+ this.removePendingSyncAdmissionExpiry(reservation);
3337
+ reservation.active = false;
3338
+ reservation.released = true;
3339
+ this.pendingSyncActiveAdmissionReservations -= 1;
3340
+ this.pendingSyncAdmissionReservations.delete(reservation);
3341
+ const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
3342
+ reservation.peer,
3343
+ );
3344
+ peerReservations?.delete(reservation);
3345
+ if (peerReservations?.size === 0) {
3346
+ this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
3347
+ }
3348
+ this.clearPendingSyncExpiryTimerIfIdle();
3349
+ }
3350
+ return "consumed";
3351
+ }
752
3352
 
753
- return {
754
- id,
755
- done: deferred.promise,
756
- cancel: () => {
757
- this.finalizeRepairSession(id, false);
758
- },
759
- };
3353
+ private transferPendingSyncAdmissionIdentity(
3354
+ peer: string,
3355
+ identity: SyncableKey,
3356
+ ): number | undefined {
3357
+ const reservations =
3358
+ this.pendingSyncAdmissionReservationsByIdentity.get(identity);
3359
+ if (!reservations) {
3360
+ return undefined;
3361
+ }
3362
+ for (const reservation of reservations) {
3363
+ if (
3364
+ reservation.peer !== peer ||
3365
+ reservation.released ||
3366
+ reservation.expiresAt <= Date.now() ||
3367
+ !this.removePendingSyncAdmissionIdentity(reservation, identity, {
3368
+ retainQuota: true,
3369
+ })
3370
+ ) {
3371
+ continue;
3372
+ }
3373
+ // The original resolver may be non-abortable and still retains its input
3374
+ // arrays. Keep that reservation charged until its queueSync finally
3375
+ // settles; the queued claim is counted separately and can disappear
3376
+ // without returning the resolver's quota early.
3377
+ return reservation.expiresAt;
3378
+ }
3379
+ return undefined;
760
3380
  }
761
3381
 
762
- async onMaybeMissingEntries(properties: {
763
- entries: Map<string, SyncEntryCoordinates<R>>;
764
- targets: string[];
765
- }): Promise<void> {
766
- await this.onMaybeMissingHashes({
767
- hashes: this.getPrioritizedHashes(properties.entries),
768
- targets: properties.targets,
769
- });
3382
+ private invalidatePendingSyncAdmission(
3383
+ reservation?: PendingSyncAdmissionReservation,
3384
+ ): void {
3385
+ if (!reservation || reservation.released || !reservation.active) {
3386
+ return;
3387
+ }
3388
+ // Expiry/disconnect invalidates late lookup results, but it must not return
3389
+ // the quota slot while the underlying storage/index work is still alive.
3390
+ // Those lookups are not generally abortable; only queueSync's finally block
3391
+ // may release their active-work accounting.
3392
+ this.removePendingSyncAdmissionExpiry(reservation);
3393
+ reservation.active = false;
3394
+ this.pendingSyncActiveAdmissionReservations -= 1;
3395
+ this.clearPendingSyncExpiryTimerIfIdle();
770
3396
  }
771
3397
 
772
- async onMaybeMissingHashes(properties: {
773
- hashes: Iterable<string>;
774
- targets: string[];
775
- }): Promise<void> {
776
- const profile = this.syncOptions?.profile;
777
- const startedAt = syncProfileStart(profile);
778
- const hashes = [...properties.hashes];
779
- const chunks = this.chunk(hashes, this.maxHashesPerMessage);
780
- try {
781
- await chunks.reduce(
782
- (promise, chunk) =>
783
- promise.then(() =>
784
- this.rpc.send(new RequestMaybeSync({ hashes: chunk }), {
785
- priority: SYNC_MESSAGE_PRIORITY,
786
- mode: new SilentDelivery({
787
- to: properties.targets,
788
- redundancy: 1,
789
- }),
790
- }),
791
- ),
792
- Promise.resolve(),
793
- );
794
- } finally {
795
- if (profile) {
796
- emitSyncProfileDuration(profile, startedAt, {
797
- name: "simple.onMaybeMissingEntries",
798
- entries: hashes.length,
799
- messages: chunks.length,
800
- targets: properties.targets.length,
801
- });
3398
+ private releasePendingSyncAdmission(
3399
+ reservation?: PendingSyncAdmissionReservation,
3400
+ ): void {
3401
+ if (!reservation || reservation.released) {
3402
+ return;
3403
+ }
3404
+ this.removePendingSyncAdmissionExpiry(reservation);
3405
+ for (const identity of [...reservation.identities]) {
3406
+ this.removePendingSyncAdmissionIdentity(reservation, identity);
3407
+ }
3408
+ if (reservation.retainedSettled > 0) {
3409
+ const retainedSettled = reservation.retainedSettled;
3410
+ reservation.retainedSettled = 0;
3411
+ reservation.remaining -= retainedSettled;
3412
+ this.pendingSyncAdmissionCount -= retainedSettled;
3413
+ const peerCount =
3414
+ (this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) -
3415
+ retainedSettled;
3416
+ if (peerCount === 0) {
3417
+ this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
3418
+ } else {
3419
+ this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
802
3420
  }
803
3421
  }
3422
+ if (reservation.active) {
3423
+ this.pendingSyncActiveAdmissionReservations -= 1;
3424
+ }
3425
+ reservation.active = false;
3426
+ reservation.released = true;
3427
+ this.pendingSyncAdmissionReservations.delete(reservation);
3428
+ const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
3429
+ reservation.peer,
3430
+ );
3431
+ peerReservations?.delete(reservation);
3432
+ if (peerReservations?.size === 0) {
3433
+ this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
3434
+ }
3435
+ this.clearPendingSyncExpiryTimerIfIdle();
804
3436
  }
805
3437
 
806
- /**
807
- * Ship exchange heads to one peer: the fused native raw path when the
808
- * peer advertised raw capability and the shared log provided a fused
809
- * sender, otherwise the TS message path (raw or plain by capability).
810
- * Returns the number of messages sent and whether the fused path ran.
811
- */
812
- private async shipExchangeHeads(
813
- hashes: string[],
814
- to: PublicSignKey,
815
- canReceiveRaw: boolean,
816
- ): Promise<{ messages: number; fused: boolean }> {
817
- if (canReceiveRaw && this.sendRawExchangeHeads) {
818
- const sentMessages = await this.sendRawExchangeHeads(hashes, [
819
- to.hashcode(),
820
- ]);
821
- if (sentMessages !== undefined) {
822
- return { messages: sentMessages, fused: true };
823
- }
3438
+ private clearPendingSyncAdmissions(peer?: string): void {
3439
+ const reservations =
3440
+ peer == null
3441
+ ? this.pendingSyncAdmissionReservations
3442
+ : this.pendingSyncAdmissionReservationsByPeer.get(peer);
3443
+ if (!reservations) {
3444
+ return;
824
3445
  }
825
- let messages = 0;
826
- const messageGenerator = canReceiveRaw
827
- ? createRawExchangeHeadsMessages(
828
- this.log,
829
- hashes,
830
- this.syncOptions?.profile,
831
- )
832
- : createExchangeHeadsMessages(this.log, hashes);
833
- for await (const message of messageGenerator) {
834
- messages += 1;
835
- await this.rpc.send(message, {
836
- mode: new SilentDelivery({ to: [to], redundancy: 1 }),
837
- });
3446
+ for (const reservation of [...reservations]) {
3447
+ this.invalidatePendingSyncAdmission(reservation);
838
3448
  }
839
- return { messages, fused: false };
840
3449
  }
841
3450
 
842
- async onMessage(
843
- msg: TransportMessage,
844
- context: RequestContext,
845
- ): Promise<boolean> {
846
- const from = context.from!;
847
- if (msg instanceof RequestMaybeSync) {
848
- await this.queueSync(msg.hashes, from);
849
- return true;
850
- } else if (
851
- msg instanceof ResponseMaybeSync ||
852
- msg instanceof ResponseMaybeSyncCapabilities
853
- ) {
854
- // TODO perhaps send less messages to more receivers for performance reasons?
855
- // TODO wait for previous send to target before trying to send more?
3451
+ private swapPendingSyncExpiry(left: number, right: number): void {
3452
+ const leftNode = this.pendingSyncExpiryHeap[left]!;
3453
+ const rightNode = this.pendingSyncExpiryHeap[right]!;
3454
+ this.pendingSyncExpiryHeap[left] = rightNode;
3455
+ this.pendingSyncExpiryHeap[right] = leftNode;
3456
+ rightNode.heapIndex = left;
3457
+ leftNode.heapIndex = right;
3458
+ }
856
3459
 
857
- const profile = this.syncOptions?.profile;
858
- const startedAt = syncProfileStart(profile);
859
- const hashes = this.filterRecentlySentExchangeHeads(msg.hashes, from);
860
- let messages = 0;
861
- let fused = false;
862
- try {
863
- ({ messages, fused } = await this.shipExchangeHeads(
864
- hashes,
865
- context.from!,
866
- canReceiveRawExchangeHeads(msg),
867
- ));
868
- } finally {
869
- if (profile) {
870
- emitSyncProfileDuration(profile, startedAt, {
871
- name: "simple.exchangeHeads",
872
- entries: hashes.length,
873
- messages,
874
- targets: 1,
875
- details: { source: "responseMaybeSync", fused },
876
- });
877
- }
3460
+ private pushPendingSyncExpiry(node: PendingSyncExpiryNode): void {
3461
+ node.heapIndex = this.pendingSyncExpiryHeap.length;
3462
+ this.pendingSyncExpiryHeap.push(node);
3463
+ let index = node.heapIndex;
3464
+ while (index > 0) {
3465
+ const parent = Math.floor((index - 1) / 2);
3466
+ if (
3467
+ this.pendingSyncExpiryHeap[parent]!.expiresAt <=
3468
+ this.pendingSyncExpiryHeap[index]!.expiresAt
3469
+ ) {
3470
+ break;
878
3471
  }
879
- return true;
880
- } else if (
881
- msg instanceof RequestMaybeSyncCoordinate ||
882
- msg instanceof RequestMaybeSyncCoordinateCapabilities
3472
+ this.swapPendingSyncExpiry(parent, index);
3473
+ index = parent;
3474
+ }
3475
+ }
3476
+
3477
+ private removePendingSyncExpiry(node: PendingSyncExpiryNode): void {
3478
+ const index = node.heapIndex;
3479
+ if (
3480
+ index < 0 ||
3481
+ index >= this.pendingSyncExpiryHeap.length ||
3482
+ this.pendingSyncExpiryHeap[index] !== node
883
3483
  ) {
884
- const profile = this.syncOptions?.profile;
885
- const lookupStartedAt = syncProfileStart(profile);
886
- const hashes = await getHashesFromSymbols(
887
- msg.hashNumbers,
888
- this.entryIndex,
889
- this.coordinateToHash,
890
- this.resolveHashesForSymbols,
891
- this.resolveHashListForSymbols,
892
- );
893
- if (profile) {
894
- emitSyncProfileDuration(profile, lookupStartedAt, {
895
- name: "simple.coordinateLookup",
896
- entries: hashLookupResultSize(hashes),
897
- symbols: msg.hashNumbers.length,
898
- });
3484
+ return;
3485
+ }
3486
+ const last = this.pendingSyncExpiryHeap.pop()!;
3487
+ node.heapIndex = -1;
3488
+ if (index >= this.pendingSyncExpiryHeap.length) {
3489
+ return;
3490
+ }
3491
+ this.pendingSyncExpiryHeap[index] = last;
3492
+ last.heapIndex = index;
3493
+
3494
+ let current = index;
3495
+ while (current > 0) {
3496
+ const parent = Math.floor((current - 1) / 2);
3497
+ if (
3498
+ this.pendingSyncExpiryHeap[parent]!.expiresAt <=
3499
+ this.pendingSyncExpiryHeap[current]!.expiresAt
3500
+ ) {
3501
+ break;
3502
+ }
3503
+ this.swapPendingSyncExpiry(parent, current);
3504
+ current = parent;
3505
+ }
3506
+ for (;;) {
3507
+ const left = current * 2 + 1;
3508
+ const right = left + 1;
3509
+ let smallest = current;
3510
+ if (
3511
+ left < this.pendingSyncExpiryHeap.length &&
3512
+ this.pendingSyncExpiryHeap[left]!.expiresAt <
3513
+ this.pendingSyncExpiryHeap[smallest]!.expiresAt
3514
+ ) {
3515
+ smallest = left;
3516
+ }
3517
+ if (
3518
+ right < this.pendingSyncExpiryHeap.length &&
3519
+ this.pendingSyncExpiryHeap[right]!.expiresAt <
3520
+ this.pendingSyncExpiryHeap[smallest]!.expiresAt
3521
+ ) {
3522
+ smallest = right;
3523
+ }
3524
+ if (smallest === current) {
3525
+ break;
899
3526
  }
3527
+ this.swapPendingSyncExpiry(current, smallest);
3528
+ current = smallest;
3529
+ }
3530
+ }
900
3531
 
901
- const exchangeStartedAt = syncProfileStart(profile);
902
- const hashesToSend = this.filterRecentlySentExchangeHeads(hashes, from);
903
- let messages = 0;
904
- let fused = false;
905
- try {
906
- // dont set priority 1 here because this will block other messages that should higher priority
907
- ({ messages, fused } = await this.shipExchangeHeads(
908
- hashesToSend,
909
- context.from!,
910
- canReceiveRawExchangeHeads(msg),
911
- ));
912
- } finally {
913
- if (profile) {
914
- emitSyncProfileDuration(profile, exchangeStartedAt, {
915
- name: "simple.exchangeHeads",
916
- entries: hashesToSend.length,
917
- messages,
918
- targets: 1,
919
- details: { source: "requestMaybeSyncCoordinate", fused },
920
- });
3532
+ private removePendingSyncKeyExpiry(key: SyncableKey): void {
3533
+ const node = this.pendingSyncKeyExpiryNodes.get(key);
3534
+ if (!node) {
3535
+ return;
3536
+ }
3537
+ this.pendingSyncKeyExpiryNodes.delete(key);
3538
+ this.removePendingSyncExpiry(node);
3539
+ }
3540
+
3541
+ private movePendingSyncKeyExpiryEarlier(
3542
+ key: SyncableKey,
3543
+ expiresAt: number,
3544
+ ): void {
3545
+ const current = this.syncInFlightQueueExpiresAt.get(key);
3546
+ if (current == null || current <= expiresAt) {
3547
+ return;
3548
+ }
3549
+ this.removePendingSyncKeyExpiry(key);
3550
+ this.syncInFlightQueueExpiresAt.set(key, expiresAt);
3551
+ const node: PendingSyncExpiryNode = {
3552
+ kind: "key",
3553
+ key,
3554
+ expiresAt,
3555
+ heapIndex: -1,
3556
+ };
3557
+ this.pendingSyncKeyExpiryNodes.set(key, node);
3558
+ this.pushPendingSyncExpiry(node);
3559
+ this.schedulePendingSyncKeyExpiry();
3560
+ }
3561
+
3562
+ private removePendingSyncAdmissionExpiry(
3563
+ reservation: PendingSyncAdmissionReservation,
3564
+ ): void {
3565
+ const node = this.pendingSyncAdmissionExpiryNodes.get(reservation);
3566
+ if (!node) {
3567
+ return;
3568
+ }
3569
+ this.pendingSyncAdmissionExpiryNodes.delete(reservation);
3570
+ this.removePendingSyncExpiry(node);
3571
+ }
3572
+
3573
+ private expirePendingSyncKeys(now = Date.now()): void {
3574
+ for (;;) {
3575
+ const node = this.pendingSyncExpiryHeap[0];
3576
+ if (!node || node.expiresAt > now) {
3577
+ break;
3578
+ }
3579
+ this.removePendingSyncExpiry(node);
3580
+ if (node.kind === "key") {
3581
+ if (this.pendingSyncKeyExpiryNodes.get(node.key) !== node) {
3582
+ continue;
3583
+ }
3584
+ this.pendingSyncKeyExpiryNodes.delete(node.key);
3585
+ this.clearSyncProcessKey(node.key);
3586
+ } else {
3587
+ if (
3588
+ this.pendingSyncAdmissionExpiryNodes.get(node.reservation) !== node
3589
+ ) {
3590
+ continue;
921
3591
  }
3592
+ this.pendingSyncAdmissionExpiryNodes.delete(node.reservation);
3593
+ this.invalidatePendingSyncAdmission(node.reservation);
922
3594
  }
923
-
924
- return true;
925
- } else {
926
- return false; // no message was consumed
927
3595
  }
928
3596
  }
929
3597
 
930
- onReceivedEntries(properties: {
931
- entries: EntryWithRefs<any>[];
932
- from: PublicSignKey;
933
- }): Promise<void> | void {
934
- return this.onReceivedEntryHashes({
935
- hashes: properties.entries.map((entry) => entry.entry.hash),
936
- from: properties.from,
937
- });
3598
+ private clearPendingSyncExpiryTimerIfIdle(): void {
3599
+ if (
3600
+ this.pendingSyncExpiryHeap.length === 0 &&
3601
+ this.syncInFlightQueueExpiryTimer != null
3602
+ ) {
3603
+ clearTimeout(this.syncInFlightQueueExpiryTimer);
3604
+ this.syncInFlightQueueExpiryTimer = undefined;
3605
+ }
938
3606
  }
939
3607
 
940
- onReceivedEntryHashes(properties: {
941
- hashes: string[];
942
- from: PublicSignKey;
943
- }): Promise<void> | void {
944
- this.clearSyncInFlightForPeerHashes(
945
- properties.from.hashcode(),
946
- properties.hashes,
3608
+ private schedulePendingSyncKeyExpiry(): void {
3609
+ if (
3610
+ this.syncInFlightQueueExpiryTimer != null ||
3611
+ this.pendingSyncExpiryHeap.length === 0
3612
+ ) {
3613
+ return;
3614
+ }
3615
+ const expiresAt = this.pendingSyncExpiryHeap[0]!.expiresAt;
3616
+ this.syncInFlightQueueExpiryTimer = setTimeout(
3617
+ () => {
3618
+ this.syncInFlightQueueExpiryTimer = undefined;
3619
+ this.expirePendingSyncKeys();
3620
+ this.schedulePendingSyncKeyExpiry();
3621
+ },
3622
+ Math.max(0, expiresAt - Date.now()),
947
3623
  );
948
- this.markRepairSessionResolvedHashes(properties.hashes);
3624
+ this.syncInFlightQueueExpiryTimer.unref?.();
949
3625
  }
950
3626
 
951
3627
  async queueSync(
@@ -953,111 +3629,273 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
953
3629
  from: PublicSignKey,
954
3630
  options?: { skipCheck?: boolean },
955
3631
  ) {
3632
+ if (this.closed === true || keys.length === 0) {
3633
+ return;
3634
+ }
3635
+ // A delayed timer must not let expired claims or admission reservations
3636
+ // keep the exact per-peer/global quota closed to fresh work.
3637
+ this.expirePendingSyncKeys();
3638
+ this.clearPendingSyncExpiryTimerIfIdle();
3639
+ const fromHash = from.hashcode();
3640
+ const canStartLookup =
3641
+ options?.skipCheck === true || this.canStartPendingSyncLookup(fromHash);
3642
+ const peerClaimCount =
3643
+ this.syncInFlightQueueInverted.get(fromHash)?.size ?? 0;
3644
+ let availableClaims = Math.max(
3645
+ 0,
3646
+ Math.min(
3647
+ MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER -
3648
+ peerClaimCount -
3649
+ (this.pendingSyncAdmissionCountByPeer.get(fromHash) ?? 0),
3650
+ MAX_PENDING_SIMPLE_SYNC_KEYS_GLOBAL -
3651
+ this.pendingSyncClaimCount -
3652
+ this.pendingSyncAdmissionCount,
3653
+ ),
3654
+ );
3655
+ const targetEpoch = this.getOrCreateSyncDispatchTargetEpoch(fromHash);
3656
+ const ownershipLifecycleController = this.syncDispatchLifecycleController;
3657
+ const isCapturedLifecycleActive = () =>
3658
+ this.closed !== true &&
3659
+ this.syncDispatchLifecycleController === ownershipLifecycleController &&
3660
+ !ownershipLifecycleController.signal.aborted &&
3661
+ this.syncDispatchTargetEpochs.get(fromHash) === targetEpoch;
956
3662
  const requestHashes: SyncableKey[] = [];
3663
+ const existingRequestHashes: SyncableKey[] = [];
957
3664
  const profile = this.syncOptions?.profile;
958
3665
  const startedAt = syncProfileStart(profile);
959
- const resolveKnownStartedAt = syncProfileStart(profile);
960
- const knownKeys =
961
- options?.skipCheck === true
962
- ? undefined
963
- : await this.resolveKnownSyncKeys(keys);
964
- if (profile) {
965
- emitSyncProfileDuration(profile, resolveKnownStartedAt, {
966
- name: "simple.queueSync.resolveKnown",
967
- entries: keys.length,
968
- count: knownKeys?.keys.size ?? 0,
969
- details: {
970
- checkedCoordinates: knownKeys?.checkedCoordinates === true,
971
- checkedHashes: knownKeys?.checkedHashes === true,
972
- skipCheck: options?.skipCheck === true,
973
- },
974
- });
3666
+ if (!isCapturedLifecycleActive()) {
3667
+ return;
975
3668
  }
976
- const fromHash = from.hashcode();
977
- let queuedHashAliases: Map<string, SyncableKey> | undefined;
978
- const getQueuedSyncKeyForBatch = (key: SyncableKey) => {
979
- if (this.syncInFlightQueue.has(key)) {
980
- return key;
981
- }
982
- if (typeof key === "string") {
983
- if (!queuedHashAliases) {
984
- queuedHashAliases = new Map();
985
- for (const queuedKey of this.syncInFlightQueue.keys()) {
986
- if (typeof queuedKey !== "bigint") {
987
- continue;
988
- }
989
- const hash = this.coordinateToHash.get(queuedKey);
990
- if (hash) {
991
- queuedHashAliases.set(hash, queuedKey);
992
- }
3669
+ if (availableClaims === 0) {
3670
+ return;
3671
+ }
3672
+ this.refreshQueuedSyncCoordinateAliases();
3673
+ const keysToCheck: SyncableKey[] = [];
3674
+ const identitiesToCheck: SyncableKey[] = [];
3675
+ const seen = new Set<SyncableKey>();
3676
+ const pendingAdmissionIdentities =
3677
+ this.pendingSyncAdmissionIdentitiesByPeer.get(fromHash);
3678
+ // Default senders use 1,024-key messages. Capping examined input at the
3679
+ // entire per-peer allowance prevents a duplicate-filled oversized vector
3680
+ // from turning admission itself into unbounded work.
3681
+ const inspectionLimit = canStartLookup
3682
+ ? MAX_PENDING_SIMPLE_SYNC_KEYS_PER_PEER
3683
+ : Math.min(DEFAULT_MAX_HASHES_PER_MESSAGE, keys.length);
3684
+ for (
3685
+ let index = 0;
3686
+ index < keys.length && index < inspectionLimit;
3687
+ index += 1
3688
+ ) {
3689
+ const key = keys[index]!;
3690
+ const queuedKeyResult = this.getQueuedSyncKeyForAdmission(key);
3691
+ if (queuedKeyResult === QUEUED_SYNC_ALIAS_REFRESH_PENDING) {
3692
+ continue;
3693
+ }
3694
+ const queuedKey = queuedKeyResult;
3695
+ const coordinateOrHash = queuedKey ?? key;
3696
+ const identity = this.getPendingSyncKeyIdentity(coordinateOrHash);
3697
+ if (seen.has(identity)) {
3698
+ continue;
3699
+ }
3700
+ seen.add(identity);
3701
+
3702
+ if (queuedKey != null) {
3703
+ let transferredDeadline: number | undefined;
3704
+ if (
3705
+ availableClaims > 0 &&
3706
+ pendingAdmissionIdentities?.has(identity) &&
3707
+ !this.hasPendingSyncClaim(queuedKey, fromHash)
3708
+ ) {
3709
+ transferredDeadline = this.transferPendingSyncAdmissionIdentity(
3710
+ fromHash,
3711
+ identity,
3712
+ );
3713
+ if (transferredDeadline != null) {
3714
+ this.movePendingSyncKeyExpiryEarlier(
3715
+ queuedKey,
3716
+ transferredDeadline,
3717
+ );
993
3718
  }
994
3719
  }
995
- return queuedHashAliases.get(key);
3720
+ if (availableClaims === 0) {
3721
+ continue;
3722
+ }
3723
+ if (this.addPendingSyncClaim(queuedKey, from)) {
3724
+ availableClaims -= 1;
3725
+ existingRequestHashes.push(queuedKey);
3726
+ }
3727
+ continue;
996
3728
  }
997
- const hash = this.coordinateToHash.get(key);
998
- return hash && this.syncInFlightQueue.has(hash) ? hash : undefined;
999
- };
1000
3729
 
3730
+ if (
3731
+ pendingAdmissionIdentities?.has(identity) ||
3732
+ !canStartLookup ||
3733
+ availableClaims === 0
3734
+ ) {
3735
+ continue;
3736
+ }
3737
+ keysToCheck.push(key);
3738
+ identitiesToCheck.push(identity);
3739
+ availableClaims -= 1;
3740
+ }
3741
+ const admission = this.reservePendingSyncAdmission(
3742
+ fromHash,
3743
+ identitiesToCheck,
3744
+ );
3745
+ const dispatchableExistingRequestHashes =
3746
+ this.filterDispatchablePendingSyncClaims(
3747
+ existingRequestHashes,
3748
+ fromHash,
3749
+ targetEpoch,
3750
+ );
3751
+ const existingRequest =
3752
+ dispatchableExistingRequestHashes.length > 0
3753
+ ? this.requestSync(dispatchableExistingRequestHashes, [fromHash], {
3754
+ ownershipLifecycleController,
3755
+ targetEpochs: new Map([[fromHash, targetEpoch]]),
3756
+ createTargetEpochs: false,
3757
+ })
3758
+ : undefined;
3759
+ // Existing queued claims must not wait behind a potentially blocked storage
3760
+ // lookup for unrelated new keys. Observe this eagerly-started request even
3761
+ // when a later lifecycle check returns before joining it.
3762
+ void existingRequest?.catch(() => {});
3763
+ const resolveKnownStartedAt = syncProfileStart(profile);
1001
3764
  try {
3765
+ const knownKeys =
3766
+ options?.skipCheck === true || keysToCheck.length === 0
3767
+ ? undefined
3768
+ : await this.resolveKnownSyncKeys(keysToCheck);
3769
+ if (!isCapturedLifecycleActive()) {
3770
+ return;
3771
+ }
3772
+ if (profile) {
3773
+ emitSyncProfileDuration(profile, resolveKnownStartedAt, {
3774
+ name: "simple.queueSync.resolveKnown",
3775
+ entries: keysToCheck.length,
3776
+ count: knownKeys?.keys.size ?? 0,
3777
+ details: {
3778
+ checkedCoordinates: knownKeys?.checkedCoordinates === true,
3779
+ checkedHashes: knownKeys?.checkedHashes === true,
3780
+ skipCheck: options?.skipCheck === true,
3781
+ },
3782
+ });
3783
+ }
3784
+
3785
+ if (keysToCheck.length > 0) {
3786
+ // A resolver/index lookup may have populated coordinateToHash while
3787
+ // it yielded. Refresh another fixed-size slice, never the full queue.
3788
+ this.refreshQueuedSyncCoordinateAliases();
3789
+ }
1002
3790
  const loopStartedAt = syncProfileStart(profile);
1003
- for (const key of keys) {
1004
- const coordinateOrHash = getQueuedSyncKeyForBatch(key) ?? key;
3791
+ for (let index = 0; index < keysToCheck.length; index += 1) {
3792
+ const key = keysToCheck[index]!;
3793
+ const identity = identitiesToCheck[index]!;
3794
+ if (!isCapturedLifecycleActive()) {
3795
+ return;
3796
+ }
3797
+ const queuedKeyResult = this.getQueuedSyncKeyForAdmission(key);
3798
+ if (queuedKeyResult === QUEUED_SYNC_ALIAS_REFRESH_PENDING) {
3799
+ const consumption = this.consumePendingSyncAdmission(
3800
+ admission!,
3801
+ identity,
3802
+ );
3803
+ if (consumption === "invalid") {
3804
+ return;
3805
+ }
3806
+ continue;
3807
+ }
3808
+ const coordinateOrHash = queuedKeyResult ?? key;
1005
3809
  const inFlight = this.syncInFlightQueue.get(coordinateOrHash);
1006
3810
  if (inFlight) {
1007
- if (!inFlight.find((x) => x.hashcode() === fromHash)) {
1008
- inFlight.push(from);
1009
- let inverted = this.syncInFlightQueueInverted.get(fromHash);
1010
- if (!inverted) {
1011
- inverted = new Set();
1012
- this.syncInFlightQueueInverted.set(fromHash, inverted);
3811
+ if (!this.hasPendingSyncClaim(coordinateOrHash, fromHash)) {
3812
+ const consumption = this.consumePendingSyncAdmission(
3813
+ admission!,
3814
+ identity,
3815
+ );
3816
+ if (consumption === "invalid") {
3817
+ return;
3818
+ }
3819
+ if (consumption === "settled") {
3820
+ continue;
3821
+ }
3822
+ this.movePendingSyncKeyExpiryEarlier(
3823
+ coordinateOrHash,
3824
+ admission!.expiresAt,
3825
+ );
3826
+ const added = this.addPendingSyncClaim(
3827
+ coordinateOrHash,
3828
+ from,
3829
+ admission!.expiresAt,
3830
+ );
3831
+ if (added) {
3832
+ requestHashes.push(coordinateOrHash);
1013
3833
  }
1014
- inverted.add(coordinateOrHash);
1015
3834
  }
1016
- } else if (
1017
- options?.skipCheck ||
1018
- !(await this.checkHasCoordinateOrHash(
1019
- coordinateOrHash,
1020
- knownKeys,
1021
- ))
1022
- ) {
1023
- // Track the initial sender so we can retry if the first request is lost.
1024
- this.syncInFlightQueue.set(coordinateOrHash, [from]);
1025
- let inverted = this.syncInFlightQueueInverted.get(fromHash);
1026
- if (!inverted) {
1027
- inverted = new Set();
1028
- this.syncInFlightQueueInverted.set(fromHash, inverted);
3835
+ } else {
3836
+ const has =
3837
+ options?.skipCheck !== true &&
3838
+ (await this.checkHasCoordinateOrHash(coordinateOrHash, knownKeys));
3839
+ if (!isCapturedLifecycleActive()) {
3840
+ return;
1029
3841
  }
1030
- inverted.add(coordinateOrHash);
1031
- requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash)
1032
- if (
1033
- queuedHashAliases &&
1034
- typeof coordinateOrHash === "bigint"
1035
- ) {
1036
- const hash = this.coordinateToHash.get(coordinateOrHash);
1037
- if (hash) {
1038
- queuedHashAliases.set(hash, coordinateOrHash);
1039
- }
3842
+ if (has) {
3843
+ this.clearPendingSyncAdmissionIdentity(identity);
3844
+ continue;
3845
+ }
3846
+ const consumption = this.consumePendingSyncAdmission(
3847
+ admission!,
3848
+ identity,
3849
+ );
3850
+ if (consumption === "invalid") {
3851
+ return;
3852
+ }
3853
+ if (consumption === "settled") {
3854
+ continue;
1040
3855
  }
3856
+ // Track the initial sender so we can retry if the first request is lost.
3857
+ this.addPendingSyncClaim(
3858
+ coordinateOrHash,
3859
+ from,
3860
+ admission!.expiresAt,
3861
+ );
3862
+ requestHashes.push(coordinateOrHash); // request immediately (first time we have seen this hash)
1041
3863
  }
1042
3864
  }
1043
3865
  if (profile) {
1044
3866
  emitSyncProfileDuration(profile, loopStartedAt, {
1045
3867
  name: "simple.queueSync.plan",
1046
- entries: keys.length,
3868
+ entries: keysToCheck.length,
1047
3869
  count: requestHashes.length,
1048
3870
  targets: 1,
1049
3871
  });
1050
3872
  }
1051
3873
 
1052
- requestHashes.length > 0 &&
1053
- (await this.requestSync(requestHashes, [fromHash]));
3874
+ // Persistent admission work is complete. Do not let an unrelated
3875
+ // blocked transport send retain unused quota.
3876
+ this.releasePendingSyncAdmission(admission);
3877
+ const dispatchableRequestHashes =
3878
+ this.filterDispatchablePendingSyncClaims(
3879
+ requestHashes,
3880
+ fromHash,
3881
+ targetEpoch,
3882
+ );
3883
+ dispatchableRequestHashes.length > 0 &&
3884
+ (await this.requestSync(dispatchableRequestHashes, [fromHash], {
3885
+ ownershipLifecycleController,
3886
+ targetEpochs: new Map([[fromHash, targetEpoch]]),
3887
+ createTargetEpochs: false,
3888
+ }));
3889
+ await existingRequest;
1054
3890
  } finally {
3891
+ this.releasePendingSyncAdmission(admission);
1055
3892
  if (profile) {
1056
3893
  emitSyncProfileDuration(profile, startedAt, {
1057
3894
  name: "simple.queueSync",
1058
3895
  entries: keys.length,
1059
3896
  targets: 1,
1060
3897
  details: {
3898
+ admitted: keysToCheck.length,
1061
3899
  requested: requestHashes.length,
1062
3900
  skipCheck: options?.skipCheck === true,
1063
3901
  },
@@ -1066,10 +3904,24 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1066
3904
  }
1067
3905
  }
1068
3906
 
1069
- private async requestSync(hashes: SyncableKey[], to: Set<string> | string[]) {
1070
- if (hashes.length === 0) {
3907
+ private async requestSync(
3908
+ hashes: SyncableKey[],
3909
+ to: Set<string> | string[],
3910
+ options?: {
3911
+ ownershipLifecycleController?: AbortController;
3912
+ targetEpochs?: Map<string, SyncDispatchTargetEpoch>;
3913
+ createTargetEpochs?: boolean;
3914
+ },
3915
+ ) {
3916
+ const targets = [...new Set(to)];
3917
+ if (hashes.length === 0 || targets.length === 0) {
1071
3918
  return;
1072
3919
  }
3920
+ const lifecycle = this.captureSyncDispatchLifecycle(targets, undefined, {
3921
+ ownershipLifecycleController: options?.ownershipLifecycleController,
3922
+ targetEpochs: options?.targetEpochs,
3923
+ createTargetEpochs: options?.createTargetEpochs,
3924
+ });
1073
3925
  const profile = this.syncOptions?.profile;
1074
3926
  const startedAt = syncProfileStart(profile);
1075
3927
  let coordinateMessages = 0;
@@ -1079,14 +3931,12 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1079
3931
 
1080
3932
  try {
1081
3933
  const now = +new Date();
1082
- for (const node of to) {
1083
- let map = this.syncInFlight.get(node);
1084
- if (!map) {
1085
- map = new Map();
1086
- this.syncInFlight.set(node, map);
3934
+ for (const node of targets) {
3935
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, node)) {
3936
+ continue;
1087
3937
  }
1088
3938
  for (const hash of hashes) {
1089
- map.set(hash, { timestamp: now });
3939
+ this.setSyncInFlightTargetKey(node, hash, now);
1090
3940
  }
1091
3941
  }
1092
3942
 
@@ -1102,48 +3952,85 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1102
3952
  coordinateHashCount = coordinateHashes.length;
1103
3953
  stringHashCount = stringHashes.length;
1104
3954
 
1105
- if (coordinateHashes.length > 0) {
1106
- const chunks = this.chunk(
1107
- coordinateHashes,
1108
- this.maxCoordinatesPerMessage,
1109
- );
1110
- coordinateMessages = chunks.length;
3955
+ if (coordinateHashes.length > 0) {
3956
+ const chunks = this.chunk(
3957
+ coordinateHashes,
3958
+ this.maxCoordinatesPerMessage,
3959
+ );
3960
+ for (const target of targets) {
1111
3961
  for (const chunk of chunks) {
1112
- await this.rpc.send(
1113
- this.syncOptions?.rawExchangeHeads
1114
- ? new RequestMaybeSyncCoordinateCapabilities({
1115
- hashNumbers: chunk,
1116
- })
1117
- : new RequestMaybeSyncCoordinate({ hashNumbers: chunk }),
1118
- {
1119
- mode: new SilentDelivery({ to, redundancy: 1 }),
1120
- priority: SYNC_MESSAGE_PRIORITY,
1121
- },
1122
- );
3962
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
3963
+ break;
3964
+ }
3965
+ try {
3966
+ await this.rpc.send(
3967
+ this.syncOptions?.rawExchangeHeads
3968
+ ? new RequestMaybeSyncCoordinateCapabilities({
3969
+ hashNumbers: chunk,
3970
+ })
3971
+ : new RequestMaybeSyncCoordinate({
3972
+ hashNumbers: chunk,
3973
+ }),
3974
+ {
3975
+ mode: new SilentDelivery({
3976
+ to: [target],
3977
+ redundancy: 1,
3978
+ }),
3979
+ priority: SYNC_MESSAGE_PRIORITY,
3980
+ signal: this.getSyncDispatchSignal(lifecycle, target),
3981
+ },
3982
+ );
3983
+ coordinateMessages += 1;
3984
+ } catch (error) {
3985
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
3986
+ break;
3987
+ }
3988
+ throw error;
3989
+ }
1123
3990
  }
1124
3991
  }
1125
- if (stringHashes.length > 0) {
1126
- const chunks = this.chunk(stringHashes, this.maxHashesPerMessage);
1127
- stringMessages = chunks.length;
3992
+ }
3993
+ if (stringHashes.length > 0) {
3994
+ const chunks = this.chunk(stringHashes, this.maxHashesPerMessage);
3995
+ for (const target of targets) {
1128
3996
  for (const chunk of chunks) {
1129
- await this.rpc.send(
1130
- this.syncOptions?.rawExchangeHeads
1131
- ? new ResponseMaybeSyncCapabilities({ hashes: chunk })
1132
- : new ResponseMaybeSync({ hashes: chunk }),
1133
- {
1134
- mode: new SilentDelivery({ to, redundancy: 1 }),
1135
- priority: SYNC_MESSAGE_PRIORITY,
1136
- },
1137
- );
3997
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
3998
+ break;
3999
+ }
4000
+ try {
4001
+ await this.rpc.send(
4002
+ this.syncOptions?.rawExchangeHeads
4003
+ ? new ResponseMaybeSyncCapabilities({
4004
+ hashes: chunk,
4005
+ })
4006
+ : new ResponseMaybeSync({ hashes: chunk }),
4007
+ {
4008
+ mode: new SilentDelivery({
4009
+ to: [target],
4010
+ redundancy: 1,
4011
+ }),
4012
+ priority: SYNC_MESSAGE_PRIORITY,
4013
+ signal: this.getSyncDispatchSignal(lifecycle, target),
4014
+ },
4015
+ );
4016
+ stringMessages += 1;
4017
+ } catch (error) {
4018
+ if (!this.isSyncDispatchLifecycleActive(lifecycle, target)) {
4019
+ break;
4020
+ }
4021
+ throw error;
4022
+ }
1138
4023
  }
1139
4024
  }
4025
+ }
1140
4026
  } finally {
4027
+ this.finishSyncDispatchLifecycle(lifecycle);
1141
4028
  if (profile) {
1142
4029
  emitSyncProfileDuration(profile, startedAt, {
1143
4030
  name: "simple.requestSync",
1144
4031
  entries: hashes.length,
1145
4032
  messages: coordinateMessages + stringMessages,
1146
- targets: Array.isArray(to) ? to.length : to.size,
4033
+ targets: targets.length,
1147
4034
  details: {
1148
4035
  coordinateHashes: coordinateHashCount,
1149
4036
  stringHashes: stringHashCount,
@@ -1198,6 +4085,12 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1198
4085
  key: string | bigint,
1199
4086
  knownKeys?: KnownSyncKeys,
1200
4087
  ) {
4088
+ if (typeof key === "bigint") {
4089
+ const mappedHash = this.coordinateToHash.get(key);
4090
+ if (mappedHash != null && (await this.log.has(mappedHash))) {
4091
+ return true;
4092
+ }
4093
+ }
1201
4094
  if (knownKeys) {
1202
4095
  if (knownKeys.keys.has(key)) {
1203
4096
  return true;
@@ -1214,8 +4107,31 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1214
4107
  : this.log.has(key);
1215
4108
  }
1216
4109
  async open() {
4110
+ this.syncDispatchLifecycleController.abort();
4111
+ this.clearPendingMaybeSyncResponses();
4112
+ this.syncDispatchTargetEpochs.clear();
4113
+ this.syncDispatchLifecycleController = new AbortController();
4114
+ this.syncInFlightRetryIterator = undefined;
4115
+ this.syncInFlightRetryRemaining = 0;
4116
+ const openLifecycleController = this.syncDispatchLifecycleController;
1217
4117
  this.closed = false;
1218
- const requestSyncLoop = async () => {
4118
+ const isOpenLifecycleActive = () =>
4119
+ this.closed !== true &&
4120
+ this.syncDispatchLifecycleController === openLifecycleController &&
4121
+ !openLifecycleController.signal.aborted;
4122
+ let requestSyncLoop!: () => Promise<void>;
4123
+ const scheduleRequestSyncLoop = () => {
4124
+ if (!isOpenLifecycleActive()) {
4125
+ return;
4126
+ }
4127
+ this.syncMoreInterval = setTimeout(() => {
4128
+ void requestSyncLoop().catch(() => {
4129
+ // The loop is best-effort. Observe all failures so a transport or
4130
+ // storage rejection cannot become an unhandled process rejection.
4131
+ });
4132
+ }, 3e3);
4133
+ };
4134
+ requestSyncLoop = async () => {
1219
4135
  /**
1220
4136
  * This method fetches entries that we potentially want.
1221
4137
  * In a case in which we become replicator of a segment,
@@ -1224,75 +4140,185 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1224
4140
  * so we don't get flooded with the same entry
1225
4141
  */
1226
4142
 
1227
- const requestHashes: SyncableKey[] = [];
1228
- const from: Set<string> = new Set();
1229
- const now = Date.now();
1230
- for (const [key, value] of this.syncInFlightQueue) {
1231
- if (this.closed) {
1232
- return;
4143
+ if (!isOpenLifecycleActive()) {
4144
+ return;
4145
+ }
4146
+ try {
4147
+ const requestHashesByEpoch = new Map<
4148
+ SyncDispatchTargetEpoch,
4149
+ { target: string; hashes: SyncableKey[] }
4150
+ >();
4151
+ const now = Date.now();
4152
+ if (!this.syncInFlightRetryIterator) {
4153
+ this.syncInFlightRetryIterator = this.syncInFlightQueue.entries();
4154
+ this.syncInFlightRetryRemaining = this.syncInFlightQueue.size;
1233
4155
  }
4156
+ for (
4157
+ let inspected = 0;
4158
+ inspected < MAX_SIMPLE_SYNC_RETRY_KEYS_PER_TICK &&
4159
+ this.syncInFlightRetryRemaining > 0;
4160
+ inspected += 1
4161
+ ) {
4162
+ const next = this.syncInFlightRetryIterator.next();
4163
+ if (next.done) {
4164
+ this.syncInFlightRetryIterator = undefined;
4165
+ this.syncInFlightRetryRemaining = 0;
4166
+ break;
4167
+ }
4168
+ this.syncInFlightRetryRemaining -= 1;
4169
+ const [key] = next.value;
4170
+ if (!isOpenLifecycleActive()) {
4171
+ return;
4172
+ }
4173
+ const expiresAt = this.syncInFlightQueueExpiresAt.get(key);
4174
+ if (expiresAt != null && expiresAt <= Date.now()) {
4175
+ this.clearSyncProcessKey(key);
4176
+ continue;
4177
+ }
1234
4178
 
1235
- const has = await this.checkHasCoordinateOrHash(key);
1236
-
1237
- if (!has) {
1238
- if (value.length === 0) {
1239
- // No remaining peers to ask; drop the pending key to avoid leaking.
4179
+ const has = await this.checkHasCoordinateOrHash(key);
4180
+ if (!isOpenLifecycleActive()) {
4181
+ return;
4182
+ }
4183
+ const value = this.syncInFlightQueue.get(key);
4184
+ if (!value) {
4185
+ continue;
4186
+ }
4187
+ const currentExpiresAt = this.syncInFlightQueueExpiresAt.get(key);
4188
+ if (currentExpiresAt != null && currentExpiresAt <= Date.now()) {
1240
4189
  this.clearSyncProcessKey(key);
1241
4190
  continue;
1242
4191
  }
1243
4192
 
1244
- // Ask one peer per key per loop. If a previous request is still considered
1245
- // "recent", wait until the retry window elapses.
1246
- const candidate = value[0]!;
1247
- const publicKeyHash = candidate.hashcode();
1248
- const inflightTimestamp = this.syncInFlight
1249
- .get(publicKeyHash)
1250
- ?.get(key)?.timestamp;
1251
- if (
1252
- inflightTimestamp == null ||
1253
- now - inflightTimestamp >= SIMPLE_SYNC_RETRY_AFTER_MS
1254
- ) {
1255
- requestHashes.push(key);
1256
- from.add(publicKeyHash);
1257
- if (value.length > 1) {
1258
- // Rotate for fairness across multiple possible sources.
1259
- value.push(value.shift()!);
4193
+ if (!has) {
4194
+ if (value.length === 0) {
4195
+ this.clearSyncProcessKey(key);
4196
+ continue;
4197
+ }
4198
+
4199
+ const cursor =
4200
+ (this.syncInFlightQueueRoundRobinCursor.get(key) ?? 0) %
4201
+ value.length;
4202
+ const candidate = value[cursor]!;
4203
+ const publicKeyHash = candidate.hashcode();
4204
+ const inflightTimestamp = this.syncInFlight
4205
+ .get(publicKeyHash)
4206
+ ?.get(key)?.timestamp;
4207
+ if (
4208
+ inflightTimestamp == null ||
4209
+ now - inflightTimestamp >= SIMPLE_SYNC_RETRY_AFTER_MS
4210
+ ) {
4211
+ const epoch = this.syncDispatchTargetEpochs.get(publicKeyHash);
4212
+ if (!epoch) {
4213
+ continue;
4214
+ }
4215
+ let request = requestHashesByEpoch.get(epoch);
4216
+ if (!request) {
4217
+ request = { target: publicKeyHash, hashes: [] };
4218
+ requestHashesByEpoch.set(epoch, request);
4219
+ }
4220
+ request.hashes.push(key);
4221
+ if (value.length > 1) {
4222
+ this.syncInFlightQueueRoundRobinCursor.set(
4223
+ key,
4224
+ (cursor + 1) % value.length,
4225
+ );
4226
+ }
1260
4227
  }
4228
+ } else {
4229
+ this.clearSyncProcessKey(key);
1261
4230
  }
1262
- } else {
1263
- this.clearSyncProcessKey(key);
1264
4231
  }
1265
- }
4232
+ if (this.syncInFlightRetryRemaining === 0) {
4233
+ this.syncInFlightRetryIterator = undefined;
4234
+ }
1266
4235
 
1267
- const nowMin10s = +new Date() - 2e4;
1268
- for (const [key, map] of this.syncInFlight) {
1269
- // cleanup "old" missing syncs
1270
- for (const [hash, { timestamp }] of map) {
1271
- if (timestamp < nowMin10s) {
1272
- map.delete(hash);
4236
+ if (!isOpenLifecycleActive()) {
4237
+ return;
4238
+ }
4239
+ const nowMin10s = +new Date() - 2e4;
4240
+ for (const [target, map] of this.syncInFlight) {
4241
+ for (const [key, { timestamp }] of map) {
4242
+ if (timestamp < nowMin10s) {
4243
+ this.removeSyncInFlightTargetKey(target, key);
4244
+ }
1273
4245
  }
1274
4246
  }
1275
- if (map.size === 0) {
1276
- this.syncInFlight.delete(key);
4247
+ if (!isOpenLifecycleActive()) {
4248
+ return;
1277
4249
  }
1278
- }
1279
- this.requestSync(requestHashes, from).finally(() => {
1280
- if (this.closed) {
4250
+ for (const [epoch, request] of requestHashesByEpoch) {
4251
+ if (!isOpenLifecycleActive()) {
4252
+ return;
4253
+ }
4254
+ const requestHashes = this.filterDispatchablePendingSyncClaims(
4255
+ request.hashes,
4256
+ request.target,
4257
+ epoch,
4258
+ );
4259
+ if (requestHashes.length === 0) {
4260
+ continue;
4261
+ }
4262
+ try {
4263
+ await this.requestSync(requestHashes, [request.target], {
4264
+ ownershipLifecycleController: openLifecycleController,
4265
+ targetEpochs: new Map([[request.target, epoch]]),
4266
+ createTargetEpochs: false,
4267
+ });
4268
+ } catch {
4269
+ if (!isOpenLifecycleActive()) {
4270
+ return;
4271
+ }
4272
+ // A failed target must not prevent bounded retries for unrelated
4273
+ // peers in this pass.
4274
+ }
4275
+ }
4276
+ if (!isOpenLifecycleActive()) {
1281
4277
  return;
1282
4278
  }
1283
- this.syncMoreInterval = setTimeout(requestSyncLoop, 3e3);
1284
- });
4279
+ } catch {
4280
+ if (!isOpenLifecycleActive()) {
4281
+ return;
4282
+ }
4283
+ // Retry on the next interval; the rejection is deliberately observed.
4284
+ }
4285
+ scheduleRequestSyncLoop();
1285
4286
  };
1286
4287
 
1287
- requestSyncLoop();
4288
+ void requestSyncLoop().catch(() => {
4289
+ // Defensive observation for failures outside the loop's best-effort body.
4290
+ });
1288
4291
  }
1289
4292
 
1290
4293
  async close() {
1291
4294
  this.closed = true;
4295
+ this.syncDispatchLifecycleController.abort();
4296
+ this.syncDispatchTargetEpochs.clear();
4297
+ this.clearPendingSyncAdmissions();
1292
4298
  this.syncInFlightQueue.clear();
1293
4299
  this.syncInFlightQueueInverted.clear();
4300
+ this.syncInFlightRetryIterator = undefined;
4301
+ this.syncInFlightRetryRemaining = 0;
4302
+ this.syncInFlightQueueExpiresAt.clear();
4303
+ this.pendingSyncExpiryHeap.length = 0;
4304
+ this.pendingSyncKeyExpiryNodes.clear();
4305
+ this.pendingSyncAdmissionExpiryNodes.clear();
4306
+ this.syncInFlightQueueClaimants.clear();
4307
+ this.syncInFlightQueueClaimantIndexes.clear();
4308
+ this.syncInFlightQueueRoundRobinCursor.clear();
4309
+ this.syncInFlightQueuedCoordinates.clear();
4310
+ this.syncInFlightQueuedHashByCoordinate.clear();
4311
+ this.syncInFlightQueuedCoordinatesByHash.clear();
4312
+ this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
4313
+ this.pendingSyncClaimCount = 0;
4314
+ if (this.syncInFlightQueueExpiryTimer != null) {
4315
+ clearTimeout(this.syncInFlightQueueExpiryTimer);
4316
+ this.syncInFlightQueueExpiryTimer = undefined;
4317
+ }
1294
4318
  this.syncInFlight.clear();
4319
+ this.syncInFlightTargetsByKey.clear();
1295
4320
  this.recentlySentExchangeHeads.clear();
4321
+ this.clearPendingMaybeSyncResponses();
1296
4322
  for (const sessionId of [...this.repairSessions.keys()]) {
1297
4323
  this.finalizeRepairSession(sessionId, false);
1298
4324
  }
@@ -1306,6 +4332,9 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1306
4332
  if (hashes.length === 0 || !this.hasEntryAddedState()) {
1307
4333
  return;
1308
4334
  }
4335
+ for (const hash of hashes) {
4336
+ this.clearPendingSyncAdmissionIdentity(hash);
4337
+ }
1309
4338
  this.clearSyncProcesses(hashes);
1310
4339
  this.markRepairSessionResolvedHashes(hashes);
1311
4340
  }
@@ -1314,6 +4343,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1314
4343
  if (!this.hasEntryAddedState()) {
1315
4344
  return;
1316
4345
  }
4346
+ this.clearPendingSyncAdmissionIdentity(hash);
1317
4347
  this.clearSyncProcess(hash);
1318
4348
  this.markRepairSessionResolvedHash(hash);
1319
4349
  }
@@ -1340,6 +4370,7 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1340
4370
  return (
1341
4371
  this.syncInFlightQueue.size > 0 ||
1342
4372
  this.syncInFlightQueueInverted.size > 0 ||
4373
+ this.pendingSyncAdmissionCount > 0 ||
1343
4374
  this.syncInFlight.size > 0
1344
4375
  );
1345
4376
  }
@@ -1357,43 +4388,171 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1357
4388
  }
1358
4389
  }
1359
4390
 
4391
+ const trackedClaimants = this.syncInFlightQueueClaimants.get(key);
4392
+ this.pendingSyncClaimCount = Math.max(
4393
+ 0,
4394
+ this.pendingSyncClaimCount -
4395
+ (trackedClaimants?.size ?? inflight.length),
4396
+ );
1360
4397
  this.syncInFlightQueue.delete(key);
1361
4398
  }
4399
+ this.syncInFlightQueueClaimants.delete(key);
4400
+ this.syncInFlightQueueClaimantIndexes.delete(key);
4401
+ this.syncInFlightQueueRoundRobinCursor.delete(key);
4402
+ if (typeof key === "bigint") {
4403
+ this.removeQueuedSyncCoordinateAlias(key);
4404
+ }
4405
+ this.removePendingSyncKeyExpiry(key);
4406
+ this.syncInFlightQueueExpiresAt.delete(key);
4407
+ this.clearPendingSyncExpiryTimerIfIdle();
1362
4408
 
1363
4409
  this.clearSyncInFlightKey(key);
1364
4410
  }
1365
4411
 
1366
- private clearSyncInFlightKey(key: SyncableKey) {
1367
- for (const [peer, map] of this.syncInFlight) {
1368
- map.delete(key);
1369
- if (map.size === 0) {
1370
- this.syncInFlight.delete(peer);
4412
+ private removePendingSyncClaim(key: SyncableKey, peer: string): void {
4413
+ const inflight = this.syncInFlightQueue.get(key);
4414
+ if (!inflight) {
4415
+ return;
4416
+ }
4417
+ let claimants = this.syncInFlightQueueClaimants.get(key);
4418
+ let claimantIndexes = this.syncInFlightQueueClaimantIndexes.get(key);
4419
+ if (!claimants || !claimantIndexes) {
4420
+ claimants = new Set();
4421
+ claimantIndexes = new Map();
4422
+ for (let index = 0; index < inflight.length; index += 1) {
4423
+ const claimant = inflight[index]!.hashcode();
4424
+ claimants.add(claimant);
4425
+ claimantIndexes.set(claimant, index);
4426
+ }
4427
+ if (!this.syncInFlightQueueClaimants.has(key)) {
4428
+ this.pendingSyncClaimCount += claimants.size;
1371
4429
  }
4430
+ this.syncInFlightQueueClaimants.set(key, claimants);
4431
+ this.syncInFlightQueueClaimantIndexes.set(key, claimantIndexes);
4432
+ }
4433
+ const index = claimantIndexes.get(peer);
4434
+ if (index == null) {
4435
+ return;
4436
+ }
4437
+
4438
+ const lastIndex = inflight.length - 1;
4439
+ if (index !== lastIndex) {
4440
+ const lastClaimant = inflight[lastIndex]!;
4441
+ const lastClaimantHash = lastClaimant.hashcode();
4442
+ inflight[index] = lastClaimant;
4443
+ claimantIndexes.set(lastClaimantHash, index);
4444
+ }
4445
+ inflight.pop();
4446
+ claimantIndexes.delete(peer);
4447
+ claimants.delete(peer);
4448
+ this.pendingSyncClaimCount = Math.max(0, this.pendingSyncClaimCount - 1);
4449
+ const inverted = this.syncInFlightQueueInverted.get(peer);
4450
+ inverted?.delete(key);
4451
+ if (inverted?.size === 0) {
4452
+ this.syncInFlightQueueInverted.delete(peer);
4453
+ }
4454
+ if (inflight.length > 0) {
4455
+ const cursor = this.syncInFlightQueueRoundRobinCursor.get(key) ?? 0;
4456
+ this.syncInFlightQueueRoundRobinCursor.set(
4457
+ key,
4458
+ cursor === lastIndex
4459
+ ? index % inflight.length
4460
+ : cursor % inflight.length,
4461
+ );
4462
+ return;
1372
4463
  }
4464
+
4465
+ this.syncInFlightQueue.delete(key);
4466
+ this.syncInFlightQueueClaimants.delete(key);
4467
+ this.syncInFlightQueueClaimantIndexes.delete(key);
4468
+ this.syncInFlightQueueRoundRobinCursor.delete(key);
4469
+ if (typeof key === "bigint") {
4470
+ this.removeQueuedSyncCoordinateAlias(key);
4471
+ }
4472
+ this.removePendingSyncKeyExpiry(key);
4473
+ this.syncInFlightQueueExpiresAt.delete(key);
4474
+ this.clearSyncInFlightKey(key);
4475
+ this.clearPendingSyncExpiryTimerIfIdle();
1373
4476
  }
1374
4477
 
1375
- private forEachKnownAlias(
1376
- hash: string,
1377
- callback: (key: SyncableKey) => void,
4478
+ private removeSyncInFlightTargetKey(peer: string, key: SyncableKey): void {
4479
+ const map = this.syncInFlight.get(peer);
4480
+ if (!map?.delete(key)) {
4481
+ return;
4482
+ }
4483
+ if (map.size === 0) {
4484
+ this.syncInFlight.delete(peer);
4485
+ }
4486
+ const targets = this.syncInFlightTargetsByKey.get(key);
4487
+ targets?.delete(peer);
4488
+ if (targets?.size === 0) {
4489
+ this.syncInFlightTargetsByKey.delete(key);
4490
+ }
4491
+ }
4492
+
4493
+ private setSyncInFlightTargetKey(
4494
+ peer: string,
4495
+ key: SyncableKey,
4496
+ timestamp: number,
1378
4497
  ): void {
1379
- callback(hash);
1380
- if (this.syncInFlightQueue.size === 0 && this.syncInFlight.size === 0) {
4498
+ let map = this.syncInFlight.get(peer);
4499
+ if (!map) {
4500
+ map = new Map();
4501
+ this.syncInFlight.set(peer, map);
4502
+ }
4503
+ const existing = map.get(key);
4504
+ if (!existing || existing.timestamp < timestamp) {
4505
+ map.set(key, { timestamp });
4506
+ }
4507
+ let targets = this.syncInFlightTargetsByKey.get(key);
4508
+ if (!targets) {
4509
+ targets = new Set();
4510
+ this.syncInFlightTargetsByKey.set(key, targets);
4511
+ }
4512
+ targets.add(peer);
4513
+ }
4514
+
4515
+ private clearSyncInFlightTarget(peer: string): void {
4516
+ const map = this.syncInFlight.get(peer);
4517
+ if (!map) {
1381
4518
  return;
1382
4519
  }
1383
- for (const key of this.syncInFlightQueue.keys()) {
1384
- if (typeof key === "bigint" && this.coordinateToHash.get(key) === hash) {
1385
- callback(key);
4520
+ for (const key of map.keys()) {
4521
+ const targets = this.syncInFlightTargetsByKey.get(key);
4522
+ targets?.delete(peer);
4523
+ if (targets?.size === 0) {
4524
+ this.syncInFlightTargetsByKey.delete(key);
1386
4525
  }
1387
4526
  }
1388
- for (const map of this.syncInFlight.values()) {
1389
- for (const key of map.keys()) {
1390
- if (
1391
- typeof key === "bigint" &&
1392
- this.coordinateToHash.get(key) === hash
1393
- ) {
1394
- callback(key);
4527
+ this.syncInFlight.delete(peer);
4528
+ }
4529
+
4530
+ private clearSyncInFlightKey(key: SyncableKey) {
4531
+ const targets = this.syncInFlightTargetsByKey.get(key);
4532
+ if (!targets) {
4533
+ // Defensive compatibility for tests or integrations that seed the public
4534
+ // syncInFlight map directly. Internal writes always populate the index.
4535
+ for (const [peer, map] of this.syncInFlight) {
4536
+ if (map.has(key)) {
4537
+ this.removeSyncInFlightTargetKey(peer, key);
1395
4538
  }
1396
4539
  }
4540
+ return;
4541
+ }
4542
+ for (const peer of [...targets]) {
4543
+ this.removeSyncInFlightTargetKey(peer, key);
4544
+ }
4545
+ }
4546
+
4547
+ private forEachKnownAlias(
4548
+ hash: string,
4549
+ callback: (key: SyncableKey) => void,
4550
+ ): void {
4551
+ callback(hash);
4552
+ for (const coordinate of [
4553
+ ...(this.syncInFlightQueuedCoordinatesByHash.get(hash) ?? []),
4554
+ ]) {
4555
+ callback(coordinate);
1397
4556
  }
1398
4557
  }
1399
4558
 
@@ -1402,10 +4561,10 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1402
4561
  if (!map) {
1403
4562
  return;
1404
4563
  }
1405
- this.forEachKnownAlias(hash, (key) => map.delete(key));
1406
- if (map.size === 0) {
1407
- this.syncInFlight.delete(publicKeyHash);
1408
- }
4564
+ this.refreshQueuedSyncCoordinateAliases();
4565
+ this.forEachKnownAlias(hash, (key) =>
4566
+ this.removeSyncInFlightTargetKey(publicKeyHash, key),
4567
+ );
1409
4568
  }
1410
4569
 
1411
4570
  private clearSyncInFlightForPeerHashes(
@@ -1416,26 +4575,16 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1416
4575
  if (!map || hashes.length === 0) {
1417
4576
  return;
1418
4577
  }
1419
- const keys = new Set<SyncableKey>(hashes);
1420
- const hashSet = new Set(hashes);
1421
- for (const key of map.keys()) {
1422
- if (typeof key !== "bigint") {
1423
- continue;
1424
- }
1425
- const hash = this.coordinateToHash.get(key);
1426
- if (hash != null && hashSet.has(hash)) {
1427
- keys.add(key);
1428
- }
1429
- }
1430
- for (const key of keys) {
1431
- map.delete(key);
1432
- }
1433
- if (map.size === 0) {
1434
- this.syncInFlight.delete(publicKeyHash);
4578
+ this.refreshQueuedSyncCoordinateAliases();
4579
+ for (const hash of hashes) {
4580
+ this.forEachKnownAlias(hash, (key) =>
4581
+ this.removeSyncInFlightTargetKey(publicKeyHash, key),
4582
+ );
1435
4583
  }
1436
4584
  }
1437
4585
 
1438
4586
  private clearSyncProcess(hash: string) {
4587
+ this.refreshQueuedSyncCoordinateAliases();
1439
4588
  this.forEachKnownAlias(hash, (key) => this.clearSyncProcessKey(key));
1440
4589
  }
1441
4590
 
@@ -1443,24 +4592,10 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1443
4592
  if (hashes.length === 0) {
1444
4593
  return;
1445
4594
  }
1446
- const keys = new Set<SyncableKey>(hashes);
1447
- const hashSet = new Set(hashes);
1448
- const maybeAddAlias = (key: SyncableKey) => {
1449
- if (typeof key !== "bigint") {
1450
- return;
1451
- }
1452
- const hash = this.coordinateToHash.get(key);
1453
- if (hash != null && hashSet.has(hash)) {
1454
- keys.add(key);
1455
- }
1456
- };
1457
- for (const key of this.syncInFlightQueue.keys()) {
1458
- maybeAddAlias(key);
1459
- }
1460
- for (const map of this.syncInFlight.values()) {
1461
- for (const key of map.keys()) {
1462
- maybeAddAlias(key);
1463
- }
4595
+ this.refreshQueuedSyncCoordinateAliases();
4596
+ const keys = new Set<SyncableKey>();
4597
+ for (const hash of hashes) {
4598
+ this.forEachKnownAlias(hash, (key) => keys.add(key));
1464
4599
  }
1465
4600
  for (const key of keys) {
1466
4601
  this.clearSyncProcessKey(key);
@@ -1472,20 +4607,30 @@ export class SimpleSyncronizer<R extends "u32" | "u64">
1472
4607
  return this.clearSyncProcessPublicKeyHash(publicKeyHash);
1473
4608
  }
1474
4609
  private clearSyncProcessPublicKeyHash(publicKeyHash: string) {
1475
- this.syncInFlight.delete(publicKeyHash);
4610
+ this.syncDispatchTargetEpochs.delete(publicKeyHash);
4611
+ this.clearPendingSyncAdmissions(publicKeyHash);
4612
+ for (const targetLifecycle of [
4613
+ ...(this.syncDispatchTargets.get(publicKeyHash) ?? []),
4614
+ ]) {
4615
+ if (targetLifecycle.lifecycle.abortAllOnTargetDisconnect) {
4616
+ this.abortSyncDispatchLifecycle(
4617
+ targetLifecycle.lifecycle,
4618
+ new Error("sync target disconnected"),
4619
+ );
4620
+ } else {
4621
+ this.abortSyncDispatchTarget(
4622
+ targetLifecycle,
4623
+ new Error("sync target disconnected"),
4624
+ );
4625
+ }
4626
+ }
4627
+ this.clearSyncInFlightTarget(publicKeyHash);
1476
4628
  this.recentlySentExchangeHeads.delete(publicKeyHash);
4629
+ this.clearPendingMaybeSyncResponses(publicKeyHash);
1477
4630
  const map = this.syncInFlightQueueInverted.get(publicKeyHash);
1478
4631
  if (map) {
1479
- for (const hash of map) {
1480
- const arr = this.syncInFlightQueue.get(hash);
1481
- if (arr) {
1482
- const filtered = arr.filter((x) => x.hashcode() !== publicKeyHash);
1483
- if (filtered.length > 0) {
1484
- this.syncInFlightQueue.set(hash, filtered);
1485
- } else {
1486
- this.syncInFlightQueue.delete(hash);
1487
- }
1488
- }
4632
+ for (const hash of [...map]) {
4633
+ this.removePendingSyncClaim(hash, publicKeyHash);
1489
4634
  }
1490
4635
  this.syncInFlightQueueInverted.delete(publicKeyHash);
1491
4636
  }