@peerbit/shared-log 13.2.24 → 13.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/checked-prune.d.ts +12 -0
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +33 -0
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/coordinate-persistence.d.ts +211 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -0
- package/dist/src/coordinate-persistence.js +1133 -0
- package/dist/src/coordinate-persistence.js.map +1 -0
- package/dist/src/errors.d.ts +1 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +19 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +127 -133
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1400 -4269
- package/dist/src/index.js.map +1 -1
- package/dist/src/instance-lifecycle.d.ts +117 -0
- package/dist/src/instance-lifecycle.d.ts.map +1 -0
- package/dist/src/instance-lifecycle.js +254 -0
- package/dist/src/instance-lifecycle.js.map +1 -0
- package/dist/src/join-warmup.d.ts +86 -0
- package/dist/src/join-warmup.d.ts.map +1 -0
- package/dist/src/join-warmup.js +351 -0
- package/dist/src/join-warmup.js.map +1 -0
- package/dist/src/native-write-through-block-store.d.ts +123 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -0
- package/dist/src/native-write-through-block-store.js +989 -0
- package/dist/src/native-write-through-block-store.js.map +1 -0
- package/dist/src/peer-session.d.ts +117 -0
- package/dist/src/peer-session.d.ts.map +1 -0
- package/dist/src/peer-session.js +259 -0
- package/dist/src/peer-session.js.map +1 -0
- package/dist/src/replication-announcement.d.ts +116 -0
- package/dist/src/replication-announcement.d.ts.map +1 -0
- package/dist/src/replication-announcement.js +515 -0
- package/dist/src/replication-announcement.js.map +1 -0
- package/dist/src/replicator-liveness.d.ts +59 -0
- package/dist/src/replicator-liveness.d.ts.map +1 -0
- package/dist/src/replicator-liveness.js +304 -0
- package/dist/src/replicator-liveness.js.map +1 -0
- package/dist/src/sync/dispatch-lifecycle.d.ts +48 -0
- package/dist/src/sync/dispatch-lifecycle.d.ts.map +1 -0
- package/dist/src/sync/dispatch-lifecycle.js +141 -0
- package/dist/src/sync/dispatch-lifecycle.js.map +1 -0
- package/dist/src/sync/factory.d.ts +29 -0
- package/dist/src/sync/factory.d.ts.map +1 -0
- package/dist/src/sync/factory.js +87 -0
- package/dist/src/sync/factory.js.map +1 -0
- package/dist/src/sync/pending-sync-store.d.ts +105 -0
- package/dist/src/sync/pending-sync-store.d.ts.map +1 -0
- package/dist/src/sync/pending-sync-store.js +877 -0
- package/dist/src/sync/pending-sync-store.js.map +1 -0
- package/dist/src/sync/rateless-iblt.d.ts +4 -1
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +102 -106
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +10 -41
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +259 -952
- package/dist/src/sync/simple.js.map +1 -1
- package/dist/src/sync/sync-peer-state.d.ts +16 -0
- package/dist/src/sync/sync-peer-state.d.ts.map +1 -0
- package/dist/src/sync/sync-peer-state.js +82 -0
- package/dist/src/sync/sync-peer-state.js.map +1 -0
- package/package.json +12 -12
- package/src/checked-prune.ts +35 -0
- package/src/coordinate-persistence.ts +1796 -0
- package/src/errors.ts +21 -0
- package/src/index.ts +4014 -7858
- package/src/instance-lifecycle.ts +351 -0
- package/src/join-warmup.ts +507 -0
- package/src/native-write-through-block-store.ts +1170 -0
- package/src/peer-session.ts +336 -0
- package/src/replication-announcement.ts +744 -0
- package/src/replicator-liveness.ts +439 -0
- package/src/sync/dispatch-lifecycle.ts +230 -0
- package/src/sync/factory.ts +154 -0
- package/src/sync/pending-sync-store.ts +1085 -0
- package/src/sync/rateless-iblt.ts +122 -132
- package/src/sync/simple.ts +334 -1144
- package/src/sync/sync-peer-state.ts +107 -0
|
@@ -0,0 +1,1085 @@
|
|
|
1
|
+
import type { Cache } from "@peerbit/cache";
|
|
2
|
+
import type { PublicSignKey } from "@peerbit/crypto";
|
|
3
|
+
import type { SyncableKey } from "./index.js";
|
|
4
|
+
|
|
5
|
+
// The keyed record store for pending simple-sync claims (stage-4 sync
|
|
6
|
+
// unification). One record per retained key replaces the former lockstep
|
|
7
|
+
// map family (claimants, claimant indexes, round-robin cursors and key
|
|
8
|
+
// expiry nodes); the expiry heap holds the records themselves, unioned with
|
|
9
|
+
// admission-reservation expiry nodes. The public queue maps survive as the
|
|
10
|
+
// store's physical indexes — the SAME Map instances tests and integrations
|
|
11
|
+
// have always seeded and observed — and directly seeded keys hydrate into
|
|
12
|
+
// records on first access, exactly like the former defensive branches.
|
|
13
|
+
//
|
|
14
|
+
// Lifetimes: a record exists while its key is retained (one add path:
|
|
15
|
+
// addPendingSyncClaim; unified removal: clearSyncProcessKey /
|
|
16
|
+
// removePendingSyncClaim / TTL expiry / close). Admission reservations keep
|
|
17
|
+
// their quota charged until their originating lookup settles — invalidation
|
|
18
|
+
// (expiry/disconnect/close) deliberately does not return the quota early.
|
|
19
|
+
|
|
20
|
+
// Late coordinate-to-hash cache fills are discovered incrementally. Keep this
|
|
21
|
+
// independent of retained queue size so an empty or repeated request cannot
|
|
22
|
+
// force an O(global pending keys) reverse-alias rebuild.
|
|
23
|
+
const MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE = 128;
|
|
24
|
+
export const QUEUED_SYNC_ALIAS_REFRESH_PENDING = Symbol(
|
|
25
|
+
"queued-sync-alias-refresh-pending",
|
|
26
|
+
);
|
|
27
|
+
// This is an absolute first-seen lifetime. Repeated claims and additional peers
|
|
28
|
+
// deliberately do not slide the deadline.
|
|
29
|
+
export const PENDING_SIMPLE_SYNC_KEY_TTL_MS = 60_000;
|
|
30
|
+
|
|
31
|
+
export type PendingSyncAdmissionReservation = {
|
|
32
|
+
peer: string;
|
|
33
|
+
remaining: number;
|
|
34
|
+
active: boolean;
|
|
35
|
+
released: boolean;
|
|
36
|
+
expiresAt: number;
|
|
37
|
+
identities: Set<SyncableKey>;
|
|
38
|
+
retainedSettled: number;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export type PendingSyncRecord = {
|
|
42
|
+
kind: "key";
|
|
43
|
+
key: SyncableKey;
|
|
44
|
+
peers: PublicSignKey[];
|
|
45
|
+
claimants: Set<string>;
|
|
46
|
+
claimantIndexes: Map<string, number>;
|
|
47
|
+
rrCursor: number;
|
|
48
|
+
expiresAt: number;
|
|
49
|
+
heapIndex: number;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
type PendingSyncAdmissionExpiryNode = {
|
|
53
|
+
kind: "admission";
|
|
54
|
+
reservation: PendingSyncAdmissionReservation;
|
|
55
|
+
expiresAt: number;
|
|
56
|
+
heapIndex: number;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
type PendingSyncExpiryNode = PendingSyncRecord | PendingSyncAdmissionExpiryNode;
|
|
60
|
+
|
|
61
|
+
export type PendingSyncStoreDeps = {
|
|
62
|
+
coordinateToHash: Cache<string>;
|
|
63
|
+
isDispatchEpochCurrent: (peer: string, epoch: unknown) => boolean;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
export class PendingSyncStore {
|
|
67
|
+
// map of hash to public keys that we can ask for entries. The public
|
|
68
|
+
// legacy view instances: the same Maps the synchronizer has always
|
|
69
|
+
// exposed, kept as the store's physical indexes.
|
|
70
|
+
syncInFlightQueue: Map<SyncableKey, PublicSignKey[]>;
|
|
71
|
+
syncInFlightQueueInverted: Map<string, Set<SyncableKey>>;
|
|
72
|
+
syncInFlightQueueExpiresAt: Map<SyncableKey, number>;
|
|
73
|
+
records: Map<SyncableKey, PendingSyncRecord>;
|
|
74
|
+
pendingSyncExpiryHeap: PendingSyncExpiryNode[];
|
|
75
|
+
syncInFlightQueueExpiryTimer?: ReturnType<typeof setTimeout>;
|
|
76
|
+
pendingSyncAdmissionExpiryNodes: Map<
|
|
77
|
+
PendingSyncAdmissionReservation,
|
|
78
|
+
PendingSyncAdmissionExpiryNode
|
|
79
|
+
>;
|
|
80
|
+
syncInFlightQueuedCoordinates: Set<bigint>;
|
|
81
|
+
syncInFlightQueuedHashByCoordinate: Map<bigint, string>;
|
|
82
|
+
syncInFlightQueuedCoordinatesByHash: Map<string, Set<bigint>>;
|
|
83
|
+
syncInFlightQueuedCoordinateRefreshIterator?: IterableIterator<bigint>;
|
|
84
|
+
pendingSyncClaimCount: number;
|
|
85
|
+
pendingSyncAdmissionCount: number;
|
|
86
|
+
pendingSyncActiveAdmissionReservations: number;
|
|
87
|
+
pendingSyncAdmissionCountByPeer: Map<string, number>;
|
|
88
|
+
pendingSyncAdmissionIdentitiesByPeer: Map<string, Set<SyncableKey>>;
|
|
89
|
+
pendingSyncAdmissionReservations: Set<PendingSyncAdmissionReservation>;
|
|
90
|
+
pendingSyncAdmissionReservationsByPeer: Map<
|
|
91
|
+
string,
|
|
92
|
+
Set<PendingSyncAdmissionReservation>
|
|
93
|
+
>;
|
|
94
|
+
pendingSyncAdmissionReservationsByIdentity: Map<
|
|
95
|
+
SyncableKey,
|
|
96
|
+
Set<PendingSyncAdmissionReservation>
|
|
97
|
+
>;
|
|
98
|
+
|
|
99
|
+
// map of hash to public keys that we have asked for entries
|
|
100
|
+
syncInFlight: Map<string, Map<SyncableKey, { timestamp: number }>>;
|
|
101
|
+
syncInFlightTargetsByKey: Map<SyncableKey, Set<string>>;
|
|
102
|
+
|
|
103
|
+
constructor(private readonly deps: PendingSyncStoreDeps) {
|
|
104
|
+
this.syncInFlightQueue = new Map();
|
|
105
|
+
this.syncInFlightQueueInverted = new Map();
|
|
106
|
+
this.syncInFlightQueueExpiresAt = new Map();
|
|
107
|
+
this.records = new Map();
|
|
108
|
+
this.pendingSyncExpiryHeap = [];
|
|
109
|
+
this.pendingSyncAdmissionExpiryNodes = new Map();
|
|
110
|
+
this.syncInFlightQueuedCoordinates = new Set();
|
|
111
|
+
this.syncInFlightQueuedHashByCoordinate = new Map();
|
|
112
|
+
this.syncInFlightQueuedCoordinatesByHash = new Map();
|
|
113
|
+
this.pendingSyncClaimCount = 0;
|
|
114
|
+
this.pendingSyncAdmissionCount = 0;
|
|
115
|
+
this.pendingSyncActiveAdmissionReservations = 0;
|
|
116
|
+
this.pendingSyncAdmissionCountByPeer = new Map();
|
|
117
|
+
this.pendingSyncAdmissionIdentitiesByPeer = new Map();
|
|
118
|
+
this.pendingSyncAdmissionReservations = new Set();
|
|
119
|
+
this.pendingSyncAdmissionReservationsByPeer = new Map();
|
|
120
|
+
this.pendingSyncAdmissionReservationsByIdentity = new Map();
|
|
121
|
+
this.syncInFlight = new Map();
|
|
122
|
+
this.syncInFlightTargetsByKey = new Map();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Defensive compatibility for callers/tests that seed the public queue
|
|
126
|
+
// maps directly. Internally every retained key gets its record at
|
|
127
|
+
// creation; a seeded key hydrates here (counting its claimants) and never
|
|
128
|
+
// enters the expiry heap, exactly like the former hydration branches.
|
|
129
|
+
private hydrateRecordFromQueue(
|
|
130
|
+
key: SyncableKey,
|
|
131
|
+
peers: PublicSignKey[],
|
|
132
|
+
): PendingSyncRecord {
|
|
133
|
+
const claimants = new Set(peers.map((peer) => peer.hashcode()));
|
|
134
|
+
const claimantIndexes = new Map<string, number>();
|
|
135
|
+
for (let index = 0; index < peers.length; index += 1) {
|
|
136
|
+
claimantIndexes.set(peers[index]!.hashcode(), index);
|
|
137
|
+
}
|
|
138
|
+
const record: PendingSyncRecord = {
|
|
139
|
+
kind: "key",
|
|
140
|
+
key,
|
|
141
|
+
peers,
|
|
142
|
+
claimants,
|
|
143
|
+
claimantIndexes,
|
|
144
|
+
rrCursor: 0,
|
|
145
|
+
expiresAt:
|
|
146
|
+
this.syncInFlightQueueExpiresAt.get(key) ?? Number.POSITIVE_INFINITY,
|
|
147
|
+
heapIndex: -1,
|
|
148
|
+
};
|
|
149
|
+
this.records.set(key, record);
|
|
150
|
+
this.pendingSyncClaimCount += claimants.size;
|
|
151
|
+
return record;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
getPendingSyncKeyIdentity(key: SyncableKey): SyncableKey {
|
|
155
|
+
if (typeof key === "string") {
|
|
156
|
+
return key;
|
|
157
|
+
}
|
|
158
|
+
const hash = this.deps.coordinateToHash.get(key);
|
|
159
|
+
return hash ?? key;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private removeQueuedSyncCoordinateAlias(key: bigint): void {
|
|
163
|
+
this.syncInFlightQueuedCoordinates.delete(key);
|
|
164
|
+
if (this.syncInFlightQueuedCoordinates.size === 0) {
|
|
165
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
166
|
+
}
|
|
167
|
+
const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
|
|
168
|
+
this.syncInFlightQueuedHashByCoordinate.delete(key);
|
|
169
|
+
if (previousHash != null) {
|
|
170
|
+
const coordinates =
|
|
171
|
+
this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
|
|
172
|
+
coordinates?.delete(key);
|
|
173
|
+
if (coordinates?.size === 0) {
|
|
174
|
+
this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
private refreshQueuedSyncCoordinateAlias(key: bigint): void {
|
|
180
|
+
if (!this.syncInFlightQueue.has(key)) {
|
|
181
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
this.syncInFlightQueuedCoordinates.add(key);
|
|
185
|
+
const hash = this.deps.coordinateToHash.get(key) ?? undefined;
|
|
186
|
+
const previousHash = this.syncInFlightQueuedHashByCoordinate.get(key);
|
|
187
|
+
if (previousHash !== hash) {
|
|
188
|
+
if (previousHash != null) {
|
|
189
|
+
const coordinates =
|
|
190
|
+
this.syncInFlightQueuedCoordinatesByHash.get(previousHash);
|
|
191
|
+
coordinates?.delete(key);
|
|
192
|
+
if (coordinates?.size === 0) {
|
|
193
|
+
this.syncInFlightQueuedCoordinatesByHash.delete(previousHash);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (hash == null) {
|
|
197
|
+
this.syncInFlightQueuedHashByCoordinate.delete(key);
|
|
198
|
+
} else {
|
|
199
|
+
this.syncInFlightQueuedHashByCoordinate.set(key, hash);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (hash != null) {
|
|
203
|
+
let coordinates = this.syncInFlightQueuedCoordinatesByHash.get(hash);
|
|
204
|
+
if (!coordinates) {
|
|
205
|
+
coordinates = new Set();
|
|
206
|
+
this.syncInFlightQueuedCoordinatesByHash.set(hash, coordinates);
|
|
207
|
+
}
|
|
208
|
+
coordinates.add(key);
|
|
209
|
+
this.reconcileQueuedSyncCoordinateAlias(key, hash);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private reconcileQueuedSyncCoordinateAlias(
|
|
214
|
+
coordinate: bigint,
|
|
215
|
+
hash: string,
|
|
216
|
+
): void {
|
|
217
|
+
const now = Date.now();
|
|
218
|
+
const coordinateExpiresAt = this.syncInFlightQueueExpiresAt.get(coordinate);
|
|
219
|
+
if (coordinateExpiresAt != null && coordinateExpiresAt <= now) {
|
|
220
|
+
this.clearSyncProcessKey(coordinate);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const hashExpiresAt = this.syncInFlightQueueExpiresAt.get(hash);
|
|
224
|
+
if (hashExpiresAt != null && hashExpiresAt <= now) {
|
|
225
|
+
this.clearSyncProcessKey(hash);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const hashClaimants = this.syncInFlightQueue.get(hash);
|
|
229
|
+
if (!hashClaimants || !this.syncInFlightQueue.has(coordinate)) {
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const expiresAt = Math.min(
|
|
233
|
+
this.syncInFlightQueueExpiresAt.get(coordinate) ?? Infinity,
|
|
234
|
+
this.syncInFlightQueueExpiresAt.get(hash) ?? Infinity,
|
|
235
|
+
);
|
|
236
|
+
for (const claimant of [...hashClaimants]) {
|
|
237
|
+
this.addPendingSyncClaim(coordinate, claimant, expiresAt);
|
|
238
|
+
}
|
|
239
|
+
if (Number.isFinite(expiresAt)) {
|
|
240
|
+
this.movePendingSyncKeyExpiryEarlier(coordinate, expiresAt);
|
|
241
|
+
}
|
|
242
|
+
for (const target of [...(this.syncInFlightTargetsByKey.get(hash) ?? [])]) {
|
|
243
|
+
const state = this.syncInFlight.get(target)?.get(hash);
|
|
244
|
+
if (state) {
|
|
245
|
+
this.setSyncInFlightTargetKey(target, coordinate, state.timestamp);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
this.clearSyncProcessKey(hash);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
refreshQueuedSyncCoordinateAliases(): void {
|
|
252
|
+
if (
|
|
253
|
+
this.syncInFlightQueuedCoordinates.size === 0 &&
|
|
254
|
+
this.records.size < this.syncInFlightQueue.size
|
|
255
|
+
) {
|
|
256
|
+
// Defensive compatibility for callers/tests that seed the public queue
|
|
257
|
+
// directly. Keep hydration bounded; internal writes register coordinates
|
|
258
|
+
// when the key is first admitted.
|
|
259
|
+
let inspected = 0;
|
|
260
|
+
for (const key of this.syncInFlightQueue.keys()) {
|
|
261
|
+
if (typeof key === "bigint") {
|
|
262
|
+
this.syncInFlightQueuedCoordinates.add(key);
|
|
263
|
+
}
|
|
264
|
+
inspected += 1;
|
|
265
|
+
if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (this.syncInFlightQueuedCoordinates.size === 0) {
|
|
271
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
this.syncInFlightQueuedCoordinateRefreshIterator ??=
|
|
275
|
+
this.syncInFlightQueuedCoordinates.values();
|
|
276
|
+
for (
|
|
277
|
+
let refreshed = 0;
|
|
278
|
+
refreshed < MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE;
|
|
279
|
+
refreshed += 1
|
|
280
|
+
) {
|
|
281
|
+
const next = this.syncInFlightQueuedCoordinateRefreshIterator.next();
|
|
282
|
+
if (next.done) {
|
|
283
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
this.refreshQueuedSyncCoordinateAlias(next.value);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
getQueuedSyncKeyForAdmission(
|
|
291
|
+
key: SyncableKey,
|
|
292
|
+
): SyncableKey | typeof QUEUED_SYNC_ALIAS_REFRESH_PENDING | undefined {
|
|
293
|
+
const getValidQueuedKey = (
|
|
294
|
+
candidate: SyncableKey,
|
|
295
|
+
): SyncableKey | undefined => {
|
|
296
|
+
if (!this.syncInFlightQueue.has(candidate)) {
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
const expiresAt = this.syncInFlightQueueExpiresAt.get(candidate);
|
|
300
|
+
if (expiresAt != null && expiresAt <= Date.now()) {
|
|
301
|
+
this.clearSyncProcessKey(candidate);
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
304
|
+
return candidate;
|
|
305
|
+
};
|
|
306
|
+
if (getValidQueuedKey(key) != null) {
|
|
307
|
+
if (typeof key === "bigint") {
|
|
308
|
+
this.refreshQueuedSyncCoordinateAlias(key);
|
|
309
|
+
return getValidQueuedKey(key);
|
|
310
|
+
}
|
|
311
|
+
return key;
|
|
312
|
+
}
|
|
313
|
+
if (typeof key === "string") {
|
|
314
|
+
const aliases = this.syncInFlightQueuedCoordinatesByHash.get(key);
|
|
315
|
+
if (aliases) {
|
|
316
|
+
let inspected = 0;
|
|
317
|
+
for (const alias of aliases) {
|
|
318
|
+
if (inspected >= MAX_PENDING_SIMPLE_SYNC_ALIAS_REFRESH_PER_MESSAGE) {
|
|
319
|
+
return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
|
|
320
|
+
}
|
|
321
|
+
inspected += 1;
|
|
322
|
+
this.refreshQueuedSyncCoordinateAlias(alias);
|
|
323
|
+
if (
|
|
324
|
+
this.syncInFlightQueuedCoordinatesByHash.get(key)?.has(alias) ===
|
|
325
|
+
true
|
|
326
|
+
) {
|
|
327
|
+
const validAlias = getValidQueuedKey(alias);
|
|
328
|
+
if (validAlias != null) {
|
|
329
|
+
return validAlias;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (
|
|
334
|
+
(this.syncInFlightQueuedCoordinatesByHash.get(key)?.size ?? 0) > 0
|
|
335
|
+
) {
|
|
336
|
+
return QUEUED_SYNC_ALIAS_REFRESH_PENDING;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
return undefined;
|
|
340
|
+
}
|
|
341
|
+
const hash = this.deps.coordinateToHash.get(key);
|
|
342
|
+
return hash != null ? getValidQueuedKey(hash) : undefined;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
addPendingSyncClaim(
|
|
346
|
+
key: SyncableKey,
|
|
347
|
+
from: PublicSignKey,
|
|
348
|
+
expiresAt?: number,
|
|
349
|
+
): boolean {
|
|
350
|
+
const fromHash = from.hashcode();
|
|
351
|
+
let peers = this.syncInFlightQueue.get(key);
|
|
352
|
+
let record = this.records.get(key);
|
|
353
|
+
if (!peers) {
|
|
354
|
+
peers = [];
|
|
355
|
+
this.syncInFlightQueue.set(key, peers);
|
|
356
|
+
const deadline = expiresAt ?? Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS;
|
|
357
|
+
this.syncInFlightQueueExpiresAt.set(key, deadline);
|
|
358
|
+
record = {
|
|
359
|
+
kind: "key",
|
|
360
|
+
key,
|
|
361
|
+
peers,
|
|
362
|
+
claimants: new Set(),
|
|
363
|
+
claimantIndexes: new Map(),
|
|
364
|
+
rrCursor: 0,
|
|
365
|
+
expiresAt: deadline,
|
|
366
|
+
heapIndex: -1,
|
|
367
|
+
};
|
|
368
|
+
this.records.set(key, record);
|
|
369
|
+
this.pushPendingSyncExpiry(record);
|
|
370
|
+
if (typeof key === "bigint") {
|
|
371
|
+
this.refreshQueuedSyncCoordinateAlias(key);
|
|
372
|
+
}
|
|
373
|
+
} else if (!record) {
|
|
374
|
+
record = this.hydrateRecordFromQueue(key, peers);
|
|
375
|
+
}
|
|
376
|
+
if (record.claimants.has(fromHash)) {
|
|
377
|
+
return false;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
record.claimantIndexes.set(fromHash, peers.length);
|
|
381
|
+
peers.push(from);
|
|
382
|
+
record.claimants.add(fromHash);
|
|
383
|
+
let inverted = this.syncInFlightQueueInverted.get(fromHash);
|
|
384
|
+
if (!inverted) {
|
|
385
|
+
inverted = new Set();
|
|
386
|
+
this.syncInFlightQueueInverted.set(fromHash, inverted);
|
|
387
|
+
}
|
|
388
|
+
inverted.add(key);
|
|
389
|
+
this.pendingSyncClaimCount += 1;
|
|
390
|
+
this.schedulePendingSyncKeyExpiry();
|
|
391
|
+
return true;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
hasPendingSyncClaim(key: SyncableKey, peer: string): boolean {
|
|
395
|
+
const record = this.records.get(key);
|
|
396
|
+
if (record) {
|
|
397
|
+
return record.claimants.has(peer);
|
|
398
|
+
}
|
|
399
|
+
const peers = this.syncInFlightQueue.get(key);
|
|
400
|
+
if (!peers) {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
return this.hydrateRecordFromQueue(key, peers).claimants.has(peer);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
filterDispatchablePendingSyncClaims(
|
|
407
|
+
keys: SyncableKey[],
|
|
408
|
+
peer: string,
|
|
409
|
+
epoch: unknown,
|
|
410
|
+
): SyncableKey[] {
|
|
411
|
+
if (!this.deps.isDispatchEpochCurrent(peer, epoch)) {
|
|
412
|
+
return [];
|
|
413
|
+
}
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
const dispatchable: SyncableKey[] = [];
|
|
416
|
+
for (const key of keys) {
|
|
417
|
+
const expiresAt = this.syncInFlightQueueExpiresAt.get(key);
|
|
418
|
+
if (expiresAt != null && expiresAt <= now) {
|
|
419
|
+
this.clearSyncProcessKey(key);
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
if (
|
|
423
|
+
this.syncInFlightQueue.has(key) &&
|
|
424
|
+
this.hasPendingSyncClaim(key, peer)
|
|
425
|
+
) {
|
|
426
|
+
dispatchable.push(key);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return dispatchable;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
getRoundRobinCursor(key: SyncableKey): number {
|
|
433
|
+
return this.records.get(key)?.rrCursor ?? 0;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
setRoundRobinCursor(key: SyncableKey, cursor: number): void {
|
|
437
|
+
const peers = this.syncInFlightQueue.get(key);
|
|
438
|
+
if (!peers) {
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
const record =
|
|
442
|
+
this.records.get(key) ?? this.hydrateRecordFromQueue(key, peers);
|
|
443
|
+
record.rrCursor = cursor;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
reservePendingSyncAdmission(
|
|
447
|
+
peer: string,
|
|
448
|
+
identities: SyncableKey[],
|
|
449
|
+
): PendingSyncAdmissionReservation | undefined {
|
|
450
|
+
const count = identities.length;
|
|
451
|
+
if (count <= 0) {
|
|
452
|
+
return undefined;
|
|
453
|
+
}
|
|
454
|
+
const reservation: PendingSyncAdmissionReservation = {
|
|
455
|
+
peer,
|
|
456
|
+
remaining: count,
|
|
457
|
+
active: true,
|
|
458
|
+
released: false,
|
|
459
|
+
expiresAt: Date.now() + PENDING_SIMPLE_SYNC_KEY_TTL_MS,
|
|
460
|
+
identities: new Set(identities),
|
|
461
|
+
retainedSettled: 0,
|
|
462
|
+
};
|
|
463
|
+
this.pendingSyncAdmissionReservations.add(reservation);
|
|
464
|
+
let peerReservations =
|
|
465
|
+
this.pendingSyncAdmissionReservationsByPeer.get(peer);
|
|
466
|
+
if (!peerReservations) {
|
|
467
|
+
peerReservations = new Set();
|
|
468
|
+
this.pendingSyncAdmissionReservationsByPeer.set(peer, peerReservations);
|
|
469
|
+
}
|
|
470
|
+
peerReservations.add(reservation);
|
|
471
|
+
this.pendingSyncActiveAdmissionReservations += 1;
|
|
472
|
+
this.pendingSyncAdmissionCount += count;
|
|
473
|
+
this.pendingSyncAdmissionCountByPeer.set(
|
|
474
|
+
peer,
|
|
475
|
+
(this.pendingSyncAdmissionCountByPeer.get(peer) ?? 0) + count,
|
|
476
|
+
);
|
|
477
|
+
let reservedIdentities =
|
|
478
|
+
this.pendingSyncAdmissionIdentitiesByPeer.get(peer);
|
|
479
|
+
if (!reservedIdentities) {
|
|
480
|
+
reservedIdentities = new Set();
|
|
481
|
+
this.pendingSyncAdmissionIdentitiesByPeer.set(peer, reservedIdentities);
|
|
482
|
+
}
|
|
483
|
+
for (const identity of identities) {
|
|
484
|
+
reservedIdentities.add(identity);
|
|
485
|
+
let reservationsForIdentity =
|
|
486
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
487
|
+
if (!reservationsForIdentity) {
|
|
488
|
+
reservationsForIdentity = new Set();
|
|
489
|
+
this.pendingSyncAdmissionReservationsByIdentity.set(
|
|
490
|
+
identity,
|
|
491
|
+
reservationsForIdentity,
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
reservationsForIdentity.add(reservation);
|
|
495
|
+
}
|
|
496
|
+
const expiryNode: PendingSyncAdmissionExpiryNode = {
|
|
497
|
+
kind: "admission",
|
|
498
|
+
reservation,
|
|
499
|
+
expiresAt: reservation.expiresAt,
|
|
500
|
+
heapIndex: -1,
|
|
501
|
+
};
|
|
502
|
+
this.pendingSyncAdmissionExpiryNodes.set(reservation, expiryNode);
|
|
503
|
+
this.pushPendingSyncExpiry(expiryNode);
|
|
504
|
+
this.schedulePendingSyncKeyExpiry();
|
|
505
|
+
return reservation;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
private removePendingSyncAdmissionIdentity(
|
|
509
|
+
reservation: PendingSyncAdmissionReservation,
|
|
510
|
+
identity: SyncableKey,
|
|
511
|
+
options?: { retainQuota?: boolean },
|
|
512
|
+
): boolean {
|
|
513
|
+
if (!reservation.identities.delete(identity)) {
|
|
514
|
+
return false;
|
|
515
|
+
}
|
|
516
|
+
if (options?.retainQuota === true) {
|
|
517
|
+
reservation.retainedSettled += 1;
|
|
518
|
+
} else {
|
|
519
|
+
reservation.remaining -= 1;
|
|
520
|
+
this.pendingSyncAdmissionCount -= 1;
|
|
521
|
+
const peerCount =
|
|
522
|
+
(this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) - 1;
|
|
523
|
+
if (peerCount === 0) {
|
|
524
|
+
this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
|
|
525
|
+
} else {
|
|
526
|
+
this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const reservedIdentities = this.pendingSyncAdmissionIdentitiesByPeer.get(
|
|
530
|
+
reservation.peer,
|
|
531
|
+
);
|
|
532
|
+
reservedIdentities?.delete(identity);
|
|
533
|
+
if (reservedIdentities?.size === 0) {
|
|
534
|
+
this.pendingSyncAdmissionIdentitiesByPeer.delete(reservation.peer);
|
|
535
|
+
}
|
|
536
|
+
const reservationsForIdentity =
|
|
537
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
538
|
+
reservationsForIdentity?.delete(reservation);
|
|
539
|
+
if (reservationsForIdentity?.size === 0) {
|
|
540
|
+
this.pendingSyncAdmissionReservationsByIdentity.delete(identity);
|
|
541
|
+
}
|
|
542
|
+
return true;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
clearPendingSyncAdmissionIdentity(identity: SyncableKey): void {
|
|
546
|
+
const reservations =
|
|
547
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
548
|
+
if (!reservations) {
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
for (const reservation of [...reservations]) {
|
|
552
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity, {
|
|
553
|
+
retainQuota: true,
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
consumePendingSyncAdmission(
|
|
559
|
+
reservation: PendingSyncAdmissionReservation,
|
|
560
|
+
identity: SyncableKey,
|
|
561
|
+
): "consumed" | "settled" | "invalid" {
|
|
562
|
+
if (!reservation.active || reservation.expiresAt <= Date.now()) {
|
|
563
|
+
if (reservation.expiresAt <= Date.now()) {
|
|
564
|
+
this.invalidatePendingSyncAdmission(reservation);
|
|
565
|
+
}
|
|
566
|
+
return "invalid";
|
|
567
|
+
}
|
|
568
|
+
if (!reservation.identities.has(identity)) {
|
|
569
|
+
return "settled";
|
|
570
|
+
}
|
|
571
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity);
|
|
572
|
+
if (reservation.remaining === 0) {
|
|
573
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
574
|
+
reservation.active = false;
|
|
575
|
+
reservation.released = true;
|
|
576
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
577
|
+
this.pendingSyncAdmissionReservations.delete(reservation);
|
|
578
|
+
const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
|
|
579
|
+
reservation.peer,
|
|
580
|
+
);
|
|
581
|
+
peerReservations?.delete(reservation);
|
|
582
|
+
if (peerReservations?.size === 0) {
|
|
583
|
+
this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
|
|
584
|
+
}
|
|
585
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
586
|
+
}
|
|
587
|
+
return "consumed";
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
transferPendingSyncAdmissionIdentity(
|
|
591
|
+
peer: string,
|
|
592
|
+
identity: SyncableKey,
|
|
593
|
+
): number | undefined {
|
|
594
|
+
const reservations =
|
|
595
|
+
this.pendingSyncAdmissionReservationsByIdentity.get(identity);
|
|
596
|
+
if (!reservations) {
|
|
597
|
+
return undefined;
|
|
598
|
+
}
|
|
599
|
+
for (const reservation of reservations) {
|
|
600
|
+
if (
|
|
601
|
+
reservation.peer !== peer ||
|
|
602
|
+
reservation.released ||
|
|
603
|
+
reservation.expiresAt <= Date.now() ||
|
|
604
|
+
!this.removePendingSyncAdmissionIdentity(reservation, identity, {
|
|
605
|
+
retainQuota: true,
|
|
606
|
+
})
|
|
607
|
+
) {
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
// The original resolver may be non-abortable and still retains its input
|
|
611
|
+
// arrays. Keep that reservation charged until its queueSync finally
|
|
612
|
+
// settles; the queued claim is counted separately and can disappear
|
|
613
|
+
// without returning the resolver's quota early.
|
|
614
|
+
return reservation.expiresAt;
|
|
615
|
+
}
|
|
616
|
+
return undefined;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
invalidatePendingSyncAdmission(
|
|
620
|
+
reservation?: PendingSyncAdmissionReservation,
|
|
621
|
+
): void {
|
|
622
|
+
if (!reservation || reservation.released || !reservation.active) {
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
// Expiry/disconnect invalidates late lookup results, but it must not return
|
|
626
|
+
// the quota slot while the underlying storage/index work is still alive.
|
|
627
|
+
// Those lookups are not generally abortable; only queueSync's finally block
|
|
628
|
+
// may release their active-work accounting.
|
|
629
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
630
|
+
reservation.active = false;
|
|
631
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
632
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
releasePendingSyncAdmission(
|
|
636
|
+
reservation?: PendingSyncAdmissionReservation,
|
|
637
|
+
): void {
|
|
638
|
+
if (!reservation || reservation.released) {
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
this.removePendingSyncAdmissionExpiry(reservation);
|
|
642
|
+
for (const identity of [...reservation.identities]) {
|
|
643
|
+
this.removePendingSyncAdmissionIdentity(reservation, identity);
|
|
644
|
+
}
|
|
645
|
+
if (reservation.retainedSettled > 0) {
|
|
646
|
+
const retainedSettled = reservation.retainedSettled;
|
|
647
|
+
reservation.retainedSettled = 0;
|
|
648
|
+
reservation.remaining -= retainedSettled;
|
|
649
|
+
this.pendingSyncAdmissionCount -= retainedSettled;
|
|
650
|
+
const peerCount =
|
|
651
|
+
(this.pendingSyncAdmissionCountByPeer.get(reservation.peer) ?? 0) -
|
|
652
|
+
retainedSettled;
|
|
653
|
+
if (peerCount === 0) {
|
|
654
|
+
this.pendingSyncAdmissionCountByPeer.delete(reservation.peer);
|
|
655
|
+
} else {
|
|
656
|
+
this.pendingSyncAdmissionCountByPeer.set(reservation.peer, peerCount);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (reservation.active) {
|
|
660
|
+
this.pendingSyncActiveAdmissionReservations -= 1;
|
|
661
|
+
}
|
|
662
|
+
reservation.active = false;
|
|
663
|
+
reservation.released = true;
|
|
664
|
+
this.pendingSyncAdmissionReservations.delete(reservation);
|
|
665
|
+
const peerReservations = this.pendingSyncAdmissionReservationsByPeer.get(
|
|
666
|
+
reservation.peer,
|
|
667
|
+
);
|
|
668
|
+
peerReservations?.delete(reservation);
|
|
669
|
+
if (peerReservations?.size === 0) {
|
|
670
|
+
this.pendingSyncAdmissionReservationsByPeer.delete(reservation.peer);
|
|
671
|
+
}
|
|
672
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
clearPendingSyncAdmissions(peer?: string): void {
|
|
676
|
+
const reservations =
|
|
677
|
+
peer == null
|
|
678
|
+
? this.pendingSyncAdmissionReservations
|
|
679
|
+
: this.pendingSyncAdmissionReservationsByPeer.get(peer);
|
|
680
|
+
if (!reservations) {
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
for (const reservation of [...reservations]) {
|
|
684
|
+
this.invalidatePendingSyncAdmission(reservation);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
private swapPendingSyncExpiry(left: number, right: number): void {
|
|
689
|
+
const leftNode = this.pendingSyncExpiryHeap[left]!;
|
|
690
|
+
const rightNode = this.pendingSyncExpiryHeap[right]!;
|
|
691
|
+
this.pendingSyncExpiryHeap[left] = rightNode;
|
|
692
|
+
this.pendingSyncExpiryHeap[right] = leftNode;
|
|
693
|
+
rightNode.heapIndex = left;
|
|
694
|
+
leftNode.heapIndex = right;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
private pushPendingSyncExpiry(node: PendingSyncExpiryNode): void {
|
|
698
|
+
node.heapIndex = this.pendingSyncExpiryHeap.length;
|
|
699
|
+
this.pendingSyncExpiryHeap.push(node);
|
|
700
|
+
let index = node.heapIndex;
|
|
701
|
+
while (index > 0) {
|
|
702
|
+
const parent = Math.floor((index - 1) / 2);
|
|
703
|
+
if (
|
|
704
|
+
this.pendingSyncExpiryHeap[parent]!.expiresAt <=
|
|
705
|
+
this.pendingSyncExpiryHeap[index]!.expiresAt
|
|
706
|
+
) {
|
|
707
|
+
break;
|
|
708
|
+
}
|
|
709
|
+
this.swapPendingSyncExpiry(parent, index);
|
|
710
|
+
index = parent;
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
private removePendingSyncExpiry(node: PendingSyncExpiryNode): void {
|
|
715
|
+
const index = node.heapIndex;
|
|
716
|
+
if (
|
|
717
|
+
index < 0 ||
|
|
718
|
+
index >= this.pendingSyncExpiryHeap.length ||
|
|
719
|
+
this.pendingSyncExpiryHeap[index] !== node
|
|
720
|
+
) {
|
|
721
|
+
return;
|
|
722
|
+
}
|
|
723
|
+
const last = this.pendingSyncExpiryHeap.pop()!;
|
|
724
|
+
node.heapIndex = -1;
|
|
725
|
+
if (index >= this.pendingSyncExpiryHeap.length) {
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
this.pendingSyncExpiryHeap[index] = last;
|
|
729
|
+
last.heapIndex = index;
|
|
730
|
+
|
|
731
|
+
let current = index;
|
|
732
|
+
while (current > 0) {
|
|
733
|
+
const parent = Math.floor((current - 1) / 2);
|
|
734
|
+
if (
|
|
735
|
+
this.pendingSyncExpiryHeap[parent]!.expiresAt <=
|
|
736
|
+
this.pendingSyncExpiryHeap[current]!.expiresAt
|
|
737
|
+
) {
|
|
738
|
+
break;
|
|
739
|
+
}
|
|
740
|
+
this.swapPendingSyncExpiry(parent, current);
|
|
741
|
+
current = parent;
|
|
742
|
+
}
|
|
743
|
+
for (;;) {
|
|
744
|
+
const left = current * 2 + 1;
|
|
745
|
+
const right = left + 1;
|
|
746
|
+
let smallest = current;
|
|
747
|
+
if (
|
|
748
|
+
left < this.pendingSyncExpiryHeap.length &&
|
|
749
|
+
this.pendingSyncExpiryHeap[left]!.expiresAt <
|
|
750
|
+
this.pendingSyncExpiryHeap[smallest]!.expiresAt
|
|
751
|
+
) {
|
|
752
|
+
smallest = left;
|
|
753
|
+
}
|
|
754
|
+
if (
|
|
755
|
+
right < this.pendingSyncExpiryHeap.length &&
|
|
756
|
+
this.pendingSyncExpiryHeap[right]!.expiresAt <
|
|
757
|
+
this.pendingSyncExpiryHeap[smallest]!.expiresAt
|
|
758
|
+
) {
|
|
759
|
+
smallest = right;
|
|
760
|
+
}
|
|
761
|
+
if (smallest === current) {
|
|
762
|
+
break;
|
|
763
|
+
}
|
|
764
|
+
this.swapPendingSyncExpiry(current, smallest);
|
|
765
|
+
current = smallest;
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
movePendingSyncKeyExpiryEarlier(key: SyncableKey, expiresAt: number): void {
|
|
770
|
+
const current = this.syncInFlightQueueExpiresAt.get(key);
|
|
771
|
+
if (current == null || current <= expiresAt) {
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
this.syncInFlightQueueExpiresAt.set(key, expiresAt);
|
|
775
|
+
const record = this.records.get(key);
|
|
776
|
+
if (!record) {
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
this.removePendingSyncExpiry(record);
|
|
780
|
+
record.expiresAt = expiresAt;
|
|
781
|
+
this.pushPendingSyncExpiry(record);
|
|
782
|
+
this.schedulePendingSyncKeyExpiry();
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
private removePendingSyncAdmissionExpiry(
|
|
786
|
+
reservation: PendingSyncAdmissionReservation,
|
|
787
|
+
): void {
|
|
788
|
+
const node = this.pendingSyncAdmissionExpiryNodes.get(reservation);
|
|
789
|
+
if (!node) {
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
this.pendingSyncAdmissionExpiryNodes.delete(reservation);
|
|
793
|
+
this.removePendingSyncExpiry(node);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
expirePendingSyncKeys(now = Date.now()): void {
|
|
797
|
+
for (;;) {
|
|
798
|
+
const node = this.pendingSyncExpiryHeap[0];
|
|
799
|
+
if (!node || node.expiresAt > now) {
|
|
800
|
+
break;
|
|
801
|
+
}
|
|
802
|
+
this.removePendingSyncExpiry(node);
|
|
803
|
+
if (node.kind === "key") {
|
|
804
|
+
if (this.records.get(node.key) !== node) {
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
this.clearSyncProcessKey(node.key);
|
|
808
|
+
} else {
|
|
809
|
+
if (
|
|
810
|
+
this.pendingSyncAdmissionExpiryNodes.get(node.reservation) !== node
|
|
811
|
+
) {
|
|
812
|
+
continue;
|
|
813
|
+
}
|
|
814
|
+
this.pendingSyncAdmissionExpiryNodes.delete(node.reservation);
|
|
815
|
+
this.invalidatePendingSyncAdmission(node.reservation);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
clearPendingSyncExpiryTimerIfIdle(): void {
|
|
821
|
+
if (
|
|
822
|
+
this.pendingSyncExpiryHeap.length === 0 &&
|
|
823
|
+
this.syncInFlightQueueExpiryTimer != null
|
|
824
|
+
) {
|
|
825
|
+
clearTimeout(this.syncInFlightQueueExpiryTimer);
|
|
826
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
private schedulePendingSyncKeyExpiry(): void {
|
|
831
|
+
if (
|
|
832
|
+
this.syncInFlightQueueExpiryTimer != null ||
|
|
833
|
+
this.pendingSyncExpiryHeap.length === 0
|
|
834
|
+
) {
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
const expiresAt = this.pendingSyncExpiryHeap[0]!.expiresAt;
|
|
838
|
+
this.syncInFlightQueueExpiryTimer = setTimeout(
|
|
839
|
+
() => {
|
|
840
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
841
|
+
this.expirePendingSyncKeys();
|
|
842
|
+
this.schedulePendingSyncKeyExpiry();
|
|
843
|
+
},
|
|
844
|
+
Math.max(0, expiresAt - Date.now()),
|
|
845
|
+
);
|
|
846
|
+
this.syncInFlightQueueExpiryTimer.unref?.();
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
hasSyncProcessState(): boolean {
|
|
850
|
+
return (
|
|
851
|
+
this.syncInFlightQueue.size > 0 ||
|
|
852
|
+
this.syncInFlightQueueInverted.size > 0 ||
|
|
853
|
+
this.pendingSyncAdmissionCount > 0 ||
|
|
854
|
+
this.syncInFlight.size > 0
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
clearSyncProcessKey(key: SyncableKey): void {
|
|
859
|
+
const inflight = this.syncInFlightQueue.get(key);
|
|
860
|
+
const record = this.records.get(key);
|
|
861
|
+
if (inflight) {
|
|
862
|
+
for (const peer of inflight) {
|
|
863
|
+
const map = this.syncInFlightQueueInverted.get(peer.hashcode());
|
|
864
|
+
if (map) {
|
|
865
|
+
map.delete(key);
|
|
866
|
+
if (map.size === 0) {
|
|
867
|
+
this.syncInFlightQueueInverted.delete(peer.hashcode());
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
this.pendingSyncClaimCount = Math.max(
|
|
873
|
+
0,
|
|
874
|
+
this.pendingSyncClaimCount -
|
|
875
|
+
(record?.claimants.size ?? inflight.length),
|
|
876
|
+
);
|
|
877
|
+
this.syncInFlightQueue.delete(key);
|
|
878
|
+
}
|
|
879
|
+
if (record) {
|
|
880
|
+
this.records.delete(key);
|
|
881
|
+
this.removePendingSyncExpiry(record);
|
|
882
|
+
}
|
|
883
|
+
if (typeof key === "bigint") {
|
|
884
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
885
|
+
}
|
|
886
|
+
this.syncInFlightQueueExpiresAt.delete(key);
|
|
887
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
888
|
+
|
|
889
|
+
this.clearSyncInFlightKey(key);
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
removePendingSyncClaim(key: SyncableKey, peer: string): void {
|
|
893
|
+
const inflight = this.syncInFlightQueue.get(key);
|
|
894
|
+
if (!inflight) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
const record =
|
|
898
|
+
this.records.get(key) ?? this.hydrateRecordFromQueue(key, inflight);
|
|
899
|
+
const index = record.claimantIndexes.get(peer);
|
|
900
|
+
if (index == null) {
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
const lastIndex = inflight.length - 1;
|
|
905
|
+
if (index !== lastIndex) {
|
|
906
|
+
const lastClaimant = inflight[lastIndex]!;
|
|
907
|
+
const lastClaimantHash = lastClaimant.hashcode();
|
|
908
|
+
inflight[index] = lastClaimant;
|
|
909
|
+
record.claimantIndexes.set(lastClaimantHash, index);
|
|
910
|
+
}
|
|
911
|
+
inflight.pop();
|
|
912
|
+
record.claimantIndexes.delete(peer);
|
|
913
|
+
record.claimants.delete(peer);
|
|
914
|
+
this.pendingSyncClaimCount = Math.max(0, this.pendingSyncClaimCount - 1);
|
|
915
|
+
const inverted = this.syncInFlightQueueInverted.get(peer);
|
|
916
|
+
inverted?.delete(key);
|
|
917
|
+
if (inverted?.size === 0) {
|
|
918
|
+
this.syncInFlightQueueInverted.delete(peer);
|
|
919
|
+
}
|
|
920
|
+
if (inflight.length > 0) {
|
|
921
|
+
const cursor = record.rrCursor;
|
|
922
|
+
record.rrCursor =
|
|
923
|
+
cursor === lastIndex
|
|
924
|
+
? index % inflight.length
|
|
925
|
+
: cursor % inflight.length;
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
this.syncInFlightQueue.delete(key);
|
|
930
|
+
this.records.delete(key);
|
|
931
|
+
if (typeof key === "bigint") {
|
|
932
|
+
this.removeQueuedSyncCoordinateAlias(key);
|
|
933
|
+
}
|
|
934
|
+
this.removePendingSyncExpiry(record);
|
|
935
|
+
this.syncInFlightQueueExpiresAt.delete(key);
|
|
936
|
+
this.clearSyncInFlightKey(key);
|
|
937
|
+
this.clearPendingSyncExpiryTimerIfIdle();
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
removeClaimsForPeer(publicKeyHash: string): void {
|
|
941
|
+
const map = this.syncInFlightQueueInverted.get(publicKeyHash);
|
|
942
|
+
if (map) {
|
|
943
|
+
for (const hash of [...map]) {
|
|
944
|
+
this.removePendingSyncClaim(hash, publicKeyHash);
|
|
945
|
+
}
|
|
946
|
+
this.syncInFlightQueueInverted.delete(publicKeyHash);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
removeSyncInFlightTargetKey(peer: string, key: SyncableKey): void {
|
|
951
|
+
const map = this.syncInFlight.get(peer);
|
|
952
|
+
if (!map?.delete(key)) {
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (map.size === 0) {
|
|
956
|
+
this.syncInFlight.delete(peer);
|
|
957
|
+
}
|
|
958
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
959
|
+
targets?.delete(peer);
|
|
960
|
+
if (targets?.size === 0) {
|
|
961
|
+
this.syncInFlightTargetsByKey.delete(key);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
setSyncInFlightTargetKey(
|
|
966
|
+
peer: string,
|
|
967
|
+
key: SyncableKey,
|
|
968
|
+
timestamp: number,
|
|
969
|
+
): void {
|
|
970
|
+
let map = this.syncInFlight.get(peer);
|
|
971
|
+
if (!map) {
|
|
972
|
+
map = new Map();
|
|
973
|
+
this.syncInFlight.set(peer, map);
|
|
974
|
+
}
|
|
975
|
+
const existing = map.get(key);
|
|
976
|
+
if (!existing || existing.timestamp < timestamp) {
|
|
977
|
+
map.set(key, { timestamp });
|
|
978
|
+
}
|
|
979
|
+
let targets = this.syncInFlightTargetsByKey.get(key);
|
|
980
|
+
if (!targets) {
|
|
981
|
+
targets = new Set();
|
|
982
|
+
this.syncInFlightTargetsByKey.set(key, targets);
|
|
983
|
+
}
|
|
984
|
+
targets.add(peer);
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
clearSyncInFlightTarget(peer: string): void {
|
|
988
|
+
const map = this.syncInFlight.get(peer);
|
|
989
|
+
if (!map) {
|
|
990
|
+
return;
|
|
991
|
+
}
|
|
992
|
+
for (const key of map.keys()) {
|
|
993
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
994
|
+
targets?.delete(peer);
|
|
995
|
+
if (targets?.size === 0) {
|
|
996
|
+
this.syncInFlightTargetsByKey.delete(key);
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
this.syncInFlight.delete(peer);
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
private clearSyncInFlightKey(key: SyncableKey): void {
|
|
1003
|
+
const targets = this.syncInFlightTargetsByKey.get(key);
|
|
1004
|
+
if (!targets) {
|
|
1005
|
+
// Defensive compatibility for tests or integrations that seed the public
|
|
1006
|
+
// syncInFlight map directly. Internal writes always populate the index.
|
|
1007
|
+
for (const [peer, map] of this.syncInFlight) {
|
|
1008
|
+
if (map.has(key)) {
|
|
1009
|
+
this.removeSyncInFlightTargetKey(peer, key);
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
for (const peer of [...targets]) {
|
|
1015
|
+
this.removeSyncInFlightTargetKey(peer, key);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
private forEachKnownAlias(
|
|
1020
|
+
hash: string,
|
|
1021
|
+
callback: (key: SyncableKey) => void,
|
|
1022
|
+
): void {
|
|
1023
|
+
callback(hash);
|
|
1024
|
+
for (const coordinate of [
|
|
1025
|
+
...(this.syncInFlightQueuedCoordinatesByHash.get(hash) ?? []),
|
|
1026
|
+
]) {
|
|
1027
|
+
callback(coordinate);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
clearSyncInFlightForPeerHashes(
|
|
1032
|
+
publicKeyHash: string,
|
|
1033
|
+
hashes: string[],
|
|
1034
|
+
): void {
|
|
1035
|
+
const map = this.syncInFlight.get(publicKeyHash);
|
|
1036
|
+
if (!map || hashes.length === 0) {
|
|
1037
|
+
return;
|
|
1038
|
+
}
|
|
1039
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
1040
|
+
for (const hash of hashes) {
|
|
1041
|
+
this.forEachKnownAlias(hash, (key) =>
|
|
1042
|
+
this.removeSyncInFlightTargetKey(publicKeyHash, key),
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
clearSyncProcess(hash: string): void {
|
|
1048
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
1049
|
+
this.forEachKnownAlias(hash, (key) => this.clearSyncProcessKey(key));
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
clearSyncProcesses(hashes: string[]): void {
|
|
1053
|
+
if (hashes.length === 0) {
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
this.refreshQueuedSyncCoordinateAliases();
|
|
1057
|
+
const keys = new Set<SyncableKey>();
|
|
1058
|
+
for (const hash of hashes) {
|
|
1059
|
+
this.forEachKnownAlias(hash, (key) => keys.add(key));
|
|
1060
|
+
}
|
|
1061
|
+
for (const key of keys) {
|
|
1062
|
+
this.clearSyncProcessKey(key);
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
clearForClose(): void {
|
|
1067
|
+
this.syncInFlightQueue.clear();
|
|
1068
|
+
this.syncInFlightQueueInverted.clear();
|
|
1069
|
+
this.syncInFlightQueueExpiresAt.clear();
|
|
1070
|
+
this.pendingSyncExpiryHeap.length = 0;
|
|
1071
|
+
this.records.clear();
|
|
1072
|
+
this.pendingSyncAdmissionExpiryNodes.clear();
|
|
1073
|
+
this.syncInFlightQueuedCoordinates.clear();
|
|
1074
|
+
this.syncInFlightQueuedHashByCoordinate.clear();
|
|
1075
|
+
this.syncInFlightQueuedCoordinatesByHash.clear();
|
|
1076
|
+
this.syncInFlightQueuedCoordinateRefreshIterator = undefined;
|
|
1077
|
+
this.pendingSyncClaimCount = 0;
|
|
1078
|
+
if (this.syncInFlightQueueExpiryTimer != null) {
|
|
1079
|
+
clearTimeout(this.syncInFlightQueueExpiryTimer);
|
|
1080
|
+
this.syncInFlightQueueExpiryTimer = undefined;
|
|
1081
|
+
}
|
|
1082
|
+
this.syncInFlight.clear();
|
|
1083
|
+
this.syncInFlightTargetsByKey.clear();
|
|
1084
|
+
}
|
|
1085
|
+
}
|