@serfab/cadre-core 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/cadre-node.d.ts +203 -28
- package/dist/cadre-node.js +445 -84
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +82 -117
- package/dist/control-database.js +217 -295
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +789 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +15 -5
- package/dist/control-write-retry.js +19 -18
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +8 -8
- package/dist/strand-formation-manager.js +15 -18
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +52 -21
- package/dist/strand-instance-manager.js +100 -34
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +6 -4
- package/dist/strand-membership-writer.js +38 -36
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/types.d.ts +39 -31
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +5158 -4771
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +2406 -2506
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-schema.ts +801 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +483 -473
- package/src/delegate-admission.ts +302 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +1 -0
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +204 -204
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +164 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +111 -111
- package/src/relay-reservation.ts +823 -823
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +441 -441
- package/src/strand-backfill.ts +366 -366
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +491 -494
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +108 -50
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1432 -1430
- package/src/strand-solicitation.ts +472 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +314 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1309 -1303
|
@@ -1,437 +1,437 @@
|
|
|
1
|
-
import debug from 'debug';
|
|
2
|
-
import type {
|
|
3
|
-
StrandInstance,
|
|
4
|
-
LatencyHint,
|
|
5
|
-
HibernationTimeouts,
|
|
6
|
-
HibernationConfig
|
|
7
|
-
} from './types.js';
|
|
8
|
-
import { HIBERNATION_TIMEOUTS } from './types.js';
|
|
9
|
-
|
|
10
|
-
const log = debug('sereus:cadre:hibernation');
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Callbacks for hibernation state changes
|
|
14
|
-
*/
|
|
15
|
-
export interface HibernationCallbacks {
|
|
16
|
-
onIdle: (strandId: string) => Promise<void>;
|
|
17
|
-
onHibernate: (strandId: string) => Promise<void>;
|
|
18
|
-
onWake: (strandId: string) => Promise<void>;
|
|
19
|
-
/**
|
|
20
|
-
* Perform a real cohort check-in for a hibernating strand. The implementation
|
|
21
|
-
* (`CadreNode.handleStrandCheckIn`) resumes the strand, gives it a bounded
|
|
22
|
-
* window to connect to reachable cohort peers and surface pending activity,
|
|
23
|
-
* then re-hibernates if still idle. The manager AWAITS this before scheduling
|
|
24
|
-
* the next (longer-delayed) check-in, so a slow check-in never overlaps the
|
|
25
|
-
* next tick. After it resolves the manager inspects `instance.status`: a
|
|
26
|
-
* strand left non-`hibernating` is treated as woken (backoff resets on the
|
|
27
|
-
* next hibernation); a strand left `hibernating` escalates the backoff.
|
|
28
|
-
*/
|
|
29
|
-
onCheckIn: (strandId: string) => Promise<void>;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Manages strand hibernation state transitions based on activity.
|
|
34
|
-
*
|
|
35
|
-
* State machine:
|
|
36
|
-
* active → idle (after idleTimeout with no activity)
|
|
37
|
-
* idle → hibernating (after hibernateTimeout with no activity)
|
|
38
|
-
* idle → active (on activity)
|
|
39
|
-
* hibernating → active (on wake signal or check-in with pending activity)
|
|
40
|
-
*/
|
|
41
|
-
export class HibernationManager {
|
|
42
|
-
private readonly config: HibernationConfig;
|
|
43
|
-
private readonly callbacks: HibernationCallbacks;
|
|
44
|
-
private readonly timers: Map<string, ReturnType<typeof setTimeout>> = new Map();
|
|
45
|
-
/**
|
|
46
|
-
* Pending check-in timers, one per hibernating strand. Unlike the old fixed
|
|
47
|
-
* `setInterval`, these are single-shot `setTimeout`s rescheduled by
|
|
48
|
-
* {@link runCheckIn} with an escalating delay — so a long-running `onCheckIn`
|
|
49
|
-
* can never overlap the next tick, and the period adapts to the backoff.
|
|
50
|
-
*/
|
|
51
|
-
private readonly checkInTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
|
|
52
|
-
/**
|
|
53
|
-
* In-flight wake promises keyed by strandId. Coalesces overlapping wake
|
|
54
|
-
* triggers (two near-simultaneous activities, or activity racing a force wake)
|
|
55
|
-
* so `onWake` — and the libp2p-node rebuild it drives — runs at most once per
|
|
56
|
-
* concurrent wake.
|
|
57
|
-
*/
|
|
58
|
-
private readonly wakePromises: Map<string, Promise<void>> = new Map();
|
|
59
|
-
private running = false;
|
|
60
|
-
|
|
61
|
-
constructor(config: HibernationConfig, callbacks: HibernationCallbacks) {
|
|
62
|
-
this.config = config;
|
|
63
|
-
this.callbacks = callbacks;
|
|
64
|
-
log('HibernationManager created, enabled=%s', config.enabled);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Get effective timeouts for a latency hint
|
|
69
|
-
*/
|
|
70
|
-
private getTimeouts(hint: LatencyHint): HibernationTimeouts {
|
|
71
|
-
const defaults = HIBERNATION_TIMEOUTS[hint];
|
|
72
|
-
const custom = this.config.customTimeouts?.[hint];
|
|
73
|
-
|
|
74
|
-
if (!custom) return defaults;
|
|
75
|
-
|
|
76
|
-
return {
|
|
77
|
-
idleTimeout: custom.idleTimeout ?? defaults.idleTimeout,
|
|
78
|
-
hibernateTimeout: custom.hibernateTimeout ?? defaults.hibernateTimeout,
|
|
79
|
-
checkInInterval: custom.checkInInterval ?? defaults.checkInInterval,
|
|
80
|
-
checkInBackoffFactor: custom.checkInBackoffFactor ?? defaults.checkInBackoffFactor,
|
|
81
|
-
checkInMaxInterval: custom.checkInMaxInterval ?? defaults.checkInMaxInterval
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
/**
|
|
86
|
-
* Start managing hibernation for all strands
|
|
87
|
-
*/
|
|
88
|
-
start(): void {
|
|
89
|
-
if (this.running) return;
|
|
90
|
-
this.running = true;
|
|
91
|
-
log('HibernationManager started');
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* Stop managing hibernation
|
|
96
|
-
*/
|
|
97
|
-
stop(): void {
|
|
98
|
-
if (!this.running) return;
|
|
99
|
-
this.running = false;
|
|
100
|
-
|
|
101
|
-
// Clear all timers
|
|
102
|
-
for (const timer of this.timers.values()) {
|
|
103
|
-
clearTimeout(timer);
|
|
104
|
-
}
|
|
105
|
-
this.timers.clear();
|
|
106
|
-
|
|
107
|
-
for (const timer of this.checkInTimers.values()) {
|
|
108
|
-
clearTimeout(timer);
|
|
109
|
-
}
|
|
110
|
-
this.checkInTimers.clear();
|
|
111
|
-
|
|
112
|
-
// In-flight wakes clean themselves up via their finally; drop the references
|
|
113
|
-
// so a fresh start coalesces cleanly.
|
|
114
|
-
this.wakePromises.clear();
|
|
115
|
-
|
|
116
|
-
log('HibernationManager stopped');
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/**
|
|
120
|
-
* Whether a strand with this instance's latency hint ever hibernates — `false`
|
|
121
|
-
* for realtime (Infinity idle timeout, see {@link HIBERNATION_TIMEOUTS}), also
|
|
122
|
-
* honouring any per-hint `customTimeouts` override. Imperative callers
|
|
123
|
-
* (`CadreNode.hibernateStrand` / `hibernateAll`) use this as the single source
|
|
124
|
-
* of truth for "skip realtime", consistent with {@link trackStrand} declining
|
|
125
|
-
* to track Infinity-timeout strands.
|
|
126
|
-
*/
|
|
127
|
-
hibernates(instance: StrandInstance): boolean {
|
|
128
|
-
return this.getTimeouts(instance.latencyHint).idleTimeout !== Infinity;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* Imperatively hibernate a tracked strand now, bypassing the idle/hibernate
|
|
133
|
-
* timers — the mobile background-entry path. Cancels the strand's pending
|
|
134
|
-
* idle/hibernate AND check-in timers so none can later re-fire `onHibernate`
|
|
135
|
-
* on the already-quiesced strand or resurrect one the caller means to keep
|
|
136
|
-
* down, then invokes `onHibernate` (quiesce + mark hibernating).
|
|
137
|
-
*
|
|
138
|
-
* Unlike the timer path ({@link handleHibernateTimeout}) it deliberately does
|
|
139
|
-
* NOT re-arm the check-in chain: a force-hibernate keeps the strand down until
|
|
140
|
-
* the caller drives the next wake on demand (push-delivered on mobile), so a
|
|
141
|
-
* stray check-in timer must not bring it back up.
|
|
142
|
-
*
|
|
143
|
-
* @returns `true` if the strand was hibernated, `false` (no-op) for a realtime
|
|
144
|
-
* strand that never hibernates.
|
|
145
|
-
*/
|
|
146
|
-
async forceHibernate(instance: StrandInstance): Promise<boolean> {
|
|
147
|
-
if (!this.hibernates(instance)) {
|
|
148
|
-
log('forceHibernate: strand %s is realtime; no-op', instance.strandId);
|
|
149
|
-
return false;
|
|
150
|
-
}
|
|
151
|
-
// Cancel idle/hibernate + check-in timers BEFORE quiescing so nothing fights
|
|
152
|
-
// the imperative hibernate (a stale hibernate timer firing on the quiesced
|
|
153
|
-
// strand, or a check-in resuming a strand the caller wants kept down).
|
|
154
|
-
this.clearTimers(instance.strandId);
|
|
155
|
-
await this.callbacks.onHibernate(instance.strandId);
|
|
156
|
-
log('forceHibernate: strand %s hibernated (timers cancelled, not re-armed)', instance.strandId);
|
|
157
|
-
return true;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Register a strand for hibernation management
|
|
162
|
-
*/
|
|
163
|
-
trackStrand(instance: StrandInstance): void {
|
|
164
|
-
if (!this.config.enabled || !this.running) return;
|
|
165
|
-
|
|
166
|
-
const { strandId, latencyHint } = instance;
|
|
167
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
168
|
-
|
|
169
|
-
// Don't track strands that never hibernate
|
|
170
|
-
if (timeouts.idleTimeout === Infinity) {
|
|
171
|
-
log('Strand %s has realtime latency hint - no hibernation', strandId);
|
|
172
|
-
return;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
log('Tracking strand %s for hibernation (hint=%s)', strandId, latencyHint);
|
|
176
|
-
this.scheduleIdleTransition(instance);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* Untrack a strand from hibernation management
|
|
181
|
-
*/
|
|
182
|
-
untrackStrand(strandId: string): void {
|
|
183
|
-
this.clearTimers(strandId);
|
|
184
|
-
log('Untracked strand %s from hibernation', strandId);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* Record activity on a strand - resets idle timer
|
|
189
|
-
*/
|
|
190
|
-
recordActivity(instance: StrandInstance): void {
|
|
191
|
-
if (!this.config.enabled || !this.running) return;
|
|
192
|
-
|
|
193
|
-
const { strandId, status, latencyHint } = instance;
|
|
194
|
-
instance.lastActivity = new Date();
|
|
195
|
-
|
|
196
|
-
// If idle or hibernating, wake up. Coalesce so two near-simultaneous
|
|
197
|
-
// activities don't each fire onWake (which would rebuild two libp2p nodes).
|
|
198
|
-
if (status === 'idle' || status === 'hibernating') {
|
|
199
|
-
log('Activity on %s strand %s - waking', status, strandId);
|
|
200
|
-
this.clearTimers(strandId);
|
|
201
|
-
// Fire-and-forget; force-wake awaiters see errors, so swallow (and log)
|
|
202
|
-
// here to avoid an unhandled rejection on this best-effort path.
|
|
203
|
-
//
|
|
204
|
-
// Once the wake settles (CadreNode rebuilds the runtime and marks the
|
|
205
|
-
// strand `active`), re-arm the idle→hibernate→check-in cycle. Without this
|
|
206
|
-
// a strand that wakes then goes quiet stays `active` forever and never
|
|
207
|
-
// re-hibernates — so the check-in backoff could never restart at base.
|
|
208
|
-
// Guarded on `active` so a wake that did not transition (a coalesced
|
|
209
|
-
// no-op, or a still-mid-flight rebuild) never arms a stray timer.
|
|
210
|
-
void this.beginWake(strandId).then(() => {
|
|
211
|
-
if (this.running && instance.status === 'active') {
|
|
212
|
-
const timeouts = this.getTimeouts(instance.latencyHint);
|
|
213
|
-
if (timeouts.idleTimeout !== Infinity) {
|
|
214
|
-
this.scheduleIdleTransition(instance);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}).catch((err) => {
|
|
218
|
-
log('Activity-driven wake failed for strand %s: %o', strandId, err);
|
|
219
|
-
});
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// Reschedule idle transition if active
|
|
224
|
-
if (status === 'active') {
|
|
225
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
226
|
-
if (timeouts.idleTimeout !== Infinity) {
|
|
227
|
-
this.scheduleIdleTransition(instance);
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/**
|
|
233
|
-
* Force wake a hibernating strand
|
|
234
|
-
*/
|
|
235
|
-
async wakeStrand(strandId: string): Promise<void> {
|
|
236
|
-
this.clearTimers(strandId);
|
|
237
|
-
await this.beginWake(strandId);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
/**
|
|
241
|
-
* Begin a wake for a strand, or coalesce with one already in flight. Ensures
|
|
242
|
-
* `onWake` runs at most once per concurrent wake — the returned promise is
|
|
243
|
-
* shared by all overlapping callers and cleared once it settles. Force-wake
|
|
244
|
-
* callers await it; activity-driven callers fire-and-forget.
|
|
245
|
-
*/
|
|
246
|
-
private beginWake(strandId: string): Promise<void> {
|
|
247
|
-
const existing = this.wakePromises.get(strandId);
|
|
248
|
-
if (existing) {
|
|
249
|
-
return existing;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
const wake = (async () => {
|
|
253
|
-
try {
|
|
254
|
-
await this.callbacks.onWake(strandId);
|
|
255
|
-
} finally {
|
|
256
|
-
this.wakePromises.delete(strandId);
|
|
257
|
-
}
|
|
258
|
-
})();
|
|
259
|
-
this.wakePromises.set(strandId, wake);
|
|
260
|
-
return wake;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
private scheduleIdleTransition(instance: StrandInstance): void {
|
|
264
|
-
const { strandId, latencyHint } = instance;
|
|
265
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
266
|
-
|
|
267
|
-
// Clear existing timer
|
|
268
|
-
this.clearTimer(strandId);
|
|
269
|
-
|
|
270
|
-
// Schedule idle transition
|
|
271
|
-
const timer = setTimeout(() => {
|
|
272
|
-
this.handleIdleTimeout(instance);
|
|
273
|
-
}, timeouts.idleTimeout);
|
|
274
|
-
|
|
275
|
-
this.timers.set(strandId, timer);
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
private handleIdleTimeout(instance: StrandInstance): void {
|
|
279
|
-
const { strandId, latencyHint } = instance;
|
|
280
|
-
|
|
281
|
-
if (!this.running) return;
|
|
282
|
-
|
|
283
|
-
log('Idle timeout for strand %s', strandId);
|
|
284
|
-
|
|
285
|
-
// Transition to idle. The callback can reject (e.g. a future idle handler
|
|
286
|
-
// that releases resources); catch so the timer chain never unhandled-rejects.
|
|
287
|
-
void this.callbacks.onIdle(strandId).then(() => {
|
|
288
|
-
// Schedule hibernate transition
|
|
289
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
290
|
-
if (timeouts.hibernateTimeout !== Infinity) {
|
|
291
|
-
this.scheduleHibernateTransition(instance);
|
|
292
|
-
}
|
|
293
|
-
}).catch((err) => {
|
|
294
|
-
log('onIdle failed for strand %s: %o', strandId, err);
|
|
295
|
-
});
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
private scheduleHibernateTransition(instance: StrandInstance): void {
|
|
299
|
-
const { strandId, latencyHint } = instance;
|
|
300
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
301
|
-
|
|
302
|
-
// Clear existing timer
|
|
303
|
-
this.clearTimer(strandId);
|
|
304
|
-
|
|
305
|
-
// Schedule hibernate transition
|
|
306
|
-
const timer = setTimeout(() => {
|
|
307
|
-
this.handleHibernateTimeout(instance);
|
|
308
|
-
}, timeouts.hibernateTimeout);
|
|
309
|
-
|
|
310
|
-
this.timers.set(strandId, timer);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
private handleHibernateTimeout(instance: StrandInstance): void {
|
|
314
|
-
const { strandId, latencyHint } = instance;
|
|
315
|
-
|
|
316
|
-
if (!this.running) return;
|
|
317
|
-
|
|
318
|
-
log('Hibernate timeout for strand %s', strandId);
|
|
319
|
-
|
|
320
|
-
// Transition to hibernating. onHibernate now releases strand-network
|
|
321
|
-
// resources (quiesce), so its close()/stop() can reject — catch so a failed
|
|
322
|
-
// hibernate logs instead of producing an unhandled rejection.
|
|
323
|
-
void this.callbacks.onHibernate(strandId).then(() => {
|
|
324
|
-
// Schedule periodic check-ins
|
|
325
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
326
|
-
if (timeouts.checkInInterval !== Infinity) {
|
|
327
|
-
this.scheduleCheckIn(instance);
|
|
328
|
-
}
|
|
329
|
-
}).catch((err) => {
|
|
330
|
-
log('onHibernate failed for strand %s: %o', strandId, err);
|
|
331
|
-
});
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
/**
|
|
335
|
-
* Arm the next check-in for a hibernating strand. Each call is a single-shot
|
|
336
|
-
* `setTimeout` (not a fixed `setInterval`) so the period escalates per
|
|
337
|
-
* {@link runCheckIn} and a slow `onCheckIn` never overlaps the next tick.
|
|
338
|
-
*
|
|
339
|
-
* `delay` is omitted by the chain start ({@link handleHibernateTimeout}),
|
|
340
|
-
* defaulting to the base `checkInInterval` — which is why backoff naturally
|
|
341
|
-
* resets to base each fresh hibernation cycle, with no per-strand counter to
|
|
342
|
-
* clear on wake. Subsequent ticks pass the escalated, capped delay.
|
|
343
|
-
*/
|
|
344
|
-
private scheduleCheckIn(instance: StrandInstance, delay?: number): void {
|
|
345
|
-
const { strandId, latencyHint } = instance;
|
|
346
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
347
|
-
const currentDelay = delay ?? timeouts.checkInInterval;
|
|
348
|
-
|
|
349
|
-
// Replace any existing check-in timer (no-op if the firing timer rescheduled us).
|
|
350
|
-
const existing = this.checkInTimers.get(strandId);
|
|
351
|
-
if (existing) {
|
|
352
|
-
clearTimeout(existing);
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
const timer = setTimeout(() => {
|
|
356
|
-
void this.runCheckIn(instance, currentDelay);
|
|
357
|
-
}, currentDelay);
|
|
358
|
-
|
|
359
|
-
this.checkInTimers.set(strandId, timer);
|
|
360
|
-
instance.nextCheckIn = new Date(Date.now() + currentDelay);
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
/**
|
|
364
|
-
* Run a single check-in tick: invoke `onCheckIn` (a real resume → bounded
|
|
365
|
-
* sync window → re-hibernate-if-idle cycle in `CadreNode`) and AWAIT it before
|
|
366
|
-
* deciding the next step.
|
|
367
|
-
*
|
|
368
|
-
* - If the strand woke during the check-in (`onCheckIn` left it non-
|
|
369
|
-
* `hibernating`), stop the chain — the activity path has re-armed the
|
|
370
|
-
* idle/hibernate timers, and the next hibernation restarts the chain at the
|
|
371
|
-
* base delay (backoff reset).
|
|
372
|
-
* - Otherwise escalate the delay by `checkInBackoffFactor`, capped at
|
|
373
|
-
* `checkInMaxInterval`, and reschedule.
|
|
374
|
-
*/
|
|
375
|
-
private async runCheckIn(instance: StrandInstance, currentDelay: number): Promise<void> {
|
|
376
|
-
const { strandId, latencyHint } = instance;
|
|
377
|
-
if (!this.running) return;
|
|
378
|
-
|
|
379
|
-
log('Check-in for hibernating strand %s (delay=%dms)', strandId, currentDelay);
|
|
380
|
-
|
|
381
|
-
try {
|
|
382
|
-
await this.callbacks.onCheckIn(strandId);
|
|
383
|
-
} catch (err) {
|
|
384
|
-
// A failed check-in (e.g. resume threw) must not break the chain — log and
|
|
385
|
-
// fall through to reschedule the next, longer-delayed attempt.
|
|
386
|
-
log('onCheckIn failed for strand %s: %o', strandId, err);
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
if (!this.running) return;
|
|
390
|
-
|
|
391
|
-
// The check-in either woke the strand (CadreNode left it active) or left it
|
|
392
|
-
// hibernating. Inspect the shared instance the callback just mutated.
|
|
393
|
-
if (instance.status !== 'hibernating') {
|
|
394
|
-
log('Check-in woke strand %s; backoff resets on next hibernation', strandId);
|
|
395
|
-
this.checkInTimers.delete(strandId);
|
|
396
|
-
// The chain is stopping; drop the now-stale next-check-in advertisement so
|
|
397
|
-
// `getStrand` doesn't report a phantom check-in for a strand that is awake.
|
|
398
|
-
instance.nextCheckIn = undefined;
|
|
399
|
-
return;
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
const timeouts = this.getTimeouts(latencyHint);
|
|
403
|
-
const nextDelay = Math.min(
|
|
404
|
-
currentDelay * timeouts.checkInBackoffFactor,
|
|
405
|
-
timeouts.checkInMaxInterval
|
|
406
|
-
);
|
|
407
|
-
this.scheduleCheckIn(instance, nextDelay);
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
private clearTimer(strandId: string): void {
|
|
411
|
-
const timer = this.timers.get(strandId);
|
|
412
|
-
if (timer) {
|
|
413
|
-
clearTimeout(timer);
|
|
414
|
-
this.timers.delete(strandId);
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
private clearTimers(strandId: string): void {
|
|
419
|
-
this.clearTimer(strandId);
|
|
420
|
-
|
|
421
|
-
const checkInTimer = this.checkInTimers.get(strandId);
|
|
422
|
-
if (checkInTimer) {
|
|
423
|
-
clearTimeout(checkInTimer);
|
|
424
|
-
this.checkInTimers.delete(strandId);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
/**
|
|
429
|
-
* Get the current status of hibernation tracking
|
|
430
|
-
*/
|
|
431
|
-
getStatus(): { enabled: boolean; trackedStrands: number } {
|
|
432
|
-
return {
|
|
433
|
-
enabled: this.config.enabled && this.running,
|
|
434
|
-
trackedStrands: this.timers.size + this.checkInTimers.size
|
|
435
|
-
};
|
|
436
|
-
}
|
|
437
|
-
}
|
|
1
|
+
import debug from 'debug';
|
|
2
|
+
import type {
|
|
3
|
+
StrandInstance,
|
|
4
|
+
LatencyHint,
|
|
5
|
+
HibernationTimeouts,
|
|
6
|
+
HibernationConfig
|
|
7
|
+
} from './types.js';
|
|
8
|
+
import { HIBERNATION_TIMEOUTS } from './types.js';
|
|
9
|
+
|
|
10
|
+
const log = debug('sereus:cadre:hibernation');
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Callbacks for hibernation state changes
|
|
14
|
+
*/
|
|
15
|
+
export interface HibernationCallbacks {
|
|
16
|
+
onIdle: (strandId: string) => Promise<void>;
|
|
17
|
+
onHibernate: (strandId: string) => Promise<void>;
|
|
18
|
+
onWake: (strandId: string) => Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Perform a real cohort check-in for a hibernating strand. The implementation
|
|
21
|
+
* (`CadreNode.handleStrandCheckIn`) resumes the strand, gives it a bounded
|
|
22
|
+
* window to connect to reachable cohort peers and surface pending activity,
|
|
23
|
+
* then re-hibernates if still idle. The manager AWAITS this before scheduling
|
|
24
|
+
* the next (longer-delayed) check-in, so a slow check-in never overlaps the
|
|
25
|
+
* next tick. After it resolves the manager inspects `instance.status`: a
|
|
26
|
+
* strand left non-`hibernating` is treated as woken (backoff resets on the
|
|
27
|
+
* next hibernation); a strand left `hibernating` escalates the backoff.
|
|
28
|
+
*/
|
|
29
|
+
onCheckIn: (strandId: string) => Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Manages strand hibernation state transitions based on activity.
|
|
34
|
+
*
|
|
35
|
+
* State machine:
|
|
36
|
+
* active → idle (after idleTimeout with no activity)
|
|
37
|
+
* idle → hibernating (after hibernateTimeout with no activity)
|
|
38
|
+
* idle → active (on activity)
|
|
39
|
+
* hibernating → active (on wake signal or check-in with pending activity)
|
|
40
|
+
*/
|
|
41
|
+
export class HibernationManager {
|
|
42
|
+
private readonly config: HibernationConfig;
|
|
43
|
+
private readonly callbacks: HibernationCallbacks;
|
|
44
|
+
private readonly timers: Map<string, ReturnType<typeof setTimeout>> = new Map();
|
|
45
|
+
/**
|
|
46
|
+
* Pending check-in timers, one per hibernating strand. Unlike the old fixed
|
|
47
|
+
* `setInterval`, these are single-shot `setTimeout`s rescheduled by
|
|
48
|
+
* {@link runCheckIn} with an escalating delay — so a long-running `onCheckIn`
|
|
49
|
+
* can never overlap the next tick, and the period adapts to the backoff.
|
|
50
|
+
*/
|
|
51
|
+
private readonly checkInTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
|
|
52
|
+
/**
|
|
53
|
+
* In-flight wake promises keyed by strandId. Coalesces overlapping wake
|
|
54
|
+
* triggers (two near-simultaneous activities, or activity racing a force wake)
|
|
55
|
+
* so `onWake` — and the libp2p-node rebuild it drives — runs at most once per
|
|
56
|
+
* concurrent wake.
|
|
57
|
+
*/
|
|
58
|
+
private readonly wakePromises: Map<string, Promise<void>> = new Map();
|
|
59
|
+
private running = false;
|
|
60
|
+
|
|
61
|
+
constructor(config: HibernationConfig, callbacks: HibernationCallbacks) {
|
|
62
|
+
this.config = config;
|
|
63
|
+
this.callbacks = callbacks;
|
|
64
|
+
log('HibernationManager created, enabled=%s', config.enabled);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Get effective timeouts for a latency hint
|
|
69
|
+
*/
|
|
70
|
+
private getTimeouts(hint: LatencyHint): HibernationTimeouts {
|
|
71
|
+
const defaults = HIBERNATION_TIMEOUTS[hint];
|
|
72
|
+
const custom = this.config.customTimeouts?.[hint];
|
|
73
|
+
|
|
74
|
+
if (!custom) return defaults;
|
|
75
|
+
|
|
76
|
+
return {
|
|
77
|
+
idleTimeout: custom.idleTimeout ?? defaults.idleTimeout,
|
|
78
|
+
hibernateTimeout: custom.hibernateTimeout ?? defaults.hibernateTimeout,
|
|
79
|
+
checkInInterval: custom.checkInInterval ?? defaults.checkInInterval,
|
|
80
|
+
checkInBackoffFactor: custom.checkInBackoffFactor ?? defaults.checkInBackoffFactor,
|
|
81
|
+
checkInMaxInterval: custom.checkInMaxInterval ?? defaults.checkInMaxInterval
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Start managing hibernation for all strands
|
|
87
|
+
*/
|
|
88
|
+
start(): void {
|
|
89
|
+
if (this.running) return;
|
|
90
|
+
this.running = true;
|
|
91
|
+
log('HibernationManager started');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Stop managing hibernation
|
|
96
|
+
*/
|
|
97
|
+
stop(): void {
|
|
98
|
+
if (!this.running) return;
|
|
99
|
+
this.running = false;
|
|
100
|
+
|
|
101
|
+
// Clear all timers
|
|
102
|
+
for (const timer of this.timers.values()) {
|
|
103
|
+
clearTimeout(timer);
|
|
104
|
+
}
|
|
105
|
+
this.timers.clear();
|
|
106
|
+
|
|
107
|
+
for (const timer of this.checkInTimers.values()) {
|
|
108
|
+
clearTimeout(timer);
|
|
109
|
+
}
|
|
110
|
+
this.checkInTimers.clear();
|
|
111
|
+
|
|
112
|
+
// In-flight wakes clean themselves up via their finally; drop the references
|
|
113
|
+
// so a fresh start coalesces cleanly.
|
|
114
|
+
this.wakePromises.clear();
|
|
115
|
+
|
|
116
|
+
log('HibernationManager stopped');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Whether a strand with this instance's latency hint ever hibernates — `false`
|
|
121
|
+
* for realtime (Infinity idle timeout, see {@link HIBERNATION_TIMEOUTS}), also
|
|
122
|
+
* honouring any per-hint `customTimeouts` override. Imperative callers
|
|
123
|
+
* (`CadreNode.hibernateStrand` / `hibernateAll`) use this as the single source
|
|
124
|
+
* of truth for "skip realtime", consistent with {@link trackStrand} declining
|
|
125
|
+
* to track Infinity-timeout strands.
|
|
126
|
+
*/
|
|
127
|
+
hibernates(instance: StrandInstance): boolean {
|
|
128
|
+
return this.getTimeouts(instance.latencyHint).idleTimeout !== Infinity;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Imperatively hibernate a tracked strand now, bypassing the idle/hibernate
|
|
133
|
+
* timers — the mobile background-entry path. Cancels the strand's pending
|
|
134
|
+
* idle/hibernate AND check-in timers so none can later re-fire `onHibernate`
|
|
135
|
+
* on the already-quiesced strand or resurrect one the caller means to keep
|
|
136
|
+
* down, then invokes `onHibernate` (quiesce + mark hibernating).
|
|
137
|
+
*
|
|
138
|
+
* Unlike the timer path ({@link handleHibernateTimeout}) it deliberately does
|
|
139
|
+
* NOT re-arm the check-in chain: a force-hibernate keeps the strand down until
|
|
140
|
+
* the caller drives the next wake on demand (push-delivered on mobile), so a
|
|
141
|
+
* stray check-in timer must not bring it back up.
|
|
142
|
+
*
|
|
143
|
+
* @returns `true` if the strand was hibernated, `false` (no-op) for a realtime
|
|
144
|
+
* strand that never hibernates.
|
|
145
|
+
*/
|
|
146
|
+
async forceHibernate(instance: StrandInstance): Promise<boolean> {
|
|
147
|
+
if (!this.hibernates(instance)) {
|
|
148
|
+
log('forceHibernate: strand %s is realtime; no-op', instance.strandId);
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
// Cancel idle/hibernate + check-in timers BEFORE quiescing so nothing fights
|
|
152
|
+
// the imperative hibernate (a stale hibernate timer firing on the quiesced
|
|
153
|
+
// strand, or a check-in resuming a strand the caller wants kept down).
|
|
154
|
+
this.clearTimers(instance.strandId);
|
|
155
|
+
await this.callbacks.onHibernate(instance.strandId);
|
|
156
|
+
log('forceHibernate: strand %s hibernated (timers cancelled, not re-armed)', instance.strandId);
|
|
157
|
+
return true;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Register a strand for hibernation management
|
|
162
|
+
*/
|
|
163
|
+
trackStrand(instance: StrandInstance): void {
|
|
164
|
+
if (!this.config.enabled || !this.running) return;
|
|
165
|
+
|
|
166
|
+
const { strandId, latencyHint } = instance;
|
|
167
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
168
|
+
|
|
169
|
+
// Don't track strands that never hibernate
|
|
170
|
+
if (timeouts.idleTimeout === Infinity) {
|
|
171
|
+
log('Strand %s has realtime latency hint - no hibernation', strandId);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
log('Tracking strand %s for hibernation (hint=%s)', strandId, latencyHint);
|
|
176
|
+
this.scheduleIdleTransition(instance);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Untrack a strand from hibernation management
|
|
181
|
+
*/
|
|
182
|
+
untrackStrand(strandId: string): void {
|
|
183
|
+
this.clearTimers(strandId);
|
|
184
|
+
log('Untracked strand %s from hibernation', strandId);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Record activity on a strand - resets idle timer
|
|
189
|
+
*/
|
|
190
|
+
recordActivity(instance: StrandInstance): void {
|
|
191
|
+
if (!this.config.enabled || !this.running) return;
|
|
192
|
+
|
|
193
|
+
const { strandId, status, latencyHint } = instance;
|
|
194
|
+
instance.lastActivity = new Date();
|
|
195
|
+
|
|
196
|
+
// If idle or hibernating, wake up. Coalesce so two near-simultaneous
|
|
197
|
+
// activities don't each fire onWake (which would rebuild two libp2p nodes).
|
|
198
|
+
if (status === 'idle' || status === 'hibernating') {
|
|
199
|
+
log('Activity on %s strand %s - waking', status, strandId);
|
|
200
|
+
this.clearTimers(strandId);
|
|
201
|
+
// Fire-and-forget; force-wake awaiters see errors, so swallow (and log)
|
|
202
|
+
// here to avoid an unhandled rejection on this best-effort path.
|
|
203
|
+
//
|
|
204
|
+
// Once the wake settles (CadreNode rebuilds the runtime and marks the
|
|
205
|
+
// strand `active`), re-arm the idle→hibernate→check-in cycle. Without this
|
|
206
|
+
// a strand that wakes then goes quiet stays `active` forever and never
|
|
207
|
+
// re-hibernates — so the check-in backoff could never restart at base.
|
|
208
|
+
// Guarded on `active` so a wake that did not transition (a coalesced
|
|
209
|
+
// no-op, or a still-mid-flight rebuild) never arms a stray timer.
|
|
210
|
+
void this.beginWake(strandId).then(() => {
|
|
211
|
+
if (this.running && instance.status === 'active') {
|
|
212
|
+
const timeouts = this.getTimeouts(instance.latencyHint);
|
|
213
|
+
if (timeouts.idleTimeout !== Infinity) {
|
|
214
|
+
this.scheduleIdleTransition(instance);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}).catch((err) => {
|
|
218
|
+
log('Activity-driven wake failed for strand %s: %o', strandId, err);
|
|
219
|
+
});
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Reschedule idle transition if active
|
|
224
|
+
if (status === 'active') {
|
|
225
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
226
|
+
if (timeouts.idleTimeout !== Infinity) {
|
|
227
|
+
this.scheduleIdleTransition(instance);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Force wake a hibernating strand
|
|
234
|
+
*/
|
|
235
|
+
async wakeStrand(strandId: string): Promise<void> {
|
|
236
|
+
this.clearTimers(strandId);
|
|
237
|
+
await this.beginWake(strandId);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Begin a wake for a strand, or coalesce with one already in flight. Ensures
|
|
242
|
+
* `onWake` runs at most once per concurrent wake — the returned promise is
|
|
243
|
+
* shared by all overlapping callers and cleared once it settles. Force-wake
|
|
244
|
+
* callers await it; activity-driven callers fire-and-forget.
|
|
245
|
+
*/
|
|
246
|
+
private beginWake(strandId: string): Promise<void> {
|
|
247
|
+
const existing = this.wakePromises.get(strandId);
|
|
248
|
+
if (existing) {
|
|
249
|
+
return existing;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const wake = (async () => {
|
|
253
|
+
try {
|
|
254
|
+
await this.callbacks.onWake(strandId);
|
|
255
|
+
} finally {
|
|
256
|
+
this.wakePromises.delete(strandId);
|
|
257
|
+
}
|
|
258
|
+
})();
|
|
259
|
+
this.wakePromises.set(strandId, wake);
|
|
260
|
+
return wake;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
private scheduleIdleTransition(instance: StrandInstance): void {
|
|
264
|
+
const { strandId, latencyHint } = instance;
|
|
265
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
266
|
+
|
|
267
|
+
// Clear existing timer
|
|
268
|
+
this.clearTimer(strandId);
|
|
269
|
+
|
|
270
|
+
// Schedule idle transition
|
|
271
|
+
const timer = setTimeout(() => {
|
|
272
|
+
this.handleIdleTimeout(instance);
|
|
273
|
+
}, timeouts.idleTimeout);
|
|
274
|
+
|
|
275
|
+
this.timers.set(strandId, timer);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
private handleIdleTimeout(instance: StrandInstance): void {
|
|
279
|
+
const { strandId, latencyHint } = instance;
|
|
280
|
+
|
|
281
|
+
if (!this.running) return;
|
|
282
|
+
|
|
283
|
+
log('Idle timeout for strand %s', strandId);
|
|
284
|
+
|
|
285
|
+
// Transition to idle. The callback can reject (e.g. a future idle handler
|
|
286
|
+
// that releases resources); catch so the timer chain never unhandled-rejects.
|
|
287
|
+
void this.callbacks.onIdle(strandId).then(() => {
|
|
288
|
+
// Schedule hibernate transition
|
|
289
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
290
|
+
if (timeouts.hibernateTimeout !== Infinity) {
|
|
291
|
+
this.scheduleHibernateTransition(instance);
|
|
292
|
+
}
|
|
293
|
+
}).catch((err) => {
|
|
294
|
+
log('onIdle failed for strand %s: %o', strandId, err);
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
private scheduleHibernateTransition(instance: StrandInstance): void {
|
|
299
|
+
const { strandId, latencyHint } = instance;
|
|
300
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
301
|
+
|
|
302
|
+
// Clear existing timer
|
|
303
|
+
this.clearTimer(strandId);
|
|
304
|
+
|
|
305
|
+
// Schedule hibernate transition
|
|
306
|
+
const timer = setTimeout(() => {
|
|
307
|
+
this.handleHibernateTimeout(instance);
|
|
308
|
+
}, timeouts.hibernateTimeout);
|
|
309
|
+
|
|
310
|
+
this.timers.set(strandId, timer);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
private handleHibernateTimeout(instance: StrandInstance): void {
|
|
314
|
+
const { strandId, latencyHint } = instance;
|
|
315
|
+
|
|
316
|
+
if (!this.running) return;
|
|
317
|
+
|
|
318
|
+
log('Hibernate timeout for strand %s', strandId);
|
|
319
|
+
|
|
320
|
+
// Transition to hibernating. onHibernate now releases strand-network
|
|
321
|
+
// resources (quiesce), so its close()/stop() can reject — catch so a failed
|
|
322
|
+
// hibernate logs instead of producing an unhandled rejection.
|
|
323
|
+
void this.callbacks.onHibernate(strandId).then(() => {
|
|
324
|
+
// Schedule periodic check-ins
|
|
325
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
326
|
+
if (timeouts.checkInInterval !== Infinity) {
|
|
327
|
+
this.scheduleCheckIn(instance);
|
|
328
|
+
}
|
|
329
|
+
}).catch((err) => {
|
|
330
|
+
log('onHibernate failed for strand %s: %o', strandId, err);
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Arm the next check-in for a hibernating strand. Each call is a single-shot
|
|
336
|
+
* `setTimeout` (not a fixed `setInterval`) so the period escalates per
|
|
337
|
+
* {@link runCheckIn} and a slow `onCheckIn` never overlaps the next tick.
|
|
338
|
+
*
|
|
339
|
+
* `delay` is omitted by the chain start ({@link handleHibernateTimeout}),
|
|
340
|
+
* defaulting to the base `checkInInterval` — which is why backoff naturally
|
|
341
|
+
* resets to base each fresh hibernation cycle, with no per-strand counter to
|
|
342
|
+
* clear on wake. Subsequent ticks pass the escalated, capped delay.
|
|
343
|
+
*/
|
|
344
|
+
private scheduleCheckIn(instance: StrandInstance, delay?: number): void {
|
|
345
|
+
const { strandId, latencyHint } = instance;
|
|
346
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
347
|
+
const currentDelay = delay ?? timeouts.checkInInterval;
|
|
348
|
+
|
|
349
|
+
// Replace any existing check-in timer (no-op if the firing timer rescheduled us).
|
|
350
|
+
const existing = this.checkInTimers.get(strandId);
|
|
351
|
+
if (existing) {
|
|
352
|
+
clearTimeout(existing);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const timer = setTimeout(() => {
|
|
356
|
+
void this.runCheckIn(instance, currentDelay);
|
|
357
|
+
}, currentDelay);
|
|
358
|
+
|
|
359
|
+
this.checkInTimers.set(strandId, timer);
|
|
360
|
+
instance.nextCheckIn = new Date(Date.now() + currentDelay);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Run a single check-in tick: invoke `onCheckIn` (a real resume → bounded
|
|
365
|
+
* sync window → re-hibernate-if-idle cycle in `CadreNode`) and AWAIT it before
|
|
366
|
+
* deciding the next step.
|
|
367
|
+
*
|
|
368
|
+
* - If the strand woke during the check-in (`onCheckIn` left it non-
|
|
369
|
+
* `hibernating`), stop the chain — the activity path has re-armed the
|
|
370
|
+
* idle/hibernate timers, and the next hibernation restarts the chain at the
|
|
371
|
+
* base delay (backoff reset).
|
|
372
|
+
* - Otherwise escalate the delay by `checkInBackoffFactor`, capped at
|
|
373
|
+
* `checkInMaxInterval`, and reschedule.
|
|
374
|
+
*/
|
|
375
|
+
private async runCheckIn(instance: StrandInstance, currentDelay: number): Promise<void> {
|
|
376
|
+
const { strandId, latencyHint } = instance;
|
|
377
|
+
if (!this.running) return;
|
|
378
|
+
|
|
379
|
+
log('Check-in for hibernating strand %s (delay=%dms)', strandId, currentDelay);
|
|
380
|
+
|
|
381
|
+
try {
|
|
382
|
+
await this.callbacks.onCheckIn(strandId);
|
|
383
|
+
} catch (err) {
|
|
384
|
+
// A failed check-in (e.g. resume threw) must not break the chain — log and
|
|
385
|
+
// fall through to reschedule the next, longer-delayed attempt.
|
|
386
|
+
log('onCheckIn failed for strand %s: %o', strandId, err);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if (!this.running) return;
|
|
390
|
+
|
|
391
|
+
// The check-in either woke the strand (CadreNode left it active) or left it
|
|
392
|
+
// hibernating. Inspect the shared instance the callback just mutated.
|
|
393
|
+
if (instance.status !== 'hibernating') {
|
|
394
|
+
log('Check-in woke strand %s; backoff resets on next hibernation', strandId);
|
|
395
|
+
this.checkInTimers.delete(strandId);
|
|
396
|
+
// The chain is stopping; drop the now-stale next-check-in advertisement so
|
|
397
|
+
// `getStrand` doesn't report a phantom check-in for a strand that is awake.
|
|
398
|
+
instance.nextCheckIn = undefined;
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const timeouts = this.getTimeouts(latencyHint);
|
|
403
|
+
const nextDelay = Math.min(
|
|
404
|
+
currentDelay * timeouts.checkInBackoffFactor,
|
|
405
|
+
timeouts.checkInMaxInterval
|
|
406
|
+
);
|
|
407
|
+
this.scheduleCheckIn(instance, nextDelay);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
private clearTimer(strandId: string): void {
|
|
411
|
+
const timer = this.timers.get(strandId);
|
|
412
|
+
if (timer) {
|
|
413
|
+
clearTimeout(timer);
|
|
414
|
+
this.timers.delete(strandId);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
private clearTimers(strandId: string): void {
|
|
419
|
+
this.clearTimer(strandId);
|
|
420
|
+
|
|
421
|
+
const checkInTimer = this.checkInTimers.get(strandId);
|
|
422
|
+
if (checkInTimer) {
|
|
423
|
+
clearTimeout(checkInTimer);
|
|
424
|
+
this.checkInTimers.delete(strandId);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Get the current status of hibernation tracking
|
|
430
|
+
*/
|
|
431
|
+
getStatus(): { enabled: boolean; trackedStrands: number } {
|
|
432
|
+
return {
|
|
433
|
+
enabled: this.config.enabled && this.running,
|
|
434
|
+
trackedStrands: this.timers.size + this.checkInTimers.size
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
}
|