@agentsignalnet/claude-plugin 0.2.108 → 0.2.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/scripts/inbox-daemon.mjs +91 -14
- package/scripts/inbox-hook.mjs +136 -5
- package/scripts/responder.mjs +133 -0
package/package.json
CHANGED
package/scripts/inbox-daemon.mjs
CHANGED
|
@@ -46,7 +46,8 @@ import { incumbent, livingWatchers, lockDir as sharedLockDir, apiBase } from './
|
|
|
46
46
|
// Answering when nobody is home. Off unless AGENTSIGNAL_RESPONDER says
|
|
47
47
|
// otherwise, so importing it costs nothing and changes nothing.
|
|
48
48
|
import {
|
|
49
|
-
settings as responderSettings, mayRespond,
|
|
49
|
+
settings as responderSettings, mayRespond, mayTryAgain, waitBeforeRetrying,
|
|
50
|
+
takeLock, releaseLock, heldBy,
|
|
50
51
|
record as recordResponder, readLedger, spawnResponder, hadItSwitchedOff,
|
|
51
52
|
} from './responder.mjs';
|
|
52
53
|
|
|
@@ -164,7 +165,19 @@ const FLOOR_MS = 250;
|
|
|
164
165
|
*/
|
|
165
166
|
const CANNOT_START = /Failed to authenticate|not logged in|command not found|ENOENT|Invalid API key/i;
|
|
166
167
|
|
|
167
|
-
/**
|
|
168
|
+
/**
|
|
169
|
+
* Why this machine cannot run a responder at all, once something proves it.
|
|
170
|
+
*
|
|
171
|
+
* `{ why, at, attempts }` rather than a string, and the two extra fields are
|
|
172
|
+
* the whole repair. As a string this was a latch with no way out: it was held
|
|
173
|
+
* "until the daemon restarts", by a daemon whose pidfile guard is specifically
|
|
174
|
+
* there to stop it being restarted. It gave up at 2026-08-21 23:32 and was
|
|
175
|
+
* still giving up two days later, through four messages it never mentioned in
|
|
176
|
+
* the ledger and through somebody signing back in.
|
|
177
|
+
*
|
|
178
|
+
* `at` and `attempts` let it come back on its own -- see `mayTryAgain`, which
|
|
179
|
+
* is where the arithmetic lives and where it is tested.
|
|
180
|
+
*/
|
|
168
181
|
let unavailable = null;
|
|
169
182
|
|
|
170
183
|
/**
|
|
@@ -402,12 +415,36 @@ async function tick() {
|
|
|
402
415
|
*/
|
|
403
416
|
async function considerResponding(watched) {
|
|
404
417
|
const config = responderSettings();
|
|
418
|
+
const now = Date.now();
|
|
405
419
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
420
|
+
/*
|
|
421
|
+
HAVING GIVEN UP ON THIS MACHINE, AND WHEN TO FIND OUT IF THAT STILL HOLDS.
|
|
422
|
+
|
|
423
|
+
Something proved a session cannot start here, so every message would fail
|
|
424
|
+
the same way and spend one of six hourly slots doing it. Skipping them is
|
|
425
|
+
right. Skipping them FOREVER was not, and that is what this used to do:
|
|
426
|
+
`if (unavailable) return`, with the flag cleared only by a daemon restart
|
|
427
|
+
that the pidfile guard makes sure never happens.
|
|
428
|
+
|
|
429
|
+
Now exactly one spawn is let through per interval to find out whether the
|
|
430
|
+
world changed, and the interval doubles while the answer keeps being no.
|
|
431
|
+
Advancing `at` before returning is what makes it ONE: the next cycle is two
|
|
432
|
+
seconds later and must not send a second probe after the same wait.
|
|
433
|
+
*/
|
|
434
|
+
const again = mayTryAgain(unavailable, now, config);
|
|
435
|
+
if (!again.ok) return;
|
|
409
436
|
|
|
410
|
-
|
|
437
|
+
if (unavailable) {
|
|
438
|
+
recordResponder({
|
|
439
|
+
event: 'retrying', at: now, why: unavailable.why,
|
|
440
|
+
attempts: unavailable.attempts, gave_up_ago_ms: now - unavailable.at,
|
|
441
|
+
});
|
|
442
|
+
say(
|
|
443
|
+
`trying one responder again after ${Math.round((now - unavailable.at) / 60_000)}m ` +
|
|
444
|
+
`(${unavailable.attempts} failed): ${String(unavailable.why).slice(0, 120)}`,
|
|
445
|
+
);
|
|
446
|
+
unavailable = { ...unavailable, at: now };
|
|
447
|
+
}
|
|
411
448
|
const ledger = readLedger();
|
|
412
449
|
const lockHeldBy = heldBy();
|
|
413
450
|
|
|
@@ -553,20 +590,60 @@ async function considerResponding(watched) {
|
|
|
553
590
|
Measured here: exit 1 in well under a second, with the daemon's own
|
|
554
591
|
arguments and the right working directory.
|
|
555
592
|
|
|
556
|
-
|
|
557
|
-
fixes
|
|
558
|
-
next time they open a session
|
|
559
|
-
|
|
593
|
+
It USED to be held until the daemon restarted, on the reasoning that
|
|
594
|
+
what fixes this is a person signing in, and that a restart is what
|
|
595
|
+
happens the next time they open a session. Both halves were wrong.
|
|
596
|
+
The daemon outlives sessions on purpose and a pidfile guard turns
|
|
597
|
+
every later SessionStart into 'already watching; nothing to do', so
|
|
598
|
+
the restart never came -- two days and one hour of a live daemon
|
|
599
|
+
evaluating nothing, four messages absent from the ledger entirely,
|
|
600
|
+
and signing back in changing none of it.
|
|
601
|
+
|
|
602
|
+
The objection recorded here against a timer -- "a timer would just
|
|
603
|
+
resume the same doomed spawns" -- was right about a timer that
|
|
604
|
+
resumes ALL of them, and that is not what replaced it. One spawn is
|
|
605
|
+
let through per interval, and the interval doubles while the answer
|
|
606
|
+
is still no. The cost of being wrong is one sub-second failure, not
|
|
607
|
+
six an hour; the cost of being right is answering again within five
|
|
608
|
+
minutes of somebody signing in.
|
|
560
609
|
*/
|
|
561
610
|
if (code !== 0 && why && CANNOT_START.test(why)) {
|
|
562
|
-
|
|
563
|
-
|
|
611
|
+
const attempts = (unavailable?.attempts ?? 0) + 1;
|
|
612
|
+
unavailable = { why, at: Date.now(), attempts };
|
|
613
|
+
const waitMs = waitBeforeRetrying(attempts, config);
|
|
614
|
+
const waitSaid = waitMs < 60_000
|
|
615
|
+
? `${Math.max(1, Math.round(waitMs / 1000))}s`
|
|
616
|
+
: `${Math.round(waitMs / 60_000)}m`;
|
|
617
|
+
recordResponder({
|
|
618
|
+
event: 'unavailable', at: Date.now(), why: why.slice(0, 300),
|
|
619
|
+
attempts, retry_in_ms: waitMs,
|
|
620
|
+
});
|
|
564
621
|
complain(
|
|
565
622
|
`the responder cannot start sessions on this machine: ${why.slice(0, 200)}. ` +
|
|
566
|
-
|
|
567
|
-
'problem, running `claude` once
|
|
623
|
+
`Trying one more in about ${waitSaid} (attempt ${attempts}). If that is a ` +
|
|
624
|
+
'sign-in problem, running `claude auth login` once is the fix, and it ' +
|
|
625
|
+
'takes effect on its own -- no restart needed.',
|
|
568
626
|
);
|
|
569
627
|
}
|
|
628
|
+
|
|
629
|
+
/*
|
|
630
|
+
AND THE OTHER DIRECTION, which had no code at all.
|
|
631
|
+
|
|
632
|
+
Nothing ever cleared `unavailable`, so even if a restart had happened
|
|
633
|
+
the flag only went from set to absent by the process dying. A session
|
|
634
|
+
that ran to completion is the only proof that this machine can start
|
|
635
|
+
one, and it is proof that arrives for free -- so take it, and reset
|
|
636
|
+
the backoff with it, or the next expiry starts counting from an hour
|
|
637
|
+
rather than from five minutes.
|
|
638
|
+
*/
|
|
639
|
+
if (code === 0 && unavailable) {
|
|
640
|
+
recordResponder({
|
|
641
|
+
event: 'available', at: Date.now(),
|
|
642
|
+
after_attempts: unavailable.attempts, was: String(unavailable.why).slice(0, 200),
|
|
643
|
+
});
|
|
644
|
+
say('a responder session completed; this machine can start them again');
|
|
645
|
+
unavailable = null;
|
|
646
|
+
}
|
|
570
647
|
recordResponder({
|
|
571
648
|
event: 'finished', at: Date.now(),
|
|
572
649
|
delivery_id: message.delivery_id, exit_code: code,
|
package/scripts/inbox-hook.mjs
CHANGED
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
* that matters -- a dropped interrupt looks exactly like no message.
|
|
39
39
|
*/
|
|
40
40
|
import {
|
|
41
|
-
readFileSync, writeFileSync, renameSync, mkdirSync, existsSync, appendFileSync,
|
|
41
|
+
readFileSync, writeFileSync, renameSync, mkdirSync, existsSync, appendFileSync, unlinkSync,
|
|
42
42
|
} from 'node:fs';
|
|
43
43
|
import { dirname, join } from 'node:path';
|
|
44
44
|
import { fileURLToPath } from 'node:url';
|
|
@@ -47,6 +47,44 @@ import { ensureWatcher, lockDir, apiBase } from './watchers.mjs';
|
|
|
47
47
|
|
|
48
48
|
const mode = process.argv[2] ?? 'check';
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* A RESPONDER'S SESSION IS NOT SOMEBODY WORKING, and every hook in this file
|
|
52
|
+
* is written for somebody working.
|
|
53
|
+
*
|
|
54
|
+
* `spawnResponder` has always set AGENTSIGNAL_FROM_RESPONDER=1 and nothing has
|
|
55
|
+
* ever read it. A marker with no reader is not a guard, and the cost of that
|
|
56
|
+
* was measured on 2026-08-23: the first responder to spawn in two days ran for
|
|
57
|
+
* twenty-one minutes at 0.0% CPU, answered nothing, and held the single-flight
|
|
58
|
+
* lock the whole time so every other message was refused
|
|
59
|
+
* `another_responder_running`. It left two orphaned `inbox-hook.mjs watch`
|
|
60
|
+
* processes behind, which is what named the cause.
|
|
61
|
+
*
|
|
62
|
+
* All three modes are wrong here, and only the first one is the hang:
|
|
63
|
+
*
|
|
64
|
+
* watch polls for ten minutes and only returns early if a message arrives.
|
|
65
|
+
* The comment below says a quiet channel "costs a sleeping process
|
|
66
|
+
* and nothing else", which is true of an interactive session and
|
|
67
|
+
* false of `claude --print`: that cannot exit while a hook of its
|
|
68
|
+
* own is outstanding. So the responder waits for a message to arrive
|
|
69
|
+
* before it will finish answering the message it already has. It is
|
|
70
|
+
* waiting for itself.
|
|
71
|
+
*
|
|
72
|
+
* check is worse and would have outlived the hang. It calls `markHandled`
|
|
73
|
+
* on every waiting message -- so a responder spawned to answer ONE
|
|
74
|
+
* delivery would quietly claim all the others, and `mayRespond`
|
|
75
|
+
* refuses a handled message forever. Messages nobody ever saw would
|
|
76
|
+
* be permanently marked as seen by a model.
|
|
77
|
+
*
|
|
78
|
+
* finish holds the turn open when something is waiting, which is the same
|
|
79
|
+
* hang wearing a different hat.
|
|
80
|
+
*
|
|
81
|
+
* A responder was told exactly which delivery to answer and must not be
|
|
82
|
+
* interrupted, must not claim anything else, and must not hold itself open.
|
|
83
|
+
* Exit 0 is silence in every mode -- `watch` reserves exit 2 for the wake --
|
|
84
|
+
* so one line covers all three.
|
|
85
|
+
*/
|
|
86
|
+
if (process.env.AGENTSIGNAL_FROM_RESPONDER === '1') process.exit(0);
|
|
87
|
+
|
|
50
88
|
/**
|
|
51
89
|
* The same candidate list the daemon and the MCP server use, and it must stay
|
|
52
90
|
* that way -- see the note in inbox-daemon.mjs. A hook reading a different
|
|
@@ -228,9 +266,92 @@ if (mode === 'check' || mode === 'finish') {
|
|
|
228
266
|
}
|
|
229
267
|
|
|
230
268
|
if (mode === 'watch') {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
269
|
+
/*
|
|
270
|
+
ONE WATCHER PER SESSION, NOT ONE PER TOOL CALL.
|
|
271
|
+
|
|
272
|
+
The line below used to read "a quiet channel costs a sleeping process and
|
|
273
|
+
nothing else", singular, and that is the assumption this replaces. This
|
|
274
|
+
hook is PostToolUse: it fires after EVERY tool call and then polls for ten
|
|
275
|
+
minutes. So the steady-state population is not one, it is the session's
|
|
276
|
+
tool-call rate times ten minutes, and each one is a node runtime.
|
|
277
|
+
|
|
278
|
+
Counted on 2026-08-23: 92 of them alive at once across four sessions, one
|
|
279
|
+
session holding 67, together 4.1GB of resident memory -- to read one small
|
|
280
|
+
JSON file once a second. No CPU to speak of; they sleep. Nothing was stuck
|
|
281
|
+
and nothing leaked, which is exactly why it went unnoticed for so long:
|
|
282
|
+
every single one expires on its own, correctly, ten minutes after the tool
|
|
283
|
+
call that started it.
|
|
284
|
+
|
|
285
|
+
THE DEADLINE MOVES RATHER THAN THE PROCESS MULTIPLYING. The naive fix --
|
|
286
|
+
"exit if somebody else is already watching" -- makes coverage WORSE than
|
|
287
|
+
today. A thirty-minute build starting nine minutes into an incumbent's life
|
|
288
|
+
would be covered for one minute instead of ten, and covering a long tool
|
|
289
|
+
call is the entire reason this mode exists. So a later hook does not exit
|
|
290
|
+
and leave; it pushes the incumbent's deadline out and then exits. Coverage
|
|
291
|
+
always runs to the last tool call plus ten minutes, which is what the
|
|
292
|
+
per-call version gave, with one process instead of sixty-seven.
|
|
293
|
+
|
|
294
|
+
Keyed on the session, because the wake is: exit 2 rewakes THIS session, so
|
|
295
|
+
a single machine-wide watcher would leave every other session unwakeable.
|
|
296
|
+
With no session id -- an older host, the SDK, a direct invocation -- there
|
|
297
|
+
is no safe key, and it falls through to the old behaviour. Wasteful beats
|
|
298
|
+
silent.
|
|
299
|
+
*/
|
|
300
|
+
const sessionKey = (process.env.CLAUDE_CODE_HOST_SESSION_ID ?? '').replace(/[^A-Za-z0-9_-]/g, '_');
|
|
301
|
+
const watchFile = sessionKey
|
|
302
|
+
? join(lockDir(), 'watching', `${sessionKey}.json`)
|
|
303
|
+
: null;
|
|
304
|
+
|
|
305
|
+
const readWatcher = () => {
|
|
306
|
+
try {
|
|
307
|
+
return JSON.parse(readFileSync(watchFile, 'utf8'));
|
|
308
|
+
} catch {
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
/** Whether that pid is still there. Signal 0 delivers nothing and only asks. */
|
|
314
|
+
const alive = (pid) => {
|
|
315
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
316
|
+
try {
|
|
317
|
+
process.kill(pid, 0);
|
|
318
|
+
return true;
|
|
319
|
+
} catch {
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
const until = () => Date.now() + WATCH_SECONDS * 1000;
|
|
325
|
+
|
|
326
|
+
if (watchFile) {
|
|
327
|
+
mkdirSync(dirname(watchFile), { recursive: true });
|
|
328
|
+
const held = readWatcher();
|
|
329
|
+
|
|
330
|
+
if (held && held.pid !== process.pid && alive(held.pid)) {
|
|
331
|
+
// Somebody is already watching for this session. Push their deadline out
|
|
332
|
+
// to cover the tool call that just finished, and stop.
|
|
333
|
+
writeAtomic(watchFile, { pid: held.pid, until: until() });
|
|
334
|
+
process.exit(0);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// Nobody, or a pid that died with its file still on disk -- a watcher lost
|
|
338
|
+
// to SIGKILL or a reboot leaves exactly that, and it is not an error.
|
|
339
|
+
writeAtomic(watchFile, { pid: process.pid, until: until() });
|
|
340
|
+
|
|
341
|
+
// Runs on process.exit() too, so the wake path below cleans up as well.
|
|
342
|
+
process.on('exit', () => {
|
|
343
|
+
if (readWatcher()?.pid === process.pid) {
|
|
344
|
+
try {
|
|
345
|
+
unlinkSync(watchFile);
|
|
346
|
+
} catch {
|
|
347
|
+
/* somebody else tidied it, or took over */
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Exit 2 is the wake. Anything else is silence.
|
|
354
|
+
const fallbackDeadline = until();
|
|
234
355
|
|
|
235
356
|
const poll = () => {
|
|
236
357
|
const messages = waiting('interrupt');
|
|
@@ -245,7 +366,17 @@ if (mode === 'watch') {
|
|
|
245
366
|
);
|
|
246
367
|
process.exit(2);
|
|
247
368
|
}
|
|
248
|
-
|
|
369
|
+
|
|
370
|
+
if (watchFile) {
|
|
371
|
+
const held = readWatcher();
|
|
372
|
+
// The file is the lease. Gone, or reassigned, means this one is no
|
|
373
|
+
// longer the session's watcher and should stop rather than shadow it.
|
|
374
|
+
if (!held || held.pid !== process.pid) process.exit(0);
|
|
375
|
+
if (Date.now() >= held.until) process.exit(0);
|
|
376
|
+
} else if (Date.now() >= fallbackDeadline) {
|
|
377
|
+
process.exit(0);
|
|
378
|
+
}
|
|
379
|
+
|
|
249
380
|
setTimeout(poll, 1000);
|
|
250
381
|
};
|
|
251
382
|
|
package/scripts/responder.mjs
CHANGED
|
@@ -99,6 +99,40 @@ export const DEFAULTS = {
|
|
|
99
99
|
peerCooldownMs: 10 * 60_000,
|
|
100
100
|
// A held lock older than this is debris from a killed responder, not a peer.
|
|
101
101
|
staleLockMs: 20 * 60_000,
|
|
102
|
+
/*
|
|
103
|
+
HOW LONG A MACHINE THAT CANNOT START SESSIONS IS GIVEN UP ON.
|
|
104
|
+
|
|
105
|
+
Something proving a session cannot start -- an expired login, a `claude`
|
|
106
|
+
that is not on PATH -- is true for every message and will be true for the
|
|
107
|
+
next one, so the daemon stops trying. That much was right, and the reason
|
|
108
|
+
is the hourly cap: six doomed spawns an hour is the whole budget spent on
|
|
109
|
+
sessions that never ran.
|
|
110
|
+
|
|
111
|
+
What was wrong was that it never started again. The flag was held "until
|
|
112
|
+
the daemon restarts", on the reasoning that a restart is what happens the
|
|
113
|
+
next time somebody opens a session. It is not. The daemon deliberately
|
|
114
|
+
outlives sessions and guards itself with a pidfile, so every later
|
|
115
|
+
SessionStart hook finds it already running and says so -- 'already watching
|
|
116
|
+
under pid N; nothing to do', hundreds of times. The recovery path assumed
|
|
117
|
+
the very thing the guard exists to prevent.
|
|
118
|
+
|
|
119
|
+
Measured on 2026-08-23: an expiry at 2026-08-21 23:32 left a daemon that
|
|
120
|
+
ran for two days and one hour without evaluating a single message. Four
|
|
121
|
+
messages arrived in that window and none of them appear in the ledger at
|
|
122
|
+
all -- not answered, not refused, not mentioned. Signing back in fixed
|
|
123
|
+
nothing, because nothing re-read the credential.
|
|
124
|
+
|
|
125
|
+
So it retries, and the retry doubles. One spawn is let through after five
|
|
126
|
+
minutes to find out whether the world changed; if it has not, the next wait
|
|
127
|
+
is ten minutes, then twenty, up to the cap. A machine signed out for a day
|
|
128
|
+
spends about ten spawns discovering that rather than 144, and a machine
|
|
129
|
+
somebody just signed back into starts answering within five minutes.
|
|
130
|
+
|
|
131
|
+
The cap matters as much as the base: without it the wait doubles into
|
|
132
|
+
weeks, which is a latch again with extra steps.
|
|
133
|
+
*/
|
|
134
|
+
unavailableRetryMs: 5 * 60_000,
|
|
135
|
+
unavailableRetryCapMs: 60 * 60_000,
|
|
102
136
|
};
|
|
103
137
|
|
|
104
138
|
/** What the dashboard's This machine page wrote, if anything. */
|
|
@@ -158,6 +192,8 @@ export function settings(env = process.env, file = undefined) {
|
|
|
158
192
|
perDay: num('AGENTSIGNAL_RESPONDER_PER_DAY', 'responder_per_day', DEFAULTS.perDay),
|
|
159
193
|
peerCooldownMs: num('AGENTSIGNAL_RESPONDER_PEER_COOLDOWN_MS', 'responder_peer_cooldown_ms', DEFAULTS.peerCooldownMs),
|
|
160
194
|
staleLockMs: num('AGENTSIGNAL_RESPONDER_STALE_LOCK_MS', 'responder_stale_lock_ms', DEFAULTS.staleLockMs),
|
|
195
|
+
unavailableRetryMs: num('AGENTSIGNAL_RESPONDER_UNAVAILABLE_RETRY_MS', 'responder_unavailable_retry_ms', DEFAULTS.unavailableRetryMs),
|
|
196
|
+
unavailableRetryCapMs: num('AGENTSIGNAL_RESPONDER_UNAVAILABLE_RETRY_CAP_MS', 'responder_unavailable_retry_cap_ms', DEFAULTS.unavailableRetryCapMs),
|
|
161
197
|
};
|
|
162
198
|
}
|
|
163
199
|
|
|
@@ -273,6 +309,53 @@ export function mayRespond(message, world) {
|
|
|
273
309
|
return { ok: true, peer };
|
|
274
310
|
}
|
|
275
311
|
|
|
312
|
+
/**
|
|
313
|
+
* How long to leave a machine alone after a spawn proved it cannot start one.
|
|
314
|
+
*
|
|
315
|
+
* Doubling from the base, capped. `attempts` is how many times a spawn has now
|
|
316
|
+
* proved it, so the first failure waits the base and not zero.
|
|
317
|
+
*
|
|
318
|
+
* The exponent is bounded before it is used rather than after. `2 ** 1024` is
|
|
319
|
+
* Infinity, and `Math.min(Infinity, cap)` is the cap -- so the arithmetic
|
|
320
|
+
* happens to survive, which is not the same as being correct, and the next
|
|
321
|
+
* person to change this line should not have to work that out. A daemon left
|
|
322
|
+
* running for a month is exactly the case this whole function exists for.
|
|
323
|
+
*/
|
|
324
|
+
export function waitBeforeRetrying(attempts, config) {
|
|
325
|
+
const base = Math.max(0, config.unavailableRetryMs ?? DEFAULTS.unavailableRetryMs);
|
|
326
|
+
const cap = Math.max(0, config.unavailableRetryCapMs ?? DEFAULTS.unavailableRetryCapMs);
|
|
327
|
+
const doublings = Math.min(Math.max(0, (attempts ?? 1) - 1), 30);
|
|
328
|
+
return Math.min(base * 2 ** doublings, cap);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* May the daemon try one more spawn after giving up on this machine?
|
|
333
|
+
*
|
|
334
|
+
* Pure, like `mayRespond`, and separate from it on purpose: this is a fact
|
|
335
|
+
* about the MACHINE, and `mayRespond` answers about a MESSAGE. Folding it in
|
|
336
|
+
* would mean every message re-deciding a question that has one answer for all
|
|
337
|
+
* of them, and the ledger could no longer say which of the two refused.
|
|
338
|
+
*
|
|
339
|
+
* `state` is null when nothing has proved anything, which is the normal case.
|
|
340
|
+
*/
|
|
341
|
+
export function mayTryAgain(state, now, config) {
|
|
342
|
+
if (!state) return { ok: true, why: 'never_gave_up' };
|
|
343
|
+
|
|
344
|
+
const wait = waitBeforeRetrying(state.attempts ?? 1, config);
|
|
345
|
+
const waited = now - (state.at ?? 0);
|
|
346
|
+
|
|
347
|
+
// A clock that went backwards -- a laptop waking, an NTP correction -- must
|
|
348
|
+
// not read as "waited a negative amount and so must wait longer than the
|
|
349
|
+
// interval". Treating it as due is the safe direction: the cost of an early
|
|
350
|
+
// retry is one spawn, and the cost of a late one is a responder that is
|
|
351
|
+
// silently off for however far the clock jumped.
|
|
352
|
+
if (waited >= 0 && waited < wait) {
|
|
353
|
+
return { ok: false, why: 'unavailable', retryInMs: wait - waited };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
return { ok: true, why: 'retrying_after_unavailable', attempts: state.attempts ?? 1 };
|
|
357
|
+
}
|
|
358
|
+
|
|
276
359
|
/** Exclusive-create, the same brake the watcher uses. `wx` fails if it exists. */
|
|
277
360
|
export function takeLock(path = LOCK, now = Date.now()) {
|
|
278
361
|
mkdirSync(ROOT, { recursive: true });
|
|
@@ -482,6 +565,56 @@ if (process.argv[2] === '--self-test') {
|
|
|
482
565
|
{ now, config: base,
|
|
483
566
|
ledger: [{ event: 'spawn', at: now - 1000, peer: 'u_peer' }] }).ok);
|
|
484
567
|
|
|
568
|
+
/*
|
|
569
|
+
GIVING UP ON A MACHINE, AND COMING BACK.
|
|
570
|
+
|
|
571
|
+
The bug these lock down: `unavailable` was a boolean with no way out, held
|
|
572
|
+
"until the daemon restarts" by a daemon that a pidfile guard keeps from
|
|
573
|
+
ever restarting. Two days and one hour of a live daemon evaluating nothing,
|
|
574
|
+
four messages that reached the spool and appear nowhere in the ledger, and
|
|
575
|
+
signing back in changed none of it.
|
|
576
|
+
|
|
577
|
+
The first check below is the one that fails on the old code: it asked
|
|
578
|
+
whether a boolean was set, so "gave up an hour ago" and "gave up a moment
|
|
579
|
+
ago" were the same value and neither ever cleared.
|
|
580
|
+
*/
|
|
581
|
+
const retryCfg = { ...base, unavailableRetryMs: 300_000, unavailableRetryCapMs: 3_600_000 };
|
|
582
|
+
const gaveUp = (at, attempts) => ({ why: 'Failed to authenticate', at, attempts });
|
|
583
|
+
|
|
584
|
+
check('a machine nothing has disproved is simply allowed',
|
|
585
|
+
mayTryAgain(null, now, retryCfg).ok);
|
|
586
|
+
|
|
587
|
+
check('THE FIX: having given up is not permanent',
|
|
588
|
+
mayTryAgain(gaveUp(now - 400_000, 1), now, retryCfg).ok);
|
|
589
|
+
|
|
590
|
+
check('but it is not reconsidered every cycle either',
|
|
591
|
+
!mayTryAgain(gaveUp(now - 1000, 1), now, retryCfg).ok);
|
|
592
|
+
|
|
593
|
+
check('and the refusal says how long is left rather than just no',
|
|
594
|
+
mayTryAgain(gaveUp(now - 100_000, 1), now, retryCfg).retryInMs === 200_000);
|
|
595
|
+
|
|
596
|
+
check('a second failure waits twice as long',
|
|
597
|
+
waitBeforeRetrying(2, retryCfg) === 600_000);
|
|
598
|
+
|
|
599
|
+
check('so a machine signed out all week is not probed all week',
|
|
600
|
+
waitBeforeRetrying(20, retryCfg) === retryCfg.unavailableRetryCapMs);
|
|
601
|
+
|
|
602
|
+
check('and the cap is a real ceiling rather than an overflow that happens to work',
|
|
603
|
+
Number.isFinite(waitBeforeRetrying(5000, retryCfg)));
|
|
604
|
+
|
|
605
|
+
/*
|
|
606
|
+
A laptop that slept, or an NTP step. `waited` goes negative, and the
|
|
607
|
+
tempting arithmetic -- `waited < wait` -- is TRUE for every negative
|
|
608
|
+
number, so a backwards clock would re-latch the machine for as long as the
|
|
609
|
+
jump lasted. Due is the safe reading: one wasted spawn against a responder
|
|
610
|
+
that is silently off for an unknown length of time.
|
|
611
|
+
*/
|
|
612
|
+
check('a clock that went backwards does not extend the wait',
|
|
613
|
+
mayTryAgain(gaveUp(now + 900_000, 1), now, retryCfg).ok);
|
|
614
|
+
|
|
615
|
+
check('the retry interval is configurable from the settings file',
|
|
616
|
+
settings({ AGENTSIGNAL_RESPONDER_UNAVAILABLE_RETRY_MS: '1000' }).unavailableRetryMs === 1000);
|
|
617
|
+
|
|
485
618
|
// The settings file is the front door now, so it gets the same treatment as
|
|
486
619
|
// the brakes: prove it opens, prove it stays shut, and prove a typo in it
|
|
487
620
|
// fails to OFF rather than to something.
|