@phnx-labs/agents-cli 1.22.117 → 1.22.118
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +235 -0
- package/dist/bootstrap.js +4 -11
- package/dist/commands/browser.d.ts +52 -70
- package/dist/commands/browser.js +192 -3562
- package/dist/commands/daemon.d.ts +39 -2
- package/dist/commands/daemon.js +72 -73
- package/dist/commands/events.js +1 -1
- package/dist/commands/exec.js +53 -1
- package/dist/commands/repo.js +13 -49
- package/dist/commands/route.js +5 -5
- package/dist/commands/routines.js +39 -12
- package/dist/commands/sessions-render.d.ts +1 -1
- package/dist/commands/sessions-render.js +0 -1
- package/dist/commands/sessions.test-fixture.d.ts +0 -1
- package/dist/commands/sessions.test-fixture.js +0 -37
- package/dist/commands/setup-browser.d.ts +17 -12
- package/dist/commands/setup-browser.js +52 -87
- package/dist/commands/setup-preferences.d.ts +0 -17
- package/dist/commands/setup-preferences.js +18 -49
- package/dist/commands/setup.js +9 -9
- package/dist/commands/ssh.js +10 -1
- package/dist/commands/sync.js +6 -44
- package/dist/commands/teams.js +0 -1
- package/dist/commands/update.d.ts +19 -0
- package/dist/commands/update.js +34 -6
- package/dist/commands/view.d.ts +12 -1
- package/dist/commands/view.js +47 -2
- package/dist/index.js +107 -26
- package/dist/lib/account-capabilities.js +0 -2
- package/dist/lib/account-catalog.d.ts +38 -0
- package/dist/lib/account-catalog.js +110 -26
- package/dist/lib/account-provider-registry.js +1 -1
- package/dist/lib/account-registry.d.ts +1 -1
- package/dist/lib/account-registry.js +36 -2
- package/dist/lib/accounting/rotate.js +22 -8
- package/dist/lib/accounting/usage-ingest.d.ts +6 -1
- package/dist/lib/accounting/usage-ingest.js +115 -45
- package/dist/lib/accounting/usage-sync.d.ts +134 -31
- package/dist/lib/accounting/usage-sync.js +227 -66
- package/dist/lib/accounting/usage.d.ts +16 -3
- package/dist/lib/accounting/usage.js +26 -8
- package/dist/lib/accounts/slots.d.ts +8 -0
- package/dist/lib/accounts/slots.js +17 -0
- package/dist/lib/add-dir.js +0 -1
- package/dist/lib/agent-cli-commands.js +0 -1
- package/dist/lib/agent-spec/agents.d.ts +6 -7
- package/dist/lib/agent-spec/agents.js +6 -60
- package/dist/lib/auth-health.d.ts +69 -1
- package/dist/lib/auth-health.js +133 -4
- package/dist/lib/browser/context.d.ts +82 -0
- package/dist/lib/browser/context.js +88 -0
- package/dist/lib/browser/paths.d.ts +25 -0
- package/dist/lib/browser/paths.js +51 -0
- package/dist/lib/browser/record.d.ts +38 -0
- package/dist/lib/browser/record.js +57 -0
- package/dist/lib/browser/sessions-list.js +1 -2
- package/dist/lib/browser-client.d.ts +158 -0
- package/dist/lib/browser-client.js +234 -0
- package/dist/lib/channels/owner-sink.d.ts +1 -18
- package/dist/lib/channels/owner-sink.js +29 -82
- package/dist/lib/channels/providers/rush.d.ts +7 -7
- package/dist/lib/channels/providers/rush.js +127 -57
- package/dist/lib/claude-statusline.d.ts +1 -1
- package/dist/lib/claude-statusline.js +15 -3
- package/dist/lib/commands.js +1 -1
- package/dist/lib/crabbox/lease.js +1 -1
- package/dist/lib/crabbox/runtimes.js +1 -3
- package/dist/lib/daemon/account-state-daemon-service.js +1 -0
- package/dist/lib/daemon/auth-sync-service.d.ts +21 -18
- package/dist/lib/daemon/auth-sync-service.js +37 -43
- package/dist/lib/daemon/daemon.d.ts +25 -14
- package/dist/lib/daemon/daemon.js +92 -154
- package/dist/lib/daemon/runner.d.ts +2 -2
- package/dist/lib/daemon/runner.js +32 -9
- package/dist/lib/daemon/self-update-service.d.ts +35 -34
- package/dist/lib/daemon/self-update-service.js +40 -36
- package/dist/lib/daemon/service.d.ts +12 -8
- package/dist/lib/daemon/supervisor.d.ts +65 -54
- package/dist/lib/daemon/supervisor.js +136 -171
- package/dist/lib/daemon/testdata/process-view-start.js +5 -2
- package/dist/lib/daemon/usage-sync-service.d.ts +21 -30
- package/dist/lib/daemon/usage-sync-service.js +46 -78
- package/dist/lib/daemon-health.d.ts +22 -5
- package/dist/lib/daemon-health.js +47 -3
- package/dist/lib/daemon-services.d.ts +1 -1
- package/dist/lib/daemon-services.js +0 -10
- package/dist/lib/daemon-ticks.js +4 -2
- package/dist/lib/devices/doctor-findings.js +8 -11
- package/dist/lib/devices/harness-inventory.js +14 -5
- package/dist/lib/exec-account-home.js +5 -0
- package/dist/lib/exec.js +0 -11
- package/dist/lib/feed/events.js +0 -1
- package/dist/lib/feed/tool-activity.js +5 -2
- package/dist/lib/fleet-shared-state.d.ts +45 -0
- package/dist/lib/fleet-shared-state.js +78 -9
- package/dist/lib/fs-atomic.js +25 -0
- package/dist/lib/harness-auth-capabilities.js +0 -1
- package/dist/lib/helper-versions.js +1 -1
- package/dist/lib/hooks/cache.d.ts +7 -0
- package/dist/lib/hooks/cache.js +27 -2
- package/dist/lib/hooks/install.js +7 -1
- package/dist/lib/hosts/ready.d.ts +38 -8
- package/dist/lib/hosts/ready.js +62 -14
- package/dist/lib/installations/active-check.d.ts +27 -0
- package/dist/lib/installations/active-check.js +61 -0
- package/dist/lib/installations/migrate.js +6 -3
- package/dist/lib/installations/shims.js +9 -7
- package/dist/lib/installations/update.js +21 -15
- package/dist/lib/installations/versions.js +12 -1
- package/dist/lib/manifest.js +1 -1
- package/dist/lib/models.js +0 -2
- package/dist/lib/open-url.js +28 -54
- package/dist/lib/profiles.js +0 -2
- package/dist/lib/refresh.js +1 -1
- package/dist/lib/routine-readiness.d.ts +35 -0
- package/dist/lib/routine-readiness.js +74 -16
- package/dist/lib/rules/compile.js +6 -0
- package/dist/lib/sandbox.js +0 -2
- package/dist/lib/secrets-policy.d.ts +89 -11
- package/dist/lib/secrets-policy.js +175 -28
- package/dist/lib/session/db.d.ts +1 -1
- package/dist/lib/session/db.js +65 -18
- package/dist/lib/session/discover.js +1 -283
- package/dist/lib/session/mirror.d.ts +4 -3
- package/dist/lib/session/mirror.js +4 -3
- package/dist/lib/session/parse.d.ts +0 -2
- package/dist/lib/session/parse.js +1 -138
- package/dist/lib/session/throughput.d.ts +1 -3
- package/dist/lib/session/throughput.js +0 -20
- package/dist/lib/session/types.d.ts +2 -2
- package/dist/lib/session/types.js +2 -2
- package/dist/lib/sessions-client.d.ts +60 -1
- package/dist/lib/sessions-client.js +227 -4
- package/dist/lib/setup-tool-status.js +5 -10
- package/dist/lib/signin-badge.d.ts +3 -2
- package/dist/lib/signin-badge.js +6 -3
- package/dist/lib/smart-launch.js +4 -1
- package/dist/lib/staleness/detectors/commands.d.ts +1 -1
- package/dist/lib/staleness/detectors/hooks.d.ts +1 -1
- package/dist/lib/staleness/detectors/mcp.d.ts +1 -1
- package/dist/lib/staleness/detectors/permissions.d.ts +1 -1
- package/dist/lib/staleness/detectors/plugins.d.ts +1 -1
- package/dist/lib/staleness/detectors/rules.d.ts +1 -1
- package/dist/lib/staleness/detectors/skills.d.ts +1 -1
- package/dist/lib/staleness/detectors/subagents.d.ts +1 -1
- package/dist/lib/staleness/detectors/workflows.d.ts +1 -1
- package/dist/lib/staleness/writers/commands.d.ts +1 -1
- package/dist/lib/staleness/writers/hooks.d.ts +1 -1
- package/dist/lib/staleness/writers/mcp.d.ts +1 -1
- package/dist/lib/staleness/writers/permissions.d.ts +1 -1
- package/dist/lib/staleness/writers/plugins.d.ts +1 -1
- package/dist/lib/staleness/writers/rules.d.ts +1 -1
- package/dist/lib/staleness/writers/skills.d.ts +1 -1
- package/dist/lib/staleness/writers/sources.js +5 -0
- package/dist/lib/staleness/writers/subagents.d.ts +1 -1
- package/dist/lib/staleness/writers/workflows.d.ts +1 -1
- package/dist/lib/state.d.ts +6 -4
- package/dist/lib/state.js +7 -17
- package/dist/lib/teams/parsers.d.ts +1 -1
- package/dist/lib/teams/parsers.js +1 -136
- package/dist/lib/teams/placement-probe.js +50 -15
- package/dist/lib/teams/scheduler.d.ts +7 -0
- package/dist/lib/types.d.ts +10 -1
- package/dist/lib/types.js +1 -1
- package/dist/lib/view-types.d.ts +32 -0
- package/dist/lib/watchdog/read.js +0 -1
- package/dist/lib/watchdog/rotate.js +1 -2
- package/dist/lib/watchdog/runner.js +2 -2
- package/dist/lib/watchdog/watchdog.d.ts +1 -1
- package/dist/lib/watchdog/watchdogTail.js +0 -31
- package/package.json +1 -1
- package/scripts/postinstall.js +8 -8
- package/dist/browser.d.ts +0 -2
- package/dist/browser.js +0 -17
- package/dist/commands/browser-picker.d.ts +0 -18
- package/dist/commands/browser-picker.js +0 -97
- package/dist/lib/browser/arc-discovery.d.ts +0 -52
- package/dist/lib/browser/arc-discovery.js +0 -197
- package/dist/lib/browser/arc-dom.d.ts +0 -14
- package/dist/lib/browser/arc-dom.js +0 -121
- package/dist/lib/browser/caller-identity.d.ts +0 -40
- package/dist/lib/browser/caller-identity.js +0 -175
- package/dist/lib/browser/cdp.d.ts +0 -51
- package/dist/lib/browser/cdp.js +0 -254
- package/dist/lib/browser/chrome.d.ts +0 -141
- package/dist/lib/browser/chrome.js +0 -704
- package/dist/lib/browser/chromium-discovery.d.ts +0 -28
- package/dist/lib/browser/chromium-discovery.js +0 -96
- package/dist/lib/browser/devices.d.ts +0 -34
- package/dist/lib/browser/devices.js +0 -61
- package/dist/lib/browser/domain-skills.d.ts +0 -71
- package/dist/lib/browser/domain-skills.js +0 -195
- package/dist/lib/browser/drivers/arc.d.ts +0 -40
- package/dist/lib/browser/drivers/arc.js +0 -276
- package/dist/lib/browser/drivers/firefox.d.ts +0 -99
- package/dist/lib/browser/drivers/firefox.js +0 -377
- package/dist/lib/browser/drivers/local.d.ts +0 -55
- package/dist/lib/browser/drivers/local.js +0 -266
- package/dist/lib/browser/drivers/ssh.d.ts +0 -120
- package/dist/lib/browser/drivers/ssh.js +0 -467
- package/dist/lib/browser/editor.d.ts +0 -3
- package/dist/lib/browser/editor.js +0 -50
- package/dist/lib/browser/ffmpeg.d.ts +0 -12
- package/dist/lib/browser/ffmpeg.js +0 -184
- package/dist/lib/browser/firefox-discovery.d.ts +0 -69
- package/dist/lib/browser/firefox-discovery.js +0 -162
- package/dist/lib/browser/har.d.ts +0 -85
- package/dist/lib/browser/har.js +0 -77
- package/dist/lib/browser/hygiene.d.ts +0 -97
- package/dist/lib/browser/hygiene.js +0 -153
- package/dist/lib/browser/index.d.ts +0 -5
- package/dist/lib/browser/index.js +0 -5
- package/dist/lib/browser/input.d.ts +0 -7
- package/dist/lib/browser/input.js +0 -92
- package/dist/lib/browser/ipc.d.ts +0 -204
- package/dist/lib/browser/ipc.js +0 -1453
- package/dist/lib/browser/login-detection.d.ts +0 -87
- package/dist/lib/browser/login-detection.js +0 -274
- package/dist/lib/browser/profiles.d.ts +0 -422
- package/dist/lib/browser/profiles.js +0 -1157
- package/dist/lib/browser/refs.d.ts +0 -89
- package/dist/lib/browser/refs.js +0 -191
- package/dist/lib/browser/registry.d.ts +0 -70
- package/dist/lib/browser/registry.js +0 -252
- package/dist/lib/browser/remote-control.d.ts +0 -59
- package/dist/lib/browser/remote-control.js +0 -85
- package/dist/lib/browser/resolve-target.d.ts +0 -66
- package/dist/lib/browser/resolve-target.js +0 -264
- package/dist/lib/browser/runtime-state.d.ts +0 -291
- package/dist/lib/browser/runtime-state.js +0 -584
- package/dist/lib/browser/secret-ref.d.ts +0 -10
- package/dist/lib/browser/secret-ref.js +0 -14
- package/dist/lib/browser/service.d.ts +0 -812
- package/dist/lib/browser/service.js +0 -4693
- package/dist/lib/browser/stream.d.ts +0 -17
- package/dist/lib/browser/stream.js +0 -72
- package/dist/lib/browser/task-index.d.ts +0 -78
- package/dist/lib/browser/task-index.js +0 -199
- package/dist/lib/browser/types.d.ts +0 -647
- package/dist/lib/browser/types.js +0 -61
- package/dist/lib/browser/upload.d.ts +0 -29
- package/dist/lib/browser/upload.js +0 -298
- package/dist/lib/daemon/browser-ipc-service.d.ts +0 -20
- package/dist/lib/daemon/browser-ipc-service.js +0 -46
- package/dist/lib/daemon/browser-task-reap-service.d.ts +0 -14
- package/dist/lib/daemon/browser-task-reap-service.js +0 -26
- package/dist/lib/fleet-shared-repo-sync.d.ts +0 -55
- package/dist/lib/fleet-shared-repo-sync.js +0 -490
|
@@ -1,53 +1,53 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* ServiceSupervisor (RUSH-3193 P1).
|
|
2
|
+
* ServiceSupervisor (RUSH-3193 P1, PHNX-4116).
|
|
3
3
|
*
|
|
4
4
|
* Owns the timer for every registered `PeriodicService`, replacing the bare
|
|
5
|
-
* `setInterval` closures in `runDaemon()`. Two failure modes motivated it,
|
|
6
|
-
*
|
|
5
|
+
* `setInterval` closures in `runDaemon()`. Two failure modes motivated it, both
|
|
6
|
+
* observed in production, and each is now handled without ever leaving a service
|
|
7
|
+
* silently dark:
|
|
7
8
|
*
|
|
8
9
|
* - A throw escaping a tick's local try/catch used to hit the process-wide
|
|
9
|
-
* `uncaughtException` handler and `process.exit(1)` the whole daemon,
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* `
|
|
16
|
-
* (
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* is drained separately in the background; `finishTick` is version-guarded so
|
|
24
|
-
* its late settlement can never disturb the fresh tick the restart started.
|
|
10
|
+
* `uncaughtException` handler and `process.exit(1)` the whole daemon, taking
|
|
11
|
+
* every OTHER service down with it. Here a thrown tick is caught per-service,
|
|
12
|
+
* recorded via `recordFailure`, and the service keeps ticking on its own
|
|
13
|
+
* interval — a throw is recoverable, so it is retried in place on the next
|
|
14
|
+
* tick and no sibling is disturbed.
|
|
15
|
+
* - A tick that HANGS on an unbounded await (SSH, keychain) used to latch its
|
|
16
|
+
* in-flight guard `true` forever, silently freezing that one service for the
|
|
17
|
+
* daemon's life (observed ~51h). A hang cannot be retried in-process — the
|
|
18
|
+
* promise may never settle — so when a tick (or a `start()`/`restart()`
|
|
19
|
+
* lifecycle call) BREACHES its deadline the supervisor records the cause,
|
|
20
|
+
* flushes it to disk, and EXITS the process (code 70). systemd
|
|
21
|
+
* (`Restart=always`, `RestartSec=30`, `StartLimitIntervalSec=0`) and launchd
|
|
22
|
+
* (`KeepAlive` + `ThrottleInterval=30`) restart the daemon within ~30s and the
|
|
23
|
+
* wedged service comes back healthy with it.
|
|
25
24
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* `
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
25
|
+
* This is the PHNX-4116 model: there is NO `parked` state and NO in-process
|
|
26
|
+
* backoff restart. A supervised service is `idle` before start, `running` while
|
|
27
|
+
* ticking, or `stopped` after a live disable / shutdown — a hang is not a fourth
|
|
28
|
+
* state to sit in, it is a reason to hand the daemon back to its OS supervisor,
|
|
29
|
+
* which is what actually "just works" like systemd/launchd. `start()`/`stop()`/
|
|
30
|
+
* `restart()` stay bounded by {@link ServiceSupervisorOptions.lifecycleDeadlineMs}
|
|
31
|
+
* so a wedged bind or close cannot stall daemon startup or shutdown; a start /
|
|
32
|
+
* restart lifecycle breach exits the same way a tick breach does.
|
|
32
33
|
*/
|
|
33
|
-
import { recordSubsystemOk, recordSubsystemError, recordSubsystemState } from '../daemon-health.js';
|
|
34
|
+
import { recordSubsystemOk, recordSubsystemError, recordSubsystemState, recordDaemonRestart } from '../daemon-health.js';
|
|
34
35
|
import { isPeriodicService } from './service.js';
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Exit code the supervisor uses when a deadline breach hands the daemon back to
|
|
38
|
+
* its OS supervisor for a restart. 70 (`EX_SOFTWARE`) marks an internal software
|
|
39
|
+
* fault, distinct from a clean shutdown (0) or a config error.
|
|
40
|
+
*/
|
|
41
|
+
const DEADLINE_EXIT_CODE = 70;
|
|
38
42
|
const DEFAULT_LIFECYCLE_DEADLINE_MS = 30_000;
|
|
39
43
|
export class ServiceSupervisor {
|
|
40
44
|
registry = new Map();
|
|
41
45
|
ctx = null;
|
|
42
|
-
parkAfterFailures;
|
|
43
|
-
backoffBaseMs;
|
|
44
|
-
backoffMaxMs;
|
|
45
46
|
lifecycleDeadlineMs;
|
|
47
|
+
exit;
|
|
46
48
|
constructor(opts = {}) {
|
|
47
|
-
this.parkAfterFailures = opts.parkAfterFailures ?? DEFAULT_PARK_AFTER_FAILURES;
|
|
48
|
-
this.backoffBaseMs = opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
49
|
-
this.backoffMaxMs = opts.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
|
|
50
49
|
this.lifecycleDeadlineMs = opts.lifecycleDeadlineMs ?? DEFAULT_LIFECYCLE_DEADLINE_MS;
|
|
50
|
+
this.exit = opts.exit ?? ((code) => process.exit(code));
|
|
51
51
|
}
|
|
52
52
|
/**
|
|
53
53
|
* Race `op` against a deadline. On breach the returned promise rejects with a
|
|
@@ -85,7 +85,6 @@ export class ServiceSupervisor {
|
|
|
85
85
|
state: options.enabled === false ? 'stopped' : 'idle',
|
|
86
86
|
lastRunMs: 0,
|
|
87
87
|
consecutiveFailures: 0,
|
|
88
|
-
restartAttempts: 0,
|
|
89
88
|
inFlight: false,
|
|
90
89
|
everStarted: false,
|
|
91
90
|
idleWaiters: [],
|
|
@@ -117,7 +116,11 @@ export class ServiceSupervisor {
|
|
|
117
116
|
for (const id of this.registry.keys())
|
|
118
117
|
await this.stopOne(id, true);
|
|
119
118
|
}
|
|
120
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* Force one service to restart right now — drives `agents daemon services
|
|
121
|
+
* restart <id>` (RUSH-3193 P4). A plain stop + start, with no intermediate
|
|
122
|
+
* `parked` state (PHNX-4116).
|
|
123
|
+
*/
|
|
121
124
|
async restartOne(id) {
|
|
122
125
|
const entry = this.registry.get(id);
|
|
123
126
|
if (!entry)
|
|
@@ -125,26 +128,8 @@ export class ServiceSupervisor {
|
|
|
125
128
|
if (entry.inFlight) {
|
|
126
129
|
throw new Error(`service '${id}' cannot restart while a tick is still in flight`);
|
|
127
130
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
// multiply the tick rate on every operator-requested restart.
|
|
131
|
-
if (entry.timer) {
|
|
132
|
-
clearInterval(entry.timer);
|
|
133
|
-
entry.timer = undefined;
|
|
134
|
-
}
|
|
135
|
-
if (entry.startupTimer) {
|
|
136
|
-
clearTimeout(entry.startupTimer);
|
|
137
|
-
entry.startupTimer = undefined;
|
|
138
|
-
}
|
|
139
|
-
if (entry.restartTimer) {
|
|
140
|
-
clearTimeout(entry.restartTimer);
|
|
141
|
-
entry.restartTimer = undefined;
|
|
142
|
-
}
|
|
143
|
-
// Block a timer callback already queued in this event-loop turn while the
|
|
144
|
-
// service tears down and starts again.
|
|
145
|
-
entry.state = 'parked';
|
|
146
|
-
recordSubsystemState(id, 'parked');
|
|
147
|
-
await this.attemptRestart(id);
|
|
131
|
+
await this.stopOne(id, false);
|
|
132
|
+
await this.startOne(id);
|
|
148
133
|
}
|
|
149
134
|
/** Whether `id` has a lifecycle owner on this supervisor, running or stopped. */
|
|
150
135
|
isRegistered(id) {
|
|
@@ -156,8 +141,7 @@ export class ServiceSupervisor {
|
|
|
156
141
|
}
|
|
157
142
|
/**
|
|
158
143
|
* Resolve when the service's current tick has really settled. Daemon control
|
|
159
|
-
* edges use this to queue a requested live transition without polling
|
|
160
|
-
* treating a deadline race as cancellation.
|
|
144
|
+
* edges use this to queue a requested live transition without polling.
|
|
161
145
|
*/
|
|
162
146
|
async awaitIdle(id) {
|
|
163
147
|
const entry = this.registry.get(id);
|
|
@@ -165,11 +149,10 @@ export class ServiceSupervisor {
|
|
|
165
149
|
throw new Error(`service '${id}' is not registered`);
|
|
166
150
|
// Resolve from clearInFlight (which also flips `inFlight` false) rather than
|
|
167
151
|
// from the tick promise directly: awaiting the tick promise resumed this
|
|
168
|
-
// caller on a shorter microtask chain than the tick's own finally
|
|
169
|
-
//
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
// itself makes the ordering deterministic.
|
|
152
|
+
// caller on a shorter microtask chain than the tick's own finally, so a
|
|
153
|
+
// queued `.then(stop)` could run before `inFlight` was cleared and `stop()`
|
|
154
|
+
// would throw "cannot stop while a tick is still in flight" (PHNX-3608 — the
|
|
155
|
+
// SIGHUP live-disable regression). Waiting on the guard makes it deterministic.
|
|
173
156
|
if (!entry.inFlight)
|
|
174
157
|
return;
|
|
175
158
|
await new Promise((resolve) => { entry.idleWaiters.push(resolve); });
|
|
@@ -229,11 +212,10 @@ export class ServiceSupervisor {
|
|
|
229
212
|
await this.withDeadline(() => entry.service.start(ctx), this.lifecycleDeadlineMs, `service '${id}' start`);
|
|
230
213
|
}
|
|
231
214
|
catch (err) {
|
|
232
|
-
// A thrown OR wedged start()
|
|
233
|
-
//
|
|
234
|
-
//
|
|
235
|
-
this.
|
|
236
|
-
this.park(id);
|
|
215
|
+
// A thrown OR wedged start()/restart() cannot leave the service silently
|
|
216
|
+
// dark, and there is no `parked` state to fall back to (PHNX-4116): record
|
|
217
|
+
// the cause and exit so systemd/launchd restart the whole daemon and retry.
|
|
218
|
+
this.exitForRestart(id, err);
|
|
237
219
|
return;
|
|
238
220
|
}
|
|
239
221
|
entry.everStarted = true;
|
|
@@ -273,14 +255,18 @@ export class ServiceSupervisor {
|
|
|
273
255
|
clearInterval(entry.timer);
|
|
274
256
|
entry.timer = undefined;
|
|
275
257
|
}
|
|
276
|
-
if (entry.restartTimer) {
|
|
277
|
-
clearTimeout(entry.restartTimer);
|
|
278
|
-
entry.restartTimer = undefined;
|
|
279
|
-
}
|
|
280
258
|
if (entry.startupTimer) {
|
|
281
259
|
clearTimeout(entry.startupTimer);
|
|
282
260
|
entry.startupTimer = undefined;
|
|
283
261
|
}
|
|
262
|
+
// Cancel the in-flight tick's deadline timer too — leaving it armed lets it
|
|
263
|
+
// fire `exitForRestart` mid-shutdown if the hung tick ignores the abort
|
|
264
|
+
// below (PHNX-4116). The `state === 'stopped'` guard in `runTick` is the
|
|
265
|
+
// belt to this braces.
|
|
266
|
+
if (entry.deadlineTimer) {
|
|
267
|
+
clearTimeout(entry.deadlineTimer);
|
|
268
|
+
entry.deadlineTimer = undefined;
|
|
269
|
+
}
|
|
284
270
|
entry.state = 'stopped';
|
|
285
271
|
recordSubsystemState(id, 'stopped');
|
|
286
272
|
// Signal any in-flight tick to unwind — a cooperating tick threads this into
|
|
@@ -315,16 +301,16 @@ export class ServiceSupervisor {
|
|
|
315
301
|
entry.timer = setInterval(() => { void this.runTick(id); }, entry.service.intervalMs);
|
|
316
302
|
}
|
|
317
303
|
/**
|
|
318
|
-
* Run one tick under a hard deadline (PHNX-3608). The tick receives
|
|
319
|
-
* `AbortSignal` that is aborted when the deadline elapses, so a cooperating
|
|
320
|
-
* tick can bound its own I/O and unwind. The deadline itself is
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
304
|
+
* Run one tick under a hard deadline (PHNX-3608, PHNX-4116). The tick receives
|
|
305
|
+
* an `AbortSignal` that is aborted when the deadline elapses, so a cooperating
|
|
306
|
+
* tick can bound its own I/O and unwind. The deadline itself is enforced with
|
|
307
|
+
* `Promise.race` — JS cannot forcibly cancel an arbitrary await.
|
|
308
|
+
*
|
|
309
|
+
* A tick that THROWS is recoverable: it is recorded and the interval keeps
|
|
310
|
+
* firing, so the service retries on its next tick. A tick that BREACHES its
|
|
311
|
+
* deadline is a hang that can never be retried in-process (the promise may
|
|
312
|
+
* never settle), so the supervisor exits the process (`exitForRestart`) and
|
|
313
|
+
* lets systemd/launchd restart the whole daemon.
|
|
328
314
|
*/
|
|
329
315
|
async runTick(id) {
|
|
330
316
|
const entry = this.registry.get(id);
|
|
@@ -339,13 +325,12 @@ export class ServiceSupervisor {
|
|
|
339
325
|
entry.inFlight = true;
|
|
340
326
|
const controller = new AbortController();
|
|
341
327
|
entry.activeController = controller;
|
|
342
|
-
let deadlineTimer;
|
|
343
328
|
let timedOut = false;
|
|
344
329
|
const tickPromise = periodicService.tick(ctx, controller.signal);
|
|
345
330
|
entry.activeTick = tickPromise;
|
|
346
331
|
try {
|
|
347
332
|
const deadline = new Promise((_, reject) => {
|
|
348
|
-
deadlineTimer = setTimeout(() => {
|
|
333
|
+
entry.deadlineTimer = setTimeout(() => {
|
|
349
334
|
timedOut = true;
|
|
350
335
|
reject(new Error(`tick exceeded deadline of ${periodicService.deadlineMs}ms`));
|
|
351
336
|
}, periodicService.deadlineMs);
|
|
@@ -357,88 +342,38 @@ export class ServiceSupervisor {
|
|
|
357
342
|
recordSubsystemOk(id);
|
|
358
343
|
}
|
|
359
344
|
catch (err) {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
//
|
|
373
|
-
//
|
|
345
|
+
// A stopped service must never exit the process: `stopOne` may have
|
|
346
|
+
// force-stopped this tick (aborting its signal) while the deadline timer
|
|
347
|
+
// was still armed, and a hang that ignored the abort would otherwise let
|
|
348
|
+
// that timer fire `exitForRestart` mid-shutdown — wrong exit code, a
|
|
349
|
+
// spurious restart-ledger entry, and `handleShutdown` cleanup skipped
|
|
350
|
+
// (PHNX-4116). This is the belt to `stopOne`'s clearTimeout braces. Read
|
|
351
|
+
// the state fresh: `stopOne` mutated it across the `await` above, past the
|
|
352
|
+
// narrowing the top-of-function `state !== 'running'` guard applied.
|
|
353
|
+
const stoppedDuringTick = this.registry.get(id)?.state === 'stopped';
|
|
354
|
+
if (timedOut && !stoppedDuringTick) {
|
|
355
|
+
// Abort the runaway tick's signal so a cooperating tick can unwind, then
|
|
356
|
+
// hand the daemon back to its OS supervisor: a hang has no in-process
|
|
357
|
+
// recovery (PHNX-4116). `exitForRestart` calls `process.exit`, so nothing
|
|
358
|
+
// below runs in production; the injected test exit returns, and the
|
|
359
|
+
// `finally` still releases the in-flight guard.
|
|
374
360
|
controller.abort();
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
this.clearInFlight(entry);
|
|
378
|
-
// Re-read the entry so its `state` is the full union, not the `'running'`
|
|
379
|
-
// narrowing from this function's entry-guard — park() set it to 'parked'.
|
|
380
|
-
const current = this.registry.get(id);
|
|
381
|
-
if (current && current.state === 'parked' && !current.restartTimer)
|
|
382
|
-
this.scheduleRestart(id);
|
|
383
|
-
void tickPromise.then(() => undefined, () => undefined);
|
|
384
|
-
}
|
|
385
|
-
else {
|
|
386
|
-
this.finishTick(id, tickPromise);
|
|
361
|
+
this.exitForRestart(id, err);
|
|
362
|
+
return;
|
|
387
363
|
}
|
|
364
|
+
// A throw is recoverable — record it and keep ticking on the next interval.
|
|
365
|
+
// A deadline breach of an already-stopped service is intentionally dropped.
|
|
366
|
+
if (!timedOut)
|
|
367
|
+
this.recordFailure(entry, id, err);
|
|
388
368
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
return;
|
|
394
|
-
entry.state = 'parked';
|
|
395
|
-
recordSubsystemState(id, 'parked');
|
|
396
|
-
if (entry.timer) {
|
|
397
|
-
clearInterval(entry.timer);
|
|
398
|
-
entry.timer = undefined;
|
|
399
|
-
}
|
|
400
|
-
this.ctx?.log('WARN', `service '${id}' parked after ${entry.consecutiveFailures} consecutive failure(s)`);
|
|
401
|
-
if (!entry.inFlight)
|
|
402
|
-
this.scheduleRestart(id);
|
|
403
|
-
}
|
|
404
|
-
scheduleRestart(id) {
|
|
405
|
-
const entry = this.registry.get(id);
|
|
406
|
-
if (!entry)
|
|
407
|
-
return;
|
|
408
|
-
const delay = Math.min(this.backoffMaxMs, this.backoffBaseMs * 2 ** entry.restartAttempts);
|
|
409
|
-
entry.restartAttempts += 1;
|
|
410
|
-
entry.restartTimer = setTimeout(() => { void this.attemptRestart(id); }, delay);
|
|
411
|
-
}
|
|
412
|
-
async attemptRestart(id) {
|
|
413
|
-
const entry = this.registry.get(id);
|
|
414
|
-
const ctx = this.ctx;
|
|
415
|
-
if (!entry || !ctx || entry.state === 'stopped')
|
|
416
|
-
return;
|
|
417
|
-
try {
|
|
418
|
-
// A wedged restart() must not silently stall the circuit breaker — bound
|
|
419
|
-
// it and treat a breach as another failed restart, retried on backoff.
|
|
420
|
-
await this.withDeadline(() => entry.service.restart(), this.lifecycleDeadlineMs, `service '${id}' restart`);
|
|
421
|
-
entry.everStarted = true; // restart() is stop()+start() on the service — a successful one means it is running again and owes a stop() at shutdown.
|
|
422
|
-
entry.state = 'running';
|
|
423
|
-
recordSubsystemState(id, 'running');
|
|
424
|
-
entry.consecutiveFailures = 0;
|
|
425
|
-
entry.restartAttempts = 0;
|
|
426
|
-
entry.lastError = undefined;
|
|
427
|
-
if (isPeriodicService(entry.service)) {
|
|
428
|
-
this.scheduleTimer(id);
|
|
429
|
-
void this.runTick(id);
|
|
430
|
-
}
|
|
431
|
-
else {
|
|
432
|
-
entry.lastRunMs = Date.now();
|
|
369
|
+
finally {
|
|
370
|
+
if (entry.deadlineTimer) {
|
|
371
|
+
clearTimeout(entry.deadlineTimer);
|
|
372
|
+
entry.deadlineTimer = undefined;
|
|
433
373
|
}
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
catch (err) {
|
|
438
|
-
this.recordFailure(entry, id, err);
|
|
439
|
-
entry.state = 'parked';
|
|
440
|
-
recordSubsystemState(id, 'parked');
|
|
441
|
-
this.scheduleRestart(id);
|
|
374
|
+
entry.activeTick = undefined;
|
|
375
|
+
entry.activeController = undefined;
|
|
376
|
+
this.clearInFlight(entry);
|
|
442
377
|
}
|
|
443
378
|
}
|
|
444
379
|
recordFailure(entry, id, err) {
|
|
@@ -448,14 +383,44 @@ export class ServiceSupervisor {
|
|
|
448
383
|
recordSubsystemError(id, message);
|
|
449
384
|
this.ctx?.log('WARN', `service '${id}' failed: ${message}`);
|
|
450
385
|
}
|
|
451
|
-
|
|
386
|
+
/**
|
|
387
|
+
* Record a deadline breach durably and exit the process so systemd/launchd
|
|
388
|
+
* restart the whole daemon (PHNX-4116). `recordSubsystemError` writes
|
|
389
|
+
* `health.json` synchronously (atomic write under a file lock), so the cause is
|
|
390
|
+
* on disk before we exit; `recordDaemonRestart` appends to the restart ledger
|
|
391
|
+
* `agents daemon status` reads for "restarts in the last 24h". Every timer is
|
|
392
|
+
* frozen first so no further tick fires between this decision and process death
|
|
393
|
+
* — and so an injected test `exit` (which does not actually exit) still observes
|
|
394
|
+
* exactly one call.
|
|
395
|
+
*/
|
|
396
|
+
exitForRestart(id, err) {
|
|
397
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
452
398
|
const entry = this.registry.get(id);
|
|
453
|
-
if (
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
399
|
+
if (entry) {
|
|
400
|
+
entry.consecutiveFailures += 1;
|
|
401
|
+
entry.lastError = cause;
|
|
402
|
+
}
|
|
403
|
+
recordSubsystemError(id, cause);
|
|
404
|
+
recordDaemonRestart(id, cause);
|
|
405
|
+
this.ctx?.log('ERROR', `service '${id}' breached its deadline (${cause}); exiting (code ${DEADLINE_EXIT_CODE}) for a supervised restart`);
|
|
406
|
+
this.freezeTimers();
|
|
407
|
+
this.exit(DEADLINE_EXIT_CODE);
|
|
408
|
+
}
|
|
409
|
+
/** Clear every service's interval / startup timer — the process is exiting. */
|
|
410
|
+
freezeTimers() {
|
|
411
|
+
for (const entry of this.registry.values()) {
|
|
412
|
+
if (entry.timer) {
|
|
413
|
+
clearInterval(entry.timer);
|
|
414
|
+
entry.timer = undefined;
|
|
415
|
+
}
|
|
416
|
+
if (entry.startupTimer) {
|
|
417
|
+
clearTimeout(entry.startupTimer);
|
|
418
|
+
entry.startupTimer = undefined;
|
|
419
|
+
}
|
|
420
|
+
if (entry.deadlineTimer) {
|
|
421
|
+
clearTimeout(entry.deadlineTimer);
|
|
422
|
+
entry.deadlineTimer = undefined;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
460
425
|
}
|
|
461
426
|
}
|
|
@@ -18,7 +18,10 @@ fs.writeFileSync(path.join(process.env.HOME, 'test-child.pid'), String(started.p
|
|
|
18
18
|
try {
|
|
19
19
|
if (!cold)
|
|
20
20
|
assert.ok(fs.existsSync(path.join(getDaemonDir(), 'health.json')), 'ordinary launcher publishes health before daemon starts');
|
|
21
|
-
|
|
21
|
+
// Daemon-liveness socket: the browser IPC socket left with the standalone
|
|
22
|
+
// browser CLI (PHNX-4101), so this waits on the feed-stream hub socket the
|
|
23
|
+
// daemon binds unconditionally on boot.
|
|
24
|
+
const socket = path.join(getCacheDir(), 'helpers', 'feed', 'feed-stream.sock');
|
|
22
25
|
let ready = false;
|
|
23
26
|
for (let i = 0; i < 80 && !ready; i++) {
|
|
24
27
|
ready = await new Promise(resolve => {
|
|
@@ -31,7 +34,7 @@ try {
|
|
|
31
34
|
if (!ready)
|
|
32
35
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
33
36
|
}
|
|
34
|
-
assert.ok(ready, 'ordinary fresh-home daemon must serve its
|
|
37
|
+
assert.ok(ready, 'ordinary fresh-home daemon must serve its feed-stream hub socket');
|
|
35
38
|
assert.equal(fs.readFileSync(path.join(getDaemonDir(), 'daemon.pid'), 'utf8').trim(), String(started.pid));
|
|
36
39
|
const owner = JSON.parse(fs.readFileSync(path.join(getTerminalsDir(), 'process-view.json'), 'utf8'));
|
|
37
40
|
assert.equal(owner.pidNamespace, fs.readlinkSync('/proc/self/ns/pid'));
|
|
@@ -1,29 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fleet
|
|
3
|
-
* PHNX-3792 session mirror).
|
|
2
|
+
* Fleet state exchange as a `PeriodicService` (PHNX-3392 usage-sync,
|
|
3
|
+
* PHNX-3792 session mirror, PHNX-4051 auth verdict, PHNX-4116 SSH transport).
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
5
|
+
* Each tick: (1) refreshes every field this box owns in its own
|
|
6
|
+
* `devices/<device>/daemon-state.json` — a headed box's Claude usage snapshot,
|
|
7
|
+
* EVERY box's lightweight session digests, and the reserved-auth readiness
|
|
8
|
+
* verdict; (2) on a headed box (`personal`/`desktop`) only, dials every dialable
|
|
9
|
+
* peer in parallel with `agents __usage-ingest --reply`, sending that envelope
|
|
10
|
+
* on stdin and storing each peer's reply as `devices/<peer>/daemon-state.json`
|
|
11
|
+
* stamped `receivedAt` (`exchangeFleetStateWithPeers`); (3) folds the peers'
|
|
12
|
+
* session digests into the local index so the picker renders remote-host
|
|
13
|
+
* previews inline. A worker never initiates: its state leaves the box only as
|
|
14
|
+
* the reply to a headed peer's dial, and a headed peer's usage rows arrive on
|
|
15
|
+
* that same dial.
|
|
16
16
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* workers never received a fresh usage snapshot, which the 40-min placement gate
|
|
23
|
-
* turned into "no ready device". Folding the auth verdict into this single
|
|
24
|
-
* committer removes the second caller entirely. Auth-sync keeps its non-git
|
|
25
|
-
* duties (worker-slot reconcile + the credential SSH pushes) under its own
|
|
26
|
-
* deadline and circuit breaker.
|
|
17
|
+
* There is no git in this tick. The exchange used to be a commit/rebase/push of
|
|
18
|
+
* the fleet-synced user repo, which needed a cross-process lock, a 45 s
|
|
19
|
+
* process-tree deadline, an 8-minute kickoff offset from auth-sync to dodge that
|
|
20
|
+
* lock, and untracked-collision backups — and still wedged every clone behind a
|
|
21
|
+
* bloated remote. Peers that time out are skipped this tick; none blocks another.
|
|
27
22
|
*/
|
|
28
23
|
import { BasePeriodicService, type DaemonContext } from './service.js';
|
|
29
24
|
import type { DaemonServiceId } from '../daemon-services.js';
|
|
@@ -36,17 +31,13 @@ import type { DaemonServiceId } from '../daemon-services.js';
|
|
|
36
31
|
* second literal, so the two files can't drift out of sync with each other.
|
|
37
32
|
*/
|
|
38
33
|
export declare const USAGE_SYNC_TICK_MS: number;
|
|
39
|
-
/**
|
|
40
|
-
|
|
41
|
-
* shared-repo lock. Auth fires at T+1m, T+16m, …; usage at T+8m, T+23m, …
|
|
42
|
-
* The git exchange deadline is 45s, so a 7-minute gap is the whole point.
|
|
43
|
-
*/
|
|
44
|
-
export declare const USAGE_SYNC_KICKOFF_MS: number;
|
|
34
|
+
/** Let the daemon's registry/device probes settle before the first fan-out. */
|
|
35
|
+
export declare const USAGE_SYNC_KICKOFF_MS = 90000;
|
|
45
36
|
export declare class UsageSyncService extends BasePeriodicService {
|
|
46
37
|
readonly id: DaemonServiceId;
|
|
47
38
|
readonly intervalMs: number;
|
|
48
39
|
readonly deadlineMs: number;
|
|
49
|
-
readonly startupDelayMs
|
|
40
|
+
readonly startupDelayMs = 90000;
|
|
50
41
|
protected onStart(_ctx: DaemonContext): Promise<void>;
|
|
51
42
|
protected onStop(): Promise<void>;
|
|
52
43
|
protected onTick(ctx: DaemonContext): Promise<void>;
|