@phnx-labs/agents-cli 1.22.46 → 1.22.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +663 -0
- package/LICENSE +102 -182
- package/README.md +96 -49
- package/dist/bootstrap.js +4 -2
- package/dist/cli/command-registry.d.ts +2 -1
- package/dist/cli/command-registry.js +6 -2
- package/dist/commands/accounts.d.ts +58 -0
- package/dist/commands/accounts.js +243 -25
- package/dist/commands/artifacts.js +7 -6
- package/dist/commands/attach.js +1 -1
- package/dist/commands/auth.js +7 -5
- package/dist/commands/browser-sessions-picker.js +9 -7
- package/dist/commands/browser.js +711 -62
- package/dist/commands/config.d.ts +0 -1
- package/dist/commands/config.js +26 -24
- package/dist/commands/cost.d.ts +1 -1
- package/dist/commands/daemon.d.ts +21 -0
- package/dist/commands/daemon.js +147 -12
- package/dist/commands/detach.d.ts +13 -0
- package/dist/commands/detach.js +11 -8
- package/dist/commands/exec.js +107 -53
- package/dist/commands/feed-watch.d.ts +2 -0
- package/dist/commands/feed-watch.js +28 -0
- package/dist/commands/feed.js +27 -0
- package/dist/commands/feedback.js +3 -19
- package/dist/commands/focus.d.ts +14 -0
- package/dist/commands/focus.js +48 -5
- package/dist/commands/go.d.ts +3 -0
- package/dist/commands/go.js +16 -3
- package/dist/commands/insights.d.ts +1 -1
- package/dist/commands/insights.js +2 -2
- package/dist/commands/inspect.js +2 -2
- package/dist/commands/lease.d.ts +1 -1
- package/dist/commands/lease.js +7 -6
- package/dist/commands/mailboxes.js +4 -3
- package/dist/commands/menubar.js +11 -4
- package/dist/commands/projects.d.ts +16 -1
- package/dist/commands/projects.js +58 -64
- package/dist/commands/resume.js +1 -1
- package/dist/commands/route.js +4 -4
- package/dist/commands/routines.js +4 -3
- package/dist/commands/run-broadcast.d.ts +1 -1
- package/dist/commands/run-broadcast.js +1 -1
- package/dist/commands/sessions-picker-factory.d.ts +2 -1
- package/dist/commands/sessions-picker-factory.js +1 -1
- package/dist/commands/sessions-picker.d.ts +22 -0
- package/dist/commands/sessions-picker.js +48 -9
- package/dist/commands/sessions-stop.d.ts +24 -0
- package/dist/commands/sessions-stop.js +66 -0
- package/dist/commands/sessions-trace.d.ts +82 -0
- package/dist/commands/sessions-trace.js +387 -0
- package/dist/commands/sessions.js +5 -0
- package/dist/commands/setup-secrets.js +23 -0
- package/dist/commands/share.d.ts +20 -5
- package/dist/commands/share.js +152 -52
- package/dist/commands/ssh.d.ts +15 -0
- package/dist/commands/ssh.js +243 -18
- package/dist/commands/teams.js +34 -2
- package/dist/commands/traces.d.ts +2 -0
- package/dist/commands/traces.js +184 -0
- package/dist/commands/view.d.ts +2 -0
- package/dist/commands/view.js +32 -62
- package/dist/commands/watchdog.js +9 -14
- package/dist/index.d.ts +1 -0
- package/dist/index.js +5 -0
- package/dist/lib/account-capabilities.d.ts +4 -0
- package/dist/lib/account-capabilities.js +13 -9
- package/dist/lib/account-registry.d.ts +2 -0
- package/dist/lib/account-registry.js +30 -2
- package/dist/lib/accounting/rotate.d.ts +9 -0
- package/dist/lib/accounting/rotate.js +12 -2
- package/dist/lib/accounting/usage.d.ts +84 -6
- package/dist/lib/accounting/usage.js +293 -75
- package/dist/lib/agent-modes.js +29 -1
- package/dist/lib/agent-spec/agents.d.ts +4 -3
- package/dist/lib/agent-spec/agents.js +25 -15
- package/dist/lib/analytics/mix-commands.d.ts +1 -1
- package/dist/lib/analytics/mix-commands.js +2 -2
- package/dist/lib/auth-health.d.ts +13 -3
- package/dist/lib/auth-health.js +50 -8
- package/dist/lib/browser/devices.d.ts +19 -0
- package/dist/lib/browser/devices.js +23 -0
- package/dist/lib/browser/drivers/ssh.d.ts +16 -2
- package/dist/lib/browser/drivers/ssh.js +36 -9
- package/dist/lib/browser/ipc.d.ts +1 -1
- package/dist/lib/browser/ipc.js +40 -9
- package/dist/lib/browser/profiles.d.ts +104 -65
- package/dist/lib/browser/profiles.js +277 -165
- package/dist/lib/browser/registry.d.ts +39 -0
- package/dist/lib/browser/registry.js +149 -0
- package/dist/lib/browser/remote-control.d.ts +28 -2
- package/dist/lib/browser/remote-control.js +35 -3
- package/dist/lib/browser/resolve-target.d.ts +68 -0
- package/dist/lib/browser/resolve-target.js +250 -0
- package/dist/lib/browser/runtime-state.d.ts +44 -11
- package/dist/lib/browser/runtime-state.js +98 -15
- package/dist/lib/browser/service.d.ts +65 -9
- package/dist/lib/browser/service.js +196 -61
- package/dist/lib/browser/sessions-list.d.ts +1 -3
- package/dist/lib/browser/sessions-list.js +3 -16
- package/dist/lib/browser/task-index.d.ts +70 -0
- package/dist/lib/browser/task-index.js +180 -0
- package/dist/lib/browser/types.d.ts +12 -1
- package/dist/lib/byok-usage.js +2 -2
- package/dist/lib/claude-statusline.d.ts +29 -0
- package/dist/lib/claude-statusline.js +139 -0
- package/dist/lib/cloud/host.d.ts +1 -1
- package/dist/lib/cloud/host.js +1 -1
- package/dist/lib/cloud/rush.js +1 -1
- package/dist/lib/codex-policy.d.ts +11 -1
- package/dist/lib/codex-policy.js +20 -5
- package/dist/lib/computer/computer-rpc.js +2 -2
- package/dist/lib/computer/download.d.ts +15 -15
- package/dist/lib/computer/download.js +35 -98
- package/dist/lib/computer/ssh-tunnel.d.ts +0 -3
- package/dist/lib/computer/ssh-tunnel.js +12 -22
- package/dist/lib/config-keys.d.ts +6 -11
- package/dist/lib/config-keys.js +18 -19
- package/dist/lib/crabbox/cli.d.ts +1 -1
- package/dist/lib/crabbox/cli.js +2 -2
- package/dist/lib/crabbox/lease.d.ts +9 -0
- package/dist/lib/crabbox/lease.js +70 -15
- package/dist/lib/crabbox/runtimes.js +1 -1
- package/dist/lib/daemon/account-state-daemon-service.d.ts +16 -0
- package/dist/lib/daemon/account-state-daemon-service.js +26 -0
- package/dist/lib/daemon/browser-ipc-service.d.ts +20 -0
- package/dist/lib/daemon/browser-ipc-service.js +46 -0
- package/dist/lib/daemon/daemon.d.ts +3 -0
- package/dist/lib/daemon/daemon.js +212 -444
- package/dist/lib/daemon/device-probe-service.d.ts +26 -0
- package/dist/lib/daemon/device-probe-service.js +47 -0
- package/dist/lib/daemon/keychain-reap-service.d.ts +17 -0
- package/dist/lib/daemon/keychain-reap-service.js +32 -0
- package/dist/lib/daemon/monitor-engine-service.d.ts +22 -0
- package/dist/lib/daemon/monitor-engine-service.js +29 -0
- package/dist/lib/daemon/runner.js +15 -2
- package/dist/lib/daemon/secrets-broker-service.d.ts +20 -0
- package/dist/lib/daemon/secrets-broker-service.js +66 -0
- package/dist/lib/daemon/self-heal-service.d.ts +27 -0
- package/dist/lib/daemon/self-heal-service.js +48 -0
- package/dist/lib/daemon/service.d.ts +102 -0
- package/dist/lib/daemon/service.js +98 -0
- package/dist/lib/daemon/session-index-service.d.ts +22 -0
- package/dist/lib/daemon/session-index-service.js +40 -0
- package/dist/lib/daemon/state-dir-check-service.d.ts +43 -0
- package/dist/lib/daemon/state-dir-check-service.js +63 -0
- package/dist/lib/daemon/supervisor.d.ts +87 -0
- package/dist/lib/daemon/supervisor.js +298 -0
- package/dist/lib/daemon/watchdog-service.d.ts +19 -0
- package/dist/lib/daemon/watchdog-service.js +40 -0
- package/dist/lib/daemon-health.d.ts +15 -0
- package/dist/lib/daemon-health.js +29 -5
- package/dist/lib/daemon-services.d.ts +8 -1
- package/dist/lib/daemon-services.js +55 -0
- package/dist/lib/daemon-ticks.d.ts +13 -2
- package/dist/lib/daemon-ticks.js +22 -20
- package/dist/lib/device-config.d.ts +25 -10
- package/dist/lib/device-config.js +90 -39
- package/dist/lib/devices/config-migration.d.ts +7 -0
- package/dist/lib/devices/config-migration.js +67 -3
- package/dist/lib/devices/health.d.ts +27 -2
- package/dist/lib/devices/health.js +39 -9
- package/dist/lib/devices/interactive-host.d.ts +15 -0
- package/dist/lib/devices/interactive-host.js +53 -0
- package/dist/lib/devices/registry.d.ts +61 -4
- package/dist/lib/devices/registry.js +94 -38
- package/dist/lib/devices/stats-cache.d.ts +26 -0
- package/dist/lib/devices/stats-cache.js +59 -7
- package/dist/lib/devices/worker-pick.d.ts +48 -0
- package/dist/lib/devices/worker-pick.js +113 -0
- package/dist/lib/exec.d.ts +70 -14
- package/dist/lib/exec.js +110 -38
- package/dist/lib/feed/answer.d.ts +29 -0
- package/dist/lib/feed/answer.js +145 -0
- package/dist/lib/feed/attention.d.ts +83 -0
- package/dist/lib/feed/attention.js +0 -0
- package/dist/lib/feed/feed.d.ts +109 -0
- package/dist/lib/feed/feed.js +154 -3
- package/dist/lib/feed/pr-status.d.ts +14 -0
- package/dist/lib/feed/pr-status.js +47 -0
- package/dist/lib/feed/watch.d.ts +59 -0
- package/dist/lib/feed/watch.js +180 -0
- package/dist/lib/fleet/capture.js +6 -0
- package/dist/lib/fleet/remote-login.js +8 -3
- package/dist/lib/fleet/types.d.ts +17 -0
- package/dist/lib/harness/adapter.d.ts +11 -0
- package/dist/lib/harness/adapters/claude.js +38 -24
- package/dist/lib/harness/adapters/codex.js +2 -4
- package/dist/lib/harness/adapters/cursor.js +8 -11
- package/dist/lib/helper-download.d.ts +108 -0
- package/dist/lib/helper-download.js +194 -0
- package/dist/lib/helper-versions.d.ts +52 -0
- package/dist/lib/helper-versions.js +58 -0
- package/dist/lib/hooks/cache.js +19 -0
- package/dist/lib/hooks/install.js +2 -2
- package/dist/lib/hooks/match.d.ts +3 -0
- package/dist/lib/hooks/match.js +25 -0
- package/dist/lib/hosts/credentials.d.ts +1 -1
- package/dist/lib/hosts/credentials.js +1 -1
- package/dist/lib/hosts/dispatch.d.ts +7 -1
- package/dist/lib/hosts/dispatch.js +24 -5
- package/dist/lib/hosts/passthrough.d.ts +8 -15
- package/dist/lib/hosts/passthrough.js +35 -16
- package/dist/lib/hosts/providers/devices.d.ts +1 -1
- package/dist/lib/hosts/providers/devices.js +1 -1
- package/dist/lib/hosts/reconnect.d.ts +179 -21
- package/dist/lib/hosts/reconnect.js +266 -45
- package/dist/lib/hosts/registry.js +7 -0
- package/dist/lib/installations/migrate.d.ts +4 -5
- package/dist/lib/installations/migrate.js +6 -7
- package/dist/lib/installations/shims.d.ts +1 -1
- package/dist/lib/installations/shims.js +23 -13
- package/dist/lib/installations/versions.js +25 -2
- package/dist/lib/linear-autoclose.d.ts +1 -1
- package/dist/lib/linear-autoclose.js +1 -1
- package/dist/lib/loop.d.ts +2 -2
- package/dist/lib/loop.js +1 -1
- package/dist/lib/menubar/download-menubar.d.ts +47 -0
- package/dist/lib/menubar/download-menubar.js +60 -0
- package/dist/lib/menubar/install-menubar.d.ts +44 -7
- package/dist/lib/menubar/install-menubar.js +105 -23
- package/dist/lib/menubar/notify-desktop.d.ts +3 -3
- package/dist/lib/menubar/notify-desktop.js +6 -6
- package/dist/lib/model-tiers.d.ts +1 -1
- package/dist/lib/open-url.d.ts +60 -2
- package/dist/lib/open-url.js +167 -12
- package/dist/lib/project-probe.d.ts +1 -1
- package/dist/lib/project-probe.js +1 -1
- package/dist/lib/project-pull.d.ts +1 -1
- package/dist/lib/project-pull.js +2 -2
- package/dist/lib/projects.d.ts +1 -1
- package/dist/lib/projects.js +1 -1
- package/dist/lib/refresh-coordinator.d.ts +1 -1
- package/dist/lib/refresh-coordinator.js +1 -1
- package/dist/lib/sandbox.js +6 -7
- package/dist/lib/secrets/download-keychain.d.ts +47 -0
- package/dist/lib/secrets/download-keychain.js +70 -0
- package/dist/lib/secrets/install-helper.d.ts +20 -1
- package/dist/lib/secrets/install-helper.js +44 -3
- package/dist/lib/session/active.d.ts +67 -0
- package/dist/lib/session/active.js +86 -2
- package/dist/lib/session/db.d.ts +11 -1
- package/dist/lib/session/db.js +142 -8
- package/dist/lib/session/discover.d.ts +23 -1
- package/dist/lib/session/discover.js +44 -10
- package/dist/lib/session/highlights.d.ts +49 -1
- package/dist/lib/session/highlights.js +95 -0
- package/dist/lib/session/host-link.d.ts +16 -2
- package/dist/lib/session/host-link.js +7 -1
- package/dist/lib/session/parse.js +33 -1
- package/dist/lib/session/prompt.d.ts +35 -0
- package/dist/lib/session/prompt.js +56 -0
- package/dist/lib/session/remote/remote-list.d.ts +9 -0
- package/dist/lib/session/remote/remote-list.js +19 -2
- package/dist/lib/session/remote/watch.d.ts +1 -1
- package/dist/lib/session/remote/watch.js +17 -3
- package/dist/lib/session/render.js +7 -3
- package/dist/lib/session/session-cache.d.ts +53 -0
- package/dist/lib/session/session-cache.js +93 -1
- package/dist/lib/session/state.js +1 -1
- package/dist/lib/session/tool-calls.d.ts +2 -0
- package/dist/lib/session/tool-calls.js +1 -1
- package/dist/lib/session/trajectory-compare.d.ts +47 -0
- package/dist/lib/session/trajectory-compare.js +116 -0
- package/dist/lib/session/trajectory-html.d.ts +19 -0
- package/dist/lib/session/trajectory-html.js +770 -0
- package/dist/lib/session/trajectory-lineage.d.ts +92 -0
- package/dist/lib/session/trajectory-lineage.js +195 -0
- package/dist/lib/session/trajectory-text.d.ts +37 -0
- package/dist/lib/session/trajectory-text.js +284 -0
- package/dist/lib/session/trajectory.d.ts +102 -0
- package/dist/lib/session/trajectory.js +384 -0
- package/dist/lib/session/types.d.ts +15 -0
- package/dist/lib/sha256-asset.d.ts +26 -0
- package/dist/lib/sha256-asset.js +41 -0
- package/dist/lib/share/backend.d.ts +95 -0
- package/dist/lib/share/backend.js +139 -0
- package/dist/lib/share/delete.d.ts +6 -2
- package/dist/lib/share/delete.js +8 -9
- package/dist/lib/share/html.d.ts +22 -0
- package/dist/lib/share/html.js +88 -0
- package/dist/lib/share/provision.d.ts +18 -4
- package/dist/lib/share/provision.js +38 -4
- package/dist/lib/share/publish.d.ts +39 -15
- package/dist/lib/share/publish.js +55 -26
- package/dist/lib/share/worker-template.js +171 -11
- package/dist/lib/sqlite.js +16 -3
- package/dist/lib/ssh-exec.d.ts +65 -0
- package/dist/lib/ssh-exec.js +120 -4
- package/dist/lib/startup/command-registry.d.ts +3 -1
- package/dist/lib/startup/command-registry.js +6 -3
- package/dist/lib/startup/dev-build.js +3 -3
- package/dist/lib/state.d.ts +2 -2
- package/dist/lib/state.js +11 -5
- package/dist/lib/subagents-registry.js +1 -1
- package/dist/lib/teams/agents.js +9 -0
- package/dist/lib/terminal/index.d.ts +1 -1
- package/dist/lib/terminal/inject.d.ts +19 -0
- package/dist/lib/terminal/inject.js +19 -9
- package/dist/lib/terminal/types.d.ts +1 -1
- package/dist/lib/terminal/types.js +1 -1
- package/dist/lib/tmux/orphan-reap.js +1 -1
- package/dist/lib/tmux/session.d.ts +46 -13
- package/dist/lib/tmux/session.js +204 -27
- package/dist/lib/traces/backend.d.ts +29 -0
- package/dist/lib/traces/backend.js +46 -0
- package/dist/lib/traces/classify.d.ts +52 -0
- package/dist/lib/traces/classify.js +72 -0
- package/dist/lib/traces/config.d.ts +10 -0
- package/dist/lib/traces/config.js +2 -0
- package/dist/lib/traces/provision.d.ts +19 -0
- package/dist/lib/traces/provision.js +30 -0
- package/dist/lib/traces/sync.d.ts +150 -0
- package/dist/lib/traces/sync.js +447 -0
- package/dist/lib/traces/worker-template.d.ts +4 -0
- package/dist/lib/traces/worker-template.js +159 -0
- package/dist/lib/types.d.ts +34 -19
- package/dist/lib/types.js +14 -0
- package/dist/lib/usage-backoff.d.ts +2 -1
- package/dist/lib/usage-backoff.js +60 -19
- package/dist/lib/usage-refresh.d.ts +18 -4
- package/dist/lib/usage-refresh.js +68 -19
- package/dist/lib/watchdog/log.d.ts +1 -1
- package/dist/lib/watchdog/log.js +1 -1
- package/dist/lib/watchdog/runner.d.ts +36 -45
- package/dist/lib/watchdog/runner.js +150 -119
- package/dist/lib/watchdog/service.d.ts +0 -1
- package/dist/lib/watchdog/service.js +0 -1
- package/dist/lib/watchdog/watchdog-agent.d.ts +38 -0
- package/dist/lib/watchdog/watchdog-agent.js +59 -0
- package/dist/lib/watchdog/watchdog.d.ts +11 -2
- package/dist/lib/watchdog/watchdog.js +37 -104
- package/dist/lib/workflows.d.ts +1 -1
- package/package.json +5 -7
- package/dist/commands/usage.d.ts +0 -27
- package/dist/commands/usage.js +0 -96
- package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/Info.plist +0 -24
- package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/Resources/AppIcon.icns +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/_CodeSignature/CodeResources +0 -128
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/Info.plist +0 -24
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/_CodeSignature/CodeResources +0 -123
- package/dist/lib/secrets/Agents CLI.app/Contents/embedded.provisionprofile +0 -0
- package/dist/lib/usage-fleet.d.ts +0 -32
- package/dist/lib/usage-fleet.js +0 -125
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* State-dir self-terminate guard as a `PeriodicService` (RUSH-3193 P3).
|
|
3
|
+
*
|
|
4
|
+
* RUSH-2367: self-terminate if this daemon's own state directory has been
|
|
5
|
+
* removed out from under it — the shape of a leaked test-fixture daemon whose
|
|
6
|
+
* /tmp HOME was deleted by its test's own cleanup while the process itself
|
|
7
|
+
* somehow survived. Nothing else can reach a daemon in that state: a
|
|
8
|
+
* different HOME resolves a different `ensureDaemonDir()` and therefore a
|
|
9
|
+
* different instance registry — without this it runs forever. Reads the
|
|
10
|
+
* lifetime marker directly, never the local `ensureDaemonDir()` wrapper,
|
|
11
|
+
* which recreates the directory as a side effect and would defeat the check.
|
|
12
|
+
* Heartbeat/status paths may recreate the directory and pid file after a
|
|
13
|
+
* deletion; they never recreate this per-lifetime token.
|
|
14
|
+
*
|
|
15
|
+
* `lifetimePath`/`lifetimeToken` are per-boot constants computed once at
|
|
16
|
+
* `runDaemon()` start and passed in via the constructor, along with the
|
|
17
|
+
* shutdown callback. Registering (and starting) this service happens AFTER
|
|
18
|
+
* `runDaemon()` defines its `handleShutdown` — unlike the other four migrated
|
|
19
|
+
* services, which register before `supervisor.startAll()` — because the
|
|
20
|
+
* supervisor fires an immediate first tick on start, and calling an
|
|
21
|
+
* as-yet-undefined `handleShutdown` const at that point would throw. See the
|
|
22
|
+
* registration site in daemon.ts for the deferred `supervisor.register()` +
|
|
23
|
+
* `supervisor.start()` pairing this requires.
|
|
24
|
+
*/
|
|
25
|
+
import { BasePeriodicService, type DaemonContext } from './service.js';
|
|
26
|
+
import type { DaemonServiceId } from '../daemon-services.js';
|
|
27
|
+
export interface StateDirCheckServiceOptions {
|
|
28
|
+
lifetimePath: string;
|
|
29
|
+
lifetimeToken: string;
|
|
30
|
+
onMissing: () => void;
|
|
31
|
+
}
|
|
32
|
+
export declare class StateDirCheckService extends BasePeriodicService {
|
|
33
|
+
readonly id: DaemonServiceId;
|
|
34
|
+
readonly intervalMs: number;
|
|
35
|
+
readonly deadlineMs = 5000;
|
|
36
|
+
private readonly lifetimePath;
|
|
37
|
+
private readonly lifetimeToken;
|
|
38
|
+
private readonly onMissing;
|
|
39
|
+
constructor(opts: StateDirCheckServiceOptions);
|
|
40
|
+
protected onStart(_ctx: DaemonContext): Promise<void>;
|
|
41
|
+
protected onStop(): Promise<void>;
|
|
42
|
+
protected onTick(ctx: DaemonContext): Promise<void>;
|
|
43
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* State-dir self-terminate guard as a `PeriodicService` (RUSH-3193 P3).
|
|
3
|
+
*
|
|
4
|
+
* RUSH-2367: self-terminate if this daemon's own state directory has been
|
|
5
|
+
* removed out from under it — the shape of a leaked test-fixture daemon whose
|
|
6
|
+
* /tmp HOME was deleted by its test's own cleanup while the process itself
|
|
7
|
+
* somehow survived. Nothing else can reach a daemon in that state: a
|
|
8
|
+
* different HOME resolves a different `ensureDaemonDir()` and therefore a
|
|
9
|
+
* different instance registry — without this it runs forever. Reads the
|
|
10
|
+
* lifetime marker directly, never the local `ensureDaemonDir()` wrapper,
|
|
11
|
+
* which recreates the directory as a side effect and would defeat the check.
|
|
12
|
+
* Heartbeat/status paths may recreate the directory and pid file after a
|
|
13
|
+
* deletion; they never recreate this per-lifetime token.
|
|
14
|
+
*
|
|
15
|
+
* `lifetimePath`/`lifetimeToken` are per-boot constants computed once at
|
|
16
|
+
* `runDaemon()` start and passed in via the constructor, along with the
|
|
17
|
+
* shutdown callback. Registering (and starting) this service happens AFTER
|
|
18
|
+
* `runDaemon()` defines its `handleShutdown` — unlike the other four migrated
|
|
19
|
+
* services, which register before `supervisor.startAll()` — because the
|
|
20
|
+
* supervisor fires an immediate first tick on start, and calling an
|
|
21
|
+
* as-yet-undefined `handleShutdown` const at that point would throw. See the
|
|
22
|
+
* registration site in daemon.ts for the deferred `supervisor.register()` +
|
|
23
|
+
* `supervisor.start()` pairing this requires.
|
|
24
|
+
*/
|
|
25
|
+
import { BasePeriodicService } from './service.js';
|
|
26
|
+
import * as fs from 'fs';
|
|
27
|
+
/** Matches the historical inline interval (daemon.ts STATE_DIR_CHECK_TICK_MS), overridable for tests. */
|
|
28
|
+
const STATE_DIR_CHECK_TICK_MS = 60_000;
|
|
29
|
+
/** Hard cap per tick — a single synchronous file read, far above what it could ever need. */
|
|
30
|
+
const STATE_DIR_CHECK_DEADLINE_MS = 5_000;
|
|
31
|
+
export class StateDirCheckService extends BasePeriodicService {
|
|
32
|
+
id = 'state-dir-check';
|
|
33
|
+
intervalMs = Number(process.env.AGENTS_DAEMON_STATE_DIR_CHECK_MS) || STATE_DIR_CHECK_TICK_MS;
|
|
34
|
+
deadlineMs = STATE_DIR_CHECK_DEADLINE_MS;
|
|
35
|
+
lifetimePath;
|
|
36
|
+
lifetimeToken;
|
|
37
|
+
onMissing;
|
|
38
|
+
constructor(opts) {
|
|
39
|
+
super();
|
|
40
|
+
this.lifetimePath = opts.lifetimePath;
|
|
41
|
+
this.lifetimeToken = opts.lifetimeToken;
|
|
42
|
+
this.onMissing = opts.onMissing;
|
|
43
|
+
}
|
|
44
|
+
async onStart(_ctx) {
|
|
45
|
+
// No connections/handles to open — each tick re-reads the lifetime marker.
|
|
46
|
+
}
|
|
47
|
+
async onStop() {
|
|
48
|
+
// Nothing to release — the supervisor's timer teardown is the only cleanup needed.
|
|
49
|
+
}
|
|
50
|
+
async onTick(ctx) {
|
|
51
|
+
let markerMatches = false;
|
|
52
|
+
try {
|
|
53
|
+
markerMatches = fs.readFileSync(this.lifetimePath, 'utf-8') === this.lifetimeToken;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// A missing state dir or marker is the condition this guard detects.
|
|
57
|
+
}
|
|
58
|
+
if (!markerMatches) {
|
|
59
|
+
ctx.log('WARN', `Daemon state dir no longer exists; exiting (self-terminate guard)`);
|
|
60
|
+
this.onMissing();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ServiceSupervisor (RUSH-3193 P1).
|
|
3
|
+
*
|
|
4
|
+
* Owns the timer for every registered `PeriodicService`, replacing the bare
|
|
5
|
+
* `setInterval` closures in `runDaemon()`. Two failure modes motivated it,
|
|
6
|
+
* both observed in production:
|
|
7
|
+
*
|
|
8
|
+
* - A throw escaping a tick's local try/catch used to hit the process-wide
|
|
9
|
+
* `uncaughtException` handler and `process.exit(1)` the whole daemon,
|
|
10
|
+
* taking every OTHER service down with it. Here, a tick failure is caught
|
|
11
|
+
* per-service and never propagates past `runTick`.
|
|
12
|
+
* - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
|
|
13
|
+
* local in-flight guard `true` forever, silently freezing that one service
|
|
14
|
+
* for the daemon's life (observed ~51h). Here, every tick races a
|
|
15
|
+
* `deadlineMs` timeout; when the deadline wins, the guard is released in
|
|
16
|
+
* `finally` regardless of whether the real tick promise ever settles, so
|
|
17
|
+
* the NEXT scheduled tick can still run.
|
|
18
|
+
*
|
|
19
|
+
* Repeated failures (thrown or timed-out) open a circuit breaker: the service
|
|
20
|
+
* is parked (its timer stopped) and retried on exponential backoff via its own
|
|
21
|
+
* `restart()`, while every sibling service keeps ticking on its own timer,
|
|
22
|
+
* unaffected.
|
|
23
|
+
*/
|
|
24
|
+
import type { DaemonServiceId } from '../daemon-services.js';
|
|
25
|
+
import type { DaemonContext, DaemonService, ServiceHealth } from './service.js';
|
|
26
|
+
export interface ServiceSupervisorOptions {
|
|
27
|
+
/** Consecutive tick failures (throw or deadline breach) before a service is parked. Default 3. */
|
|
28
|
+
parkAfterFailures?: number;
|
|
29
|
+
/** First restart backoff delay, doubled on each further failed restart attempt. Default 5s. */
|
|
30
|
+
backoffBaseMs?: number;
|
|
31
|
+
/** Backoff ceiling. Default 5 minutes. */
|
|
32
|
+
backoffMaxMs?: number;
|
|
33
|
+
}
|
|
34
|
+
export declare class ServiceSupervisor {
|
|
35
|
+
private readonly registry;
|
|
36
|
+
private ctx;
|
|
37
|
+
private readonly parkAfterFailures;
|
|
38
|
+
private readonly backoffBaseMs;
|
|
39
|
+
private readonly backoffMaxMs;
|
|
40
|
+
constructor(opts?: ServiceSupervisorOptions);
|
|
41
|
+
/**
|
|
42
|
+
* Register a service. Must be called before `startAll()`.
|
|
43
|
+
*
|
|
44
|
+
* Accepts both `PeriodicService` (ticked on a fixed interval) and
|
|
45
|
+
* lifecycle-only `DaemonService` (started once, stopped at shutdown). The
|
|
46
|
+
* supervisor uses `isPeriodicService()` to decide whether to schedule a timer
|
|
47
|
+
* for each registered entry.
|
|
48
|
+
*/
|
|
49
|
+
register(service: DaemonService): void;
|
|
50
|
+
/** Start every registered service and begin ticking each on its own timer. */
|
|
51
|
+
startAll(ctx: DaemonContext): Promise<void>;
|
|
52
|
+
/** Stop every registered service and clear all timers. */
|
|
53
|
+
stopAll(): Promise<void>;
|
|
54
|
+
/** Force one service to restart right now, outside its normal backoff schedule. Drives `agents daemon services restart <id>` (RUSH-3193 P4). */
|
|
55
|
+
restartOne(id: DaemonServiceId): Promise<void>;
|
|
56
|
+
/** Whether `id` is registered on this supervisor. A service disabled at daemon boot is never registered — see {@link start}. */
|
|
57
|
+
isRegistered(id: DaemonServiceId): boolean;
|
|
58
|
+
/** Every currently registered service id. */
|
|
59
|
+
registeredIds(): DaemonServiceId[];
|
|
60
|
+
/**
|
|
61
|
+
* Start one registered service live — drives `agents daemon services enable
|
|
62
|
+
* <id>` (RUSH-3193 P4). Only affects a service already registered on this
|
|
63
|
+
* supervisor: one disabled at daemon boot was never constructed or
|
|
64
|
+
* registered, so enabling it live is not possible without a daemon restart
|
|
65
|
+
* (the CLI falls back to that advice — see `commands/daemon.ts`).
|
|
66
|
+
*/
|
|
67
|
+
start(id: DaemonServiceId): Promise<void>;
|
|
68
|
+
/** Stop one registered service live — drives `agents daemon services disable <id>` (RUSH-3193 P4). */
|
|
69
|
+
stop(id: DaemonServiceId): Promise<void>;
|
|
70
|
+
/** Health for every registered service, keyed by service id. */
|
|
71
|
+
health(): Record<string, ServiceHealth>;
|
|
72
|
+
private startOne;
|
|
73
|
+
private stopOne;
|
|
74
|
+
private scheduleTimer;
|
|
75
|
+
/**
|
|
76
|
+
* Run one tick under a hard deadline. The deadline is enforced with
|
|
77
|
+
* `Promise.race`, not true cancellation — JS cannot abort an arbitrary
|
|
78
|
+
* in-flight await. What matters for the freeze this replaces is that
|
|
79
|
+
* `inFlight` is released in `finally` as soon as the race settles, so an
|
|
80
|
+
* abandoned tick can never block the next scheduled one again.
|
|
81
|
+
*/
|
|
82
|
+
private runTick;
|
|
83
|
+
private park;
|
|
84
|
+
private scheduleRestart;
|
|
85
|
+
private attemptRestart;
|
|
86
|
+
private recordFailure;
|
|
87
|
+
}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ServiceSupervisor (RUSH-3193 P1).
|
|
3
|
+
*
|
|
4
|
+
* Owns the timer for every registered `PeriodicService`, replacing the bare
|
|
5
|
+
* `setInterval` closures in `runDaemon()`. Two failure modes motivated it,
|
|
6
|
+
* both observed in production:
|
|
7
|
+
*
|
|
8
|
+
* - A throw escaping a tick's local try/catch used to hit the process-wide
|
|
9
|
+
* `uncaughtException` handler and `process.exit(1)` the whole daemon,
|
|
10
|
+
* taking every OTHER service down with it. Here, a tick failure is caught
|
|
11
|
+
* per-service and never propagates past `runTick`.
|
|
12
|
+
* - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
|
|
13
|
+
* local in-flight guard `true` forever, silently freezing that one service
|
|
14
|
+
* for the daemon's life (observed ~51h). Here, every tick races a
|
|
15
|
+
* `deadlineMs` timeout; when the deadline wins, the guard is released in
|
|
16
|
+
* `finally` regardless of whether the real tick promise ever settles, so
|
|
17
|
+
* the NEXT scheduled tick can still run.
|
|
18
|
+
*
|
|
19
|
+
* Repeated failures (thrown or timed-out) open a circuit breaker: the service
|
|
20
|
+
* is parked (its timer stopped) and retried on exponential backoff via its own
|
|
21
|
+
* `restart()`, while every sibling service keeps ticking on its own timer,
|
|
22
|
+
* unaffected.
|
|
23
|
+
*/
|
|
24
|
+
import { recordSubsystemOk, recordSubsystemError, recordSubsystemState } from '../daemon-health.js';
|
|
25
|
+
import { isPeriodicService } from './service.js';
|
|
26
|
+
const DEFAULT_PARK_AFTER_FAILURES = 3;
|
|
27
|
+
const DEFAULT_BACKOFF_BASE_MS = 5_000;
|
|
28
|
+
const DEFAULT_BACKOFF_MAX_MS = 5 * 60_000;
|
|
29
|
+
export class ServiceSupervisor {
|
|
30
|
+
registry = new Map();
|
|
31
|
+
ctx = null;
|
|
32
|
+
parkAfterFailures;
|
|
33
|
+
backoffBaseMs;
|
|
34
|
+
backoffMaxMs;
|
|
35
|
+
constructor(opts = {}) {
|
|
36
|
+
this.parkAfterFailures = opts.parkAfterFailures ?? DEFAULT_PARK_AFTER_FAILURES;
|
|
37
|
+
this.backoffBaseMs = opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
38
|
+
this.backoffMaxMs = opts.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Register a service. Must be called before `startAll()`.
|
|
42
|
+
*
|
|
43
|
+
* Accepts both `PeriodicService` (ticked on a fixed interval) and
|
|
44
|
+
* lifecycle-only `DaemonService` (started once, stopped at shutdown). The
|
|
45
|
+
* supervisor uses `isPeriodicService()` to decide whether to schedule a timer
|
|
46
|
+
* for each registered entry.
|
|
47
|
+
*/
|
|
48
|
+
register(service) {
|
|
49
|
+
if (this.registry.has(service.id))
|
|
50
|
+
throw new Error(`service '${service.id}' is already registered`);
|
|
51
|
+
this.registry.set(service.id, {
|
|
52
|
+
service,
|
|
53
|
+
state: 'idle',
|
|
54
|
+
lastRunMs: 0,
|
|
55
|
+
consecutiveFailures: 0,
|
|
56
|
+
restartAttempts: 0,
|
|
57
|
+
inFlight: false,
|
|
58
|
+
everStarted: false,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/** Start every registered service and begin ticking each on its own timer. */
|
|
62
|
+
async startAll(ctx) {
|
|
63
|
+
this.ctx = ctx;
|
|
64
|
+
for (const id of this.registry.keys())
|
|
65
|
+
await this.startOne(id);
|
|
66
|
+
}
|
|
67
|
+
/** Stop every registered service and clear all timers. */
|
|
68
|
+
async stopAll() {
|
|
69
|
+
for (const id of this.registry.keys())
|
|
70
|
+
await this.stopOne(id);
|
|
71
|
+
}
|
|
72
|
+
/** Force one service to restart right now, outside its normal backoff schedule. Drives `agents daemon services restart <id>` (RUSH-3193 P4). */
|
|
73
|
+
async restartOne(id) {
|
|
74
|
+
const entry = this.registry.get(id);
|
|
75
|
+
if (!entry)
|
|
76
|
+
throw new Error(`service '${id}' is not registered`);
|
|
77
|
+
if (entry.restartTimer) {
|
|
78
|
+
clearTimeout(entry.restartTimer);
|
|
79
|
+
entry.restartTimer = undefined;
|
|
80
|
+
}
|
|
81
|
+
await this.attemptRestart(id);
|
|
82
|
+
}
|
|
83
|
+
/** Whether `id` is registered on this supervisor. A service disabled at daemon boot is never registered — see {@link start}. */
|
|
84
|
+
isRegistered(id) {
|
|
85
|
+
return this.registry.has(id);
|
|
86
|
+
}
|
|
87
|
+
/** Every currently registered service id. */
|
|
88
|
+
registeredIds() {
|
|
89
|
+
return Array.from(this.registry.keys());
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Start one registered service live — drives `agents daemon services enable
|
|
93
|
+
* <id>` (RUSH-3193 P4). Only affects a service already registered on this
|
|
94
|
+
* supervisor: one disabled at daemon boot was never constructed or
|
|
95
|
+
* registered, so enabling it live is not possible without a daemon restart
|
|
96
|
+
* (the CLI falls back to that advice — see `commands/daemon.ts`).
|
|
97
|
+
*/
|
|
98
|
+
async start(id) {
|
|
99
|
+
const entry = this.registry.get(id);
|
|
100
|
+
if (!entry)
|
|
101
|
+
throw new Error(`service '${id}' is not registered`);
|
|
102
|
+
if (entry.state === 'running')
|
|
103
|
+
return;
|
|
104
|
+
await this.startOne(id);
|
|
105
|
+
}
|
|
106
|
+
/** Stop one registered service live — drives `agents daemon services disable <id>` (RUSH-3193 P4). */
|
|
107
|
+
async stop(id) {
|
|
108
|
+
const entry = this.registry.get(id);
|
|
109
|
+
if (!entry)
|
|
110
|
+
throw new Error(`service '${id}' is not registered`);
|
|
111
|
+
if (entry.state === 'stopped')
|
|
112
|
+
return;
|
|
113
|
+
await this.stopOne(id);
|
|
114
|
+
}
|
|
115
|
+
/** Health for every registered service, keyed by service id. */
|
|
116
|
+
health() {
|
|
117
|
+
const out = {};
|
|
118
|
+
for (const [id, entry] of this.registry) {
|
|
119
|
+
out[id] = {
|
|
120
|
+
state: entry.state,
|
|
121
|
+
lastRunMs: entry.lastRunMs,
|
|
122
|
+
lastError: entry.lastError,
|
|
123
|
+
consecutiveFailures: entry.consecutiveFailures,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
async startOne(id) {
|
|
129
|
+
const entry = this.registry.get(id);
|
|
130
|
+
const ctx = this.ctx;
|
|
131
|
+
if (!entry || !ctx)
|
|
132
|
+
return;
|
|
133
|
+
try {
|
|
134
|
+
await entry.service.start(ctx);
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
this.recordFailure(entry, id, err);
|
|
138
|
+
this.park(id);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
entry.everStarted = true;
|
|
142
|
+
entry.state = 'running';
|
|
143
|
+
recordSubsystemState(id, 'running');
|
|
144
|
+
if (isPeriodicService(entry.service)) {
|
|
145
|
+
const startupDelayMs = entry.service.startupDelayMs ?? 0;
|
|
146
|
+
if (startupDelayMs > 0) {
|
|
147
|
+
// Defer BOTH the first tick and the recurring interval's start until the
|
|
148
|
+
// delay elapses — starting the interval at t=0 would fire its own ticks
|
|
149
|
+
// on top of the staggered one instead of after it.
|
|
150
|
+
entry.startupTimer = setTimeout(() => {
|
|
151
|
+
entry.startupTimer = undefined;
|
|
152
|
+
this.scheduleTimer(id);
|
|
153
|
+
void this.runTick(id);
|
|
154
|
+
}, startupDelayMs);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
this.scheduleTimer(id);
|
|
158
|
+
void this.runTick(id);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
// Lifecycle-only service: record health once on successful start (no ticks).
|
|
163
|
+
entry.lastRunMs = Date.now();
|
|
164
|
+
recordSubsystemOk(id);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async stopOne(id) {
|
|
168
|
+
const entry = this.registry.get(id);
|
|
169
|
+
if (!entry)
|
|
170
|
+
return;
|
|
171
|
+
if (entry.timer) {
|
|
172
|
+
clearInterval(entry.timer);
|
|
173
|
+
entry.timer = undefined;
|
|
174
|
+
}
|
|
175
|
+
if (entry.restartTimer) {
|
|
176
|
+
clearTimeout(entry.restartTimer);
|
|
177
|
+
entry.restartTimer = undefined;
|
|
178
|
+
}
|
|
179
|
+
if (entry.startupTimer) {
|
|
180
|
+
clearTimeout(entry.startupTimer);
|
|
181
|
+
entry.startupTimer = undefined;
|
|
182
|
+
}
|
|
183
|
+
entry.state = 'stopped';
|
|
184
|
+
recordSubsystemState(id, 'stopped');
|
|
185
|
+
if (!entry.everStarted)
|
|
186
|
+
return; // start() never succeeded — nothing to stop.
|
|
187
|
+
try {
|
|
188
|
+
await entry.service.stop();
|
|
189
|
+
}
|
|
190
|
+
catch (err) {
|
|
191
|
+
this.ctx?.log('WARN', `service '${id}' stop failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
scheduleTimer(id) {
|
|
195
|
+
const entry = this.registry.get(id);
|
|
196
|
+
if (!entry || !isPeriodicService(entry.service))
|
|
197
|
+
return;
|
|
198
|
+
entry.timer = setInterval(() => { void this.runTick(id); }, entry.service.intervalMs);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Run one tick under a hard deadline. The deadline is enforced with
|
|
202
|
+
* `Promise.race`, not true cancellation — JS cannot abort an arbitrary
|
|
203
|
+
* in-flight await. What matters for the freeze this replaces is that
|
|
204
|
+
* `inFlight` is released in `finally` as soon as the race settles, so an
|
|
205
|
+
* abandoned tick can never block the next scheduled one again.
|
|
206
|
+
*/
|
|
207
|
+
async runTick(id) {
|
|
208
|
+
const entry = this.registry.get(id);
|
|
209
|
+
const ctx = this.ctx;
|
|
210
|
+
if (!entry || !ctx || entry.state !== 'running' || entry.inFlight)
|
|
211
|
+
return;
|
|
212
|
+
// runTick is only called for periodic services (from scheduleTimer and startOne).
|
|
213
|
+
// The isPeriodicService guard here keeps TypeScript narrowing correct.
|
|
214
|
+
if (!isPeriodicService(entry.service))
|
|
215
|
+
return;
|
|
216
|
+
const periodicService = entry.service;
|
|
217
|
+
entry.inFlight = true;
|
|
218
|
+
let deadlineTimer;
|
|
219
|
+
try {
|
|
220
|
+
const deadline = new Promise((_, reject) => {
|
|
221
|
+
deadlineTimer = setTimeout(() => reject(new Error(`tick exceeded deadline of ${periodicService.deadlineMs}ms`)), periodicService.deadlineMs);
|
|
222
|
+
});
|
|
223
|
+
await Promise.race([periodicService.tick(ctx), deadline]);
|
|
224
|
+
entry.consecutiveFailures = 0;
|
|
225
|
+
entry.lastError = undefined;
|
|
226
|
+
entry.lastRunMs = Date.now();
|
|
227
|
+
recordSubsystemOk(id);
|
|
228
|
+
}
|
|
229
|
+
catch (err) {
|
|
230
|
+
this.recordFailure(entry, id, err);
|
|
231
|
+
if (entry.consecutiveFailures >= this.parkAfterFailures)
|
|
232
|
+
this.park(id);
|
|
233
|
+
}
|
|
234
|
+
finally {
|
|
235
|
+
if (deadlineTimer)
|
|
236
|
+
clearTimeout(deadlineTimer);
|
|
237
|
+
entry.inFlight = false;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
park(id) {
|
|
241
|
+
const entry = this.registry.get(id);
|
|
242
|
+
if (!entry || entry.state === 'parked' || entry.state === 'stopped')
|
|
243
|
+
return;
|
|
244
|
+
entry.state = 'parked';
|
|
245
|
+
recordSubsystemState(id, 'parked');
|
|
246
|
+
if (entry.timer) {
|
|
247
|
+
clearInterval(entry.timer);
|
|
248
|
+
entry.timer = undefined;
|
|
249
|
+
}
|
|
250
|
+
this.ctx?.log('WARN', `service '${id}' parked after ${entry.consecutiveFailures} consecutive failure(s)`);
|
|
251
|
+
this.scheduleRestart(id);
|
|
252
|
+
}
|
|
253
|
+
scheduleRestart(id) {
|
|
254
|
+
const entry = this.registry.get(id);
|
|
255
|
+
if (!entry)
|
|
256
|
+
return;
|
|
257
|
+
const delay = Math.min(this.backoffMaxMs, this.backoffBaseMs * 2 ** entry.restartAttempts);
|
|
258
|
+
entry.restartAttempts += 1;
|
|
259
|
+
entry.restartTimer = setTimeout(() => { void this.attemptRestart(id); }, delay);
|
|
260
|
+
}
|
|
261
|
+
async attemptRestart(id) {
|
|
262
|
+
const entry = this.registry.get(id);
|
|
263
|
+
const ctx = this.ctx;
|
|
264
|
+
if (!entry || !ctx || entry.state === 'stopped')
|
|
265
|
+
return;
|
|
266
|
+
try {
|
|
267
|
+
await entry.service.restart();
|
|
268
|
+
entry.everStarted = true; // restart() is stop()+start() on the service — a successful one means it is running again and owes a stop() at shutdown.
|
|
269
|
+
entry.state = 'running';
|
|
270
|
+
recordSubsystemState(id, 'running');
|
|
271
|
+
entry.consecutiveFailures = 0;
|
|
272
|
+
entry.restartAttempts = 0;
|
|
273
|
+
entry.lastError = undefined;
|
|
274
|
+
if (isPeriodicService(entry.service)) {
|
|
275
|
+
this.scheduleTimer(id);
|
|
276
|
+
void this.runTick(id);
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
entry.lastRunMs = Date.now();
|
|
280
|
+
}
|
|
281
|
+
recordSubsystemOk(id);
|
|
282
|
+
ctx.log('INFO', `service '${id}' restarted`);
|
|
283
|
+
}
|
|
284
|
+
catch (err) {
|
|
285
|
+
this.recordFailure(entry, id, err);
|
|
286
|
+
entry.state = 'parked';
|
|
287
|
+
recordSubsystemState(id, 'parked');
|
|
288
|
+
this.scheduleRestart(id);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
recordFailure(entry, id, err) {
|
|
292
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
293
|
+
entry.consecutiveFailures += 1;
|
|
294
|
+
entry.lastError = message;
|
|
295
|
+
recordSubsystemError(id, message);
|
|
296
|
+
this.ctx?.log('WARN', `service '${id}' failed: ${message}`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watchdog tick as a `PeriodicService` (RUSH-3193 P3).
|
|
3
|
+
*
|
|
4
|
+
* Nudges this host's own stalled agent sessions. Gated on the `watchdog.enabled`
|
|
5
|
+
* device-config flag (`agents watchdog enable`), so the timer always fires but
|
|
6
|
+
* only does work when the user opted in — the check happens inside the tick
|
|
7
|
+
* itself, not the enable/disable registration gate, matching the pre-migration
|
|
8
|
+
* inline behavior (daemon.ts previously `WATCHDOG_TICK_MS`-interval closure).
|
|
9
|
+
*/
|
|
10
|
+
import { BasePeriodicService, type DaemonContext } from './service.js';
|
|
11
|
+
import type { DaemonServiceId } from '../daemon-services.js';
|
|
12
|
+
export declare class WatchdogService extends BasePeriodicService {
|
|
13
|
+
readonly id: DaemonServiceId;
|
|
14
|
+
readonly intervalMs: number;
|
|
15
|
+
readonly deadlineMs = 120000;
|
|
16
|
+
protected onStart(_ctx: DaemonContext): Promise<void>;
|
|
17
|
+
protected onStop(): Promise<void>;
|
|
18
|
+
protected onTick(ctx: DaemonContext): Promise<void>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watchdog tick as a `PeriodicService` (RUSH-3193 P3).
|
|
3
|
+
*
|
|
4
|
+
* Nudges this host's own stalled agent sessions. Gated on the `watchdog.enabled`
|
|
5
|
+
* device-config flag (`agents watchdog enable`), so the timer always fires but
|
|
6
|
+
* only does work when the user opted in — the check happens inside the tick
|
|
7
|
+
* itself, not the enable/disable registration gate, matching the pre-migration
|
|
8
|
+
* inline behavior (daemon.ts previously `WATCHDOG_TICK_MS`-interval closure).
|
|
9
|
+
*/
|
|
10
|
+
import { BasePeriodicService } from './service.js';
|
|
11
|
+
import { getConfigValue } from '../device-config.js';
|
|
12
|
+
import { emit } from '../feed/events.js';
|
|
13
|
+
/** Matches the historical inline interval (daemon.ts WATCHDOG_TICK_MS). */
|
|
14
|
+
const WATCHDOG_TICK_MS = 3 * 60_000;
|
|
15
|
+
/** Hard cap per tick — `runWatchdogPass` is host-local (adds no SSH fan-out of its own, `watchdog/runner.ts:580`); short enough that a hang never freezes the service for long. */
|
|
16
|
+
const WATCHDOG_DEADLINE_MS = 120_000;
|
|
17
|
+
export class WatchdogService extends BasePeriodicService {
|
|
18
|
+
id = 'watchdog';
|
|
19
|
+
intervalMs = WATCHDOG_TICK_MS;
|
|
20
|
+
deadlineMs = WATCHDOG_DEADLINE_MS;
|
|
21
|
+
async onStart(_ctx) {
|
|
22
|
+
// No connections/handles to open — each tick re-reads the enable flag.
|
|
23
|
+
}
|
|
24
|
+
async onStop() {
|
|
25
|
+
// Nothing to release — the supervisor's timer teardown is the only cleanup needed.
|
|
26
|
+
}
|
|
27
|
+
async onTick(ctx) {
|
|
28
|
+
if (getConfigValue('watchdog.enabled').value !== true)
|
|
29
|
+
return;
|
|
30
|
+
const { runWatchdogPass } = await import('../watchdog/service.js');
|
|
31
|
+
const result = await runWatchdogPass({ nudge: true });
|
|
32
|
+
ctx.log('INFO', `watchdog: ${result.counts.total} live, ${result.counts.stalled} stalled, ${result.counts.nudged} nudged`);
|
|
33
|
+
emit('watchdog.action', {
|
|
34
|
+
module: 'watchdog',
|
|
35
|
+
total: result.counts.total,
|
|
36
|
+
stalled: result.counts.stalled,
|
|
37
|
+
nudged: result.counts.nudged,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -22,6 +22,14 @@ export interface SubsystemHealth {
|
|
|
22
22
|
consecutiveFailures: number;
|
|
23
23
|
/** ISO timestamp of the most recent success, or null if it has never succeeded. */
|
|
24
24
|
lastOkAt: string | null;
|
|
25
|
+
/**
|
|
26
|
+
* `ServiceSupervisor`'s lifecycle state (`idle`/`running`/`parked`/`stopped`),
|
|
27
|
+
* written by `recordSubsystemState` on every transition. Only present for
|
|
28
|
+
* supervisor-managed subsystems (RUSH-3193 P4) — a subsystem that predates the
|
|
29
|
+
* supervisor (e.g. `daemon-start`) never has this field, which is how `agents
|
|
30
|
+
* daemon services` tells a measured state from an inferred one.
|
|
31
|
+
*/
|
|
32
|
+
state?: string;
|
|
25
33
|
}
|
|
26
34
|
/** Record a successful subsystem check-in — clears the failure streak. */
|
|
27
35
|
export declare function recordSubsystemOk(subsystem: string, at?: string): void;
|
|
@@ -40,6 +48,13 @@ export declare function recordSubsystemError(subsystem: string, error: string, a
|
|
|
40
48
|
* subsystem is left alone rather than given a blank record to decorate.
|
|
41
49
|
*/
|
|
42
50
|
export declare function recordSubsystemErrorReason(subsystem: string, error: string, at?: string): void;
|
|
51
|
+
/**
|
|
52
|
+
* Record a `ServiceSupervisor` lifecycle-state transition, without touching
|
|
53
|
+
* the ok/error streak. Cross-process readers (`agents daemon services`) have
|
|
54
|
+
* no other way to see `parked`/`stopped` vs `running` — `agents daemon
|
|
55
|
+
* status` runs as a separate process from the daemon (see module docblock).
|
|
56
|
+
*/
|
|
57
|
+
export declare function recordSubsystemState(subsystem: string, state: string): void;
|
|
43
58
|
/** Read one subsystem's health record, or null if it has never reported in. */
|
|
44
59
|
export declare function readSubsystemHealth(subsystem: string): SubsystemHealth | null;
|
|
45
60
|
/** Read every subsystem's health record, sorted by subsystem name. */
|
|
@@ -52,14 +52,26 @@ function readAll() {
|
|
|
52
52
|
return {};
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Never throws. `recordSubsystemOk`/`recordSubsystemError` are called from
|
|
57
|
+
* inside a service's error boundary (`ServiceSupervisor.runTick`'s catch, and
|
|
58
|
+
* its own catch-of-a-catch in `recordFailure`) — a write failure here (disk
|
|
59
|
+
* full, permission, or the state dir removed mid-run, which this daemon
|
|
60
|
+
* explicitly anticipates via the state-dir self-check) must degrade to a
|
|
61
|
+
* dropped health update, never escape as an unhandled rejection that would
|
|
62
|
+
* hit the process-wide handler and take down every OTHER service too.
|
|
63
|
+
*/
|
|
55
64
|
function writeAll(records) {
|
|
56
|
-
const healthPath = getHealthPath();
|
|
57
|
-
fs.mkdirSync(path.dirname(healthPath), { recursive: true });
|
|
58
|
-
fs.writeFileSync(healthPath, JSON.stringify(records), 'utf-8');
|
|
59
65
|
try {
|
|
60
|
-
|
|
66
|
+
const healthPath = getHealthPath();
|
|
67
|
+
fs.mkdirSync(path.dirname(healthPath), { recursive: true });
|
|
68
|
+
fs.writeFileSync(healthPath, JSON.stringify(records), 'utf-8');
|
|
69
|
+
try {
|
|
70
|
+
fs.chmodSync(healthPath, 0o600);
|
|
71
|
+
}
|
|
72
|
+
catch { /* best effort */ }
|
|
61
73
|
}
|
|
62
|
-
catch { /*
|
|
74
|
+
catch { /* see docblock above — health recording must never crash a caller */ }
|
|
63
75
|
}
|
|
64
76
|
function blankRecord(subsystem) {
|
|
65
77
|
return { subsystem, lastError: null, lastErrorAt: null, consecutiveFailures: 0, lastOkAt: null };
|
|
@@ -104,6 +116,18 @@ export function recordSubsystemErrorReason(subsystem, error, at = new Date().toI
|
|
|
104
116
|
all[subsystem] = { ...existing, lastError: error, lastErrorAt: at };
|
|
105
117
|
writeAll(all);
|
|
106
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Record a `ServiceSupervisor` lifecycle-state transition, without touching
|
|
121
|
+
* the ok/error streak. Cross-process readers (`agents daemon services`) have
|
|
122
|
+
* no other way to see `parked`/`stopped` vs `running` — `agents daemon
|
|
123
|
+
* status` runs as a separate process from the daemon (see module docblock).
|
|
124
|
+
*/
|
|
125
|
+
export function recordSubsystemState(subsystem, state) {
|
|
126
|
+
const all = readAll();
|
|
127
|
+
const existing = all[subsystem] ?? blankRecord(subsystem);
|
|
128
|
+
all[subsystem] = { ...existing, subsystem, state };
|
|
129
|
+
writeAll(all);
|
|
130
|
+
}
|
|
107
131
|
/** Read one subsystem's health record, or null if it has never reported in. */
|
|
108
132
|
export function readSubsystemHealth(subsystem) {
|
|
109
133
|
return readAll()[subsystem] ?? null;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* enabled without pulling in the whole daemon lifecycle.
|
|
8
8
|
*/
|
|
9
9
|
/** Every service the daemon can host. IDs are kebab-case and stable. */
|
|
10
|
-
export type DaemonServiceId = 'secrets-broker' | 'scheduler' | 'monitors' | 'browser-ipc' | 'webhook-receiver' | 'self-heal' | 'keychain-reap' | 'account-state' | 'watchdog' | 'device-probe' | 'state-dir-check';
|
|
10
|
+
export type DaemonServiceId = 'secrets-broker' | 'scheduler' | 'monitors' | 'browser-ipc' | 'webhook-receiver' | 'self-heal' | 'keychain-reap' | 'account-state' | 'watchdog' | 'device-probe' | 'state-dir-check' | 'session-index';
|
|
11
11
|
/** Human-readable metadata for each service. */
|
|
12
12
|
export interface DaemonServiceDef {
|
|
13
13
|
id: DaemonServiceId;
|
|
@@ -48,3 +48,10 @@ export declare function setDaemonServiceEnabled(id: DaemonServiceId, enabled: bo
|
|
|
48
48
|
export declare function listDaemonServiceStates(): Array<DaemonServiceDef & {
|
|
49
49
|
enabled: boolean;
|
|
50
50
|
}>;
|
|
51
|
+
/** Queue a live restart for `id`, to be picked up on the next SIGHUP reload. */
|
|
52
|
+
export declare function queueDaemonServiceRestart(id: DaemonServiceId): void;
|
|
53
|
+
/**
|
|
54
|
+
* Drain every queued restart request. Called once per SIGHUP reload by the
|
|
55
|
+
* daemon; never throws. Clears the file so a request is applied at most once.
|
|
56
|
+
*/
|
|
57
|
+
export declare function drainDaemonServiceRestartQueue(): DaemonServiceId[];
|