@phnx-labs/agents-cli 1.22.35 → 1.22.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -2
- package/README.md +75 -35
- package/dist/bin/agents +0 -0
- package/dist/bootstrap.d.ts +15 -0
- package/dist/bootstrap.js +1211 -0
- package/dist/commands/accounts.d.ts +4 -11
- package/dist/commands/accounts.js +159 -27
- package/dist/commands/attach.js +17 -4
- package/dist/commands/audit.d.ts +8 -8
- package/dist/commands/audit.js +57 -47
- package/dist/commands/browser.js +15 -11
- package/dist/commands/cloud.js +1 -1
- package/dist/commands/commands.js +0 -11
- package/dist/commands/computer-actions.js +33 -0
- package/dist/commands/computer-sessions-picker.d.ts +21 -0
- package/dist/commands/computer-sessions-picker.js +141 -0
- package/dist/commands/computer.d.ts +16 -0
- package/dist/commands/computer.js +37 -0
- package/dist/commands/config.d.ts +17 -0
- package/dist/commands/config.js +401 -0
- package/dist/commands/cp.d.ts +84 -0
- package/dist/commands/cp.js +231 -0
- package/dist/commands/daemon.js +243 -14
- package/dist/commands/detach-core.d.ts +1 -1
- package/dist/commands/detach-core.js +12 -2
- package/dist/commands/detach.js +1 -1
- package/dist/commands/doctor.js +64 -5
- package/dist/commands/events.d.ts +24 -12
- package/dist/commands/events.js +144 -28
- package/dist/commands/exec.js +85 -41
- package/dist/commands/feed.d.ts +1 -1
- package/dist/commands/feed.js +16 -8
- package/dist/commands/focus.d.ts +33 -1
- package/dist/commands/focus.js +177 -17
- package/dist/commands/funnel.d.ts +1 -1
- package/dist/commands/go.js +9 -8
- package/dist/commands/harness-wizard.d.ts +1 -0
- package/dist/commands/harness-wizard.js +18 -20
- package/dist/commands/harness.d.ts +2 -0
- package/dist/commands/harness.js +59 -26
- package/dist/commands/hooks.js +0 -11
- package/dist/commands/inspect.d.ts +94 -1
- package/dist/commands/inspect.js +866 -40
- package/dist/commands/lease.d.ts +4 -4
- package/dist/commands/lease.js +6 -6
- package/dist/commands/login.js +0 -13
- package/dist/commands/logs.d.ts +2 -2
- package/dist/commands/logs.js +2 -2
- package/dist/commands/mcp.js +7 -2
- package/dist/commands/memory.js +7 -2
- package/dist/commands/message.d.ts +1 -1
- package/dist/commands/message.js +37 -6
- package/dist/commands/models.js +4 -0
- package/dist/commands/monitors.d.ts +0 -1
- package/dist/commands/monitors.js +199 -16
- package/dist/commands/open.d.ts +18 -0
- package/dist/commands/open.js +57 -0
- package/dist/commands/packages.js +2 -107
- package/dist/commands/plugins.js +6 -3
- package/dist/commands/profiles.d.ts +1 -0
- package/dist/commands/profiles.js +16 -160
- package/dist/commands/projects.d.ts +28 -0
- package/dist/commands/projects.js +125 -5
- package/dist/commands/reconnect.js +9 -3
- package/dist/commands/repo.js +42 -1
- package/dist/commands/resource-view.d.ts +17 -0
- package/dist/commands/resource-view.js +36 -14
- package/dist/commands/resume.js +9 -2
- package/dist/commands/routines.d.ts +0 -2
- package/dist/commands/routines.js +473 -289
- package/dist/commands/secrets.js +52 -9
- package/dist/commands/sessions-export.d.ts +47 -0
- package/dist/commands/sessions-export.js +116 -6
- package/dist/commands/sessions-import.d.ts +17 -0
- package/dist/commands/sessions-import.js +114 -6
- package/dist/commands/sessions-migrate.d.ts +27 -0
- package/dist/commands/sessions-migrate.js +50 -9
- package/dist/commands/sessions-picker.js +23 -4
- package/dist/commands/sessions-resume.d.ts +5 -3
- package/dist/commands/sessions-resume.js +30 -8
- package/dist/commands/sessions-watch.d.ts +2 -0
- package/dist/commands/sessions-watch.js +53 -0
- package/dist/commands/sessions.d.ts +66 -3
- package/dist/commands/sessions.js +298 -47
- package/dist/commands/set.d.ts +1 -1
- package/dist/commands/set.js +2 -2
- package/dist/commands/setup-browser.js +2 -2
- package/dist/commands/setup-mine.js +0 -1
- package/dist/commands/setup-preferences.js +1 -1
- package/dist/commands/setup-share.js +34 -4
- package/dist/commands/setup.d.ts +5 -0
- package/dist/commands/setup.js +26 -0
- package/dist/commands/share.d.ts +67 -0
- package/dist/commands/share.js +275 -12
- package/dist/commands/skills.js +0 -11
- package/dist/commands/snapshot.js +1 -1
- package/dist/commands/ssh.d.ts +18 -0
- package/dist/commands/ssh.js +549 -210
- package/dist/commands/sync.d.ts +5 -0
- package/dist/commands/sync.js +212 -32
- package/dist/commands/teams.d.ts +13 -0
- package/dist/commands/teams.js +187 -30
- package/dist/commands/tickets.d.ts +2 -0
- package/dist/commands/tickets.js +43 -0
- package/dist/commands/update.js +2 -4
- package/dist/commands/usage.js +5 -3
- package/dist/commands/versions.js +11 -5
- package/dist/commands/view.js +15 -7
- package/dist/commands/webhook.d.ts +2 -2
- package/dist/commands/webhook.js +7 -7
- package/dist/index.d.ts +18 -2
- package/dist/index.js +53 -1264
- package/dist/lib/account-catalog.d.ts +18 -0
- package/dist/lib/account-catalog.js +38 -0
- package/dist/lib/account-provider-registry.d.ts +18 -0
- package/dist/lib/account-provider-registry.js +59 -0
- package/dist/lib/account-registry.d.ts +40 -0
- package/dist/lib/account-registry.js +239 -0
- package/dist/lib/account-schema.d.ts +44 -0
- package/dist/lib/account-schema.js +91 -0
- package/dist/lib/account-state-service.d.ts +21 -0
- package/dist/lib/account-state-service.js +60 -0
- package/dist/lib/activity.d.ts +1 -1
- package/dist/lib/activity.js +8 -1
- package/dist/lib/add-dir.d.ts +80 -0
- package/dist/lib/add-dir.js +241 -0
- package/dist/lib/agent-cli-commands.d.ts +11 -0
- package/dist/lib/agent-cli-commands.js +30 -0
- package/dist/lib/agents.js +44 -36
- package/dist/lib/answer-router.js +1 -1
- package/dist/lib/audit/log.d.ts +10 -27
- package/dist/lib/audit/log.js +20 -33
- package/dist/lib/auth-health.d.ts +9 -2
- package/dist/lib/auth-health.js +42 -9
- package/dist/lib/auto-pull-worker.js +7 -3
- package/dist/lib/auto-pull.d.ts +44 -3
- package/dist/lib/auto-pull.js +88 -5
- package/dist/lib/binary-shadow.d.ts +28 -0
- package/dist/lib/binary-shadow.js +121 -0
- package/dist/lib/brand.js +9 -3
- package/dist/lib/browser/ipc.d.ts +30 -0
- package/dist/lib/browser/ipc.js +81 -4
- package/dist/lib/browser/profiles.d.ts +3 -2
- package/dist/lib/browser/profiles.js +65 -22
- package/dist/lib/browser/service.d.ts +4 -0
- package/dist/lib/browser/service.js +30 -3
- package/dist/lib/browser/sessions-list.d.ts +22 -2
- package/dist/lib/browser/sessions-list.js +54 -6
- package/dist/lib/browser/stream.d.ts +2 -0
- package/dist/lib/browser/stream.js +1 -0
- package/dist/lib/browser/types.d.ts +19 -0
- package/dist/lib/byok-usage.d.ts +9 -2
- package/dist/lib/byok-usage.js +133 -31
- package/dist/lib/cli-entry.d.ts +11 -0
- package/dist/lib/cli-entry.js +46 -0
- package/dist/lib/cloud/host.d.ts +1 -1
- package/dist/lib/cloud/host.js +2 -2
- package/dist/lib/cloud/registry.js +1 -1
- package/dist/lib/cloud/types.d.ts +1 -1
- package/dist/lib/commands.d.ts +2 -0
- package/dist/lib/commands.js +2 -0
- package/dist/lib/computer/sessions-list.d.ts +150 -0
- package/dist/lib/computer/sessions-list.js +363 -0
- package/dist/lib/config-keys.d.ts +86 -0
- package/dist/lib/config-keys.js +193 -0
- package/dist/lib/config-machine-keys.d.ts +12 -0
- package/dist/lib/config-machine-keys.js +17 -0
- package/dist/lib/config-transfer.js +3 -4
- package/dist/lib/crabbox/cli.d.ts +2 -2
- package/dist/lib/crabbox/cli.js +3 -3
- package/dist/lib/daemon-health.d.ts +22 -0
- package/dist/lib/daemon-health.js +33 -3
- package/dist/lib/daemon-services.d.ts +50 -0
- package/dist/lib/daemon-services.js +147 -0
- package/dist/lib/daemon-ticks.d.ts +35 -66
- package/dist/lib/daemon-ticks.js +77 -159
- package/dist/lib/daemon.d.ts +72 -4
- package/dist/lib/daemon.js +792 -202
- package/dist/lib/deeplink/register.d.ts +49 -0
- package/dist/lib/deeplink/register.js +253 -0
- package/dist/lib/deeplink/url.d.ts +34 -0
- package/dist/lib/deeplink/url.js +78 -0
- package/dist/lib/device-config.d.ts +124 -29
- package/dist/lib/device-config.js +346 -141
- package/dist/lib/devices/config-migration.d.ts +42 -0
- package/dist/lib/devices/config-migration.js +203 -0
- package/dist/lib/devices/connect.js +14 -7
- package/dist/lib/devices/doctor-findings.d.ts +4 -1
- package/dist/lib/devices/doctor-findings.js +20 -1
- package/dist/lib/devices/harness-inventory.d.ts +11 -2
- package/dist/lib/devices/harness-inventory.js +0 -0
- package/dist/lib/devices/health.d.ts +1 -1
- package/dist/lib/devices/health.js +1 -1
- package/dist/lib/devices/pending.d.ts +22 -1
- package/dist/lib/devices/pending.js +67 -2
- package/dist/lib/devices/registry.d.ts +0 -25
- package/dist/lib/devices/registry.js +2 -83
- package/dist/lib/devices/resolve-profile.d.ts +20 -0
- package/dist/lib/devices/resolve-profile.js +44 -0
- package/dist/lib/devices/rollout-verify.d.ts +101 -0
- package/dist/lib/devices/rollout-verify.js +193 -0
- package/dist/lib/devices/ssh-config.js +10 -6
- package/dist/lib/event-families.d.ts +28 -0
- package/dist/lib/event-families.js +159 -0
- package/dist/lib/event-stream.d.ts +9 -1
- package/dist/lib/event-stream.js +26 -9
- package/dist/lib/events.d.ts +9 -1
- package/dist/lib/events.js +21 -6
- package/dist/lib/exec.d.ts +30 -0
- package/dist/lib/exec.js +219 -121
- package/dist/lib/feed-broadcast.d.ts +25 -0
- package/dist/lib/feed-broadcast.js +33 -0
- package/dist/lib/feed.d.ts +1 -1
- package/dist/lib/feed.js +12 -2
- package/dist/lib/fleet/capture.js +15 -0
- package/dist/lib/fleet/manifest.js +9 -0
- package/dist/lib/fleet/types.d.ts +9 -0
- package/dist/lib/fleet-status.js +10 -8
- package/dist/lib/git.d.ts +20 -6
- package/dist/lib/git.js +121 -17
- package/dist/lib/hosts/dispatch.d.ts +8 -3
- package/dist/lib/hosts/dispatch.js +29 -3
- package/dist/lib/hosts/logs.d.ts +2 -2
- package/dist/lib/hosts/logs.js +2 -2
- package/dist/lib/hosts/option.js +1 -1
- package/dist/lib/hosts/passthrough.d.ts +2 -3
- package/dist/lib/hosts/passthrough.js +10 -24
- package/dist/lib/hosts/progress.js +2 -2
- package/dist/lib/hosts/providers/devices.d.ts +3 -3
- package/dist/lib/hosts/providers/devices.js +18 -12
- package/dist/lib/hosts/ready.d.ts +5 -1
- package/dist/lib/hosts/ready.js +9 -3
- package/dist/lib/hosts/reconcile.d.ts +2 -2
- package/dist/lib/hosts/reconcile.js +2 -2
- package/dist/lib/hosts/registry.js +14 -9
- package/dist/lib/hosts/remote-cmd.js +1 -1
- package/dist/lib/hosts/remote-os.d.ts +9 -6
- package/dist/lib/hosts/remote-os.js +13 -6
- package/dist/lib/hosts/routing-flag.d.ts +23 -0
- package/dist/lib/hosts/routing-flag.js +41 -0
- package/dist/lib/hosts/run-target.js +1 -1
- package/dist/lib/hosts/session-index.js +1 -1
- package/dist/lib/hosts/tasks.d.ts +6 -6
- package/dist/lib/hosts/tasks.js +4 -4
- package/dist/lib/hosts/types.js +1 -2
- package/dist/lib/installations/resolve.js +2 -8
- package/dist/lib/isolation-boundary-report.js +0 -1
- package/dist/lib/linear-cache.d.ts +7 -0
- package/dist/lib/linear-cache.js +31 -4
- package/dist/lib/linear-project-counts.d.ts +1 -1
- package/dist/lib/linear-project-counts.js +2 -3
- package/dist/lib/linear-projects.js +1 -1
- package/dist/lib/loop.d.ts +2 -2
- package/dist/lib/loop.js +1 -1
- package/dist/lib/mailbox-target.d.ts +1 -1
- package/dist/lib/mailbox-target.js +1 -1
- package/dist/lib/manifest.js +7 -4
- package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
- package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
- package/dist/lib/menubar/snapshot.d.ts +3 -2
- package/dist/lib/menubar/snapshot.js +6 -2
- package/dist/lib/migrate-fold.d.ts +33 -0
- package/dist/lib/migrate-fold.js +130 -0
- package/dist/lib/migrate.d.ts +34 -34
- package/dist/lib/migrate.js +106 -128
- package/dist/lib/model-tier-overrides.d.ts +1 -1
- package/dist/lib/model-tier-overrides.js +15 -2
- package/dist/lib/monitors/config.d.ts +17 -3
- package/dist/lib/monitors/config.js +52 -18
- package/dist/lib/monitors/engine.d.ts +32 -1
- package/dist/lib/monitors/engine.js +94 -11
- package/dist/lib/monitors/fingerprint.d.ts +46 -0
- package/dist/lib/monitors/fingerprint.js +100 -0
- package/dist/lib/monitors/remote.d.ts +47 -0
- package/dist/lib/monitors/remote.js +81 -0
- package/dist/lib/monitors/state.d.ts +39 -0
- package/dist/lib/monitors/state.js +56 -0
- package/dist/lib/onepassword.d.ts +17 -0
- package/dist/lib/onepassword.js +52 -22
- package/dist/lib/open-url.js +1 -1
- package/dist/lib/picker.d.ts +14 -2
- package/dist/lib/picker.js +35 -13
- package/dist/lib/placement.d.ts +1 -1
- package/dist/lib/placement.js +1 -1
- package/dist/lib/pr-land-detach.d.ts +32 -0
- package/dist/lib/pr-land-detach.js +79 -0
- package/dist/lib/profiles-presets.js +13 -0
- package/dist/lib/profiles.d.ts +2 -0
- package/dist/lib/profiles.js +44 -1
- package/dist/lib/project-focus.d.ts +1 -1
- package/dist/lib/project-focus.js +1 -1
- package/dist/lib/project-probe.d.ts +4 -1
- package/dist/lib/project-probe.js +5 -4
- package/dist/lib/project-resources.js +2 -18
- package/dist/lib/project-root.d.ts +16 -0
- package/dist/lib/project-root.js +20 -2
- package/dist/lib/projects.d.ts +20 -1
- package/dist/lib/projects.js +78 -1
- package/dist/lib/refresh-coordinator.d.ts +24 -0
- package/dist/lib/refresh-coordinator.js +52 -0
- package/dist/lib/registry.d.ts +2 -2
- package/dist/lib/registry.js +1 -1
- package/dist/lib/resource-aliases.d.ts +13 -0
- package/dist/lib/resource-aliases.js +26 -0
- package/dist/lib/resources/types.d.ts +1 -1
- package/dist/lib/resources.d.ts +16 -0
- package/dist/lib/resources.js +66 -10
- package/dist/lib/rotate.d.ts +3 -4
- package/dist/lib/rotate.js +4 -5
- package/dist/lib/routine-activation.d.ts +22 -2
- package/dist/lib/routine-activation.js +63 -16
- package/dist/lib/routine-context.d.ts +9 -2
- package/dist/lib/routine-context.js +21 -6
- package/dist/lib/routines-project.d.ts +58 -48
- package/dist/lib/routines-project.js +190 -159
- package/dist/lib/routines.d.ts +41 -6
- package/dist/lib/routines.js +92 -34
- package/dist/lib/run-defaults.d.ts +5 -0
- package/dist/lib/run-defaults.js +11 -1
- package/dist/lib/runner.d.ts +10 -8
- package/dist/lib/runner.js +122 -39
- package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
- package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
- package/dist/lib/secrets/agent.d.ts +10 -3
- package/dist/lib/secrets/agent.js +43 -8
- package/dist/lib/secrets/bundles.d.ts +1 -0
- package/dist/lib/secrets/bundles.js +92 -80
- package/dist/lib/secrets/index.js +32 -9
- package/dist/lib/secrets/install-helper.d.ts +4 -4
- package/dist/lib/secrets/install-helper.js +4 -4
- package/dist/lib/secrets/push.d.ts +40 -6
- package/dist/lib/secrets/push.js +58 -11
- package/dist/lib/secrets/read-backoff.d.ts +1 -1
- package/dist/lib/secrets/read-backoff.js +1 -1
- package/dist/lib/secrets/remote.d.ts +24 -0
- package/dist/lib/secrets/remote.js +54 -4
- package/dist/lib/self-update.d.ts +108 -0
- package/dist/lib/self-update.js +276 -1
- package/dist/lib/session/active.d.ts +89 -5
- package/dist/lib/session/active.js +165 -10
- package/dist/lib/session/actor-sidecar.d.ts +1 -1
- package/dist/lib/session/actor-sidecar.js +2 -1
- package/dist/lib/session/db.d.ts +176 -2
- package/dist/lib/session/db.js +484 -17
- package/dist/lib/session/discover.js +27 -4
- package/dist/lib/session/hook-sessions.js +2 -2
- package/dist/lib/session/host-link.d.ts +2 -2
- package/dist/lib/session/host-link.js +2 -2
- package/dist/lib/session/pid-registry.d.ts +21 -0
- package/dist/lib/session/pid-registry.js +57 -0
- package/dist/lib/session/remote-active.js +15 -1
- package/dist/lib/session/remote.js +2 -2
- package/dist/lib/session/session-cache.d.ts +14 -7
- package/dist/lib/session/session-cache.js +51 -9
- package/dist/lib/session/state.d.ts +2 -2
- package/dist/lib/session/state.js +1 -1
- package/dist/lib/session/sync/config.d.ts +10 -10
- package/dist/lib/session/sync/config.js +10 -10
- package/dist/lib/session/sync/r2.d.ts +34 -0
- package/dist/lib/session/sync/r2.js +123 -0
- package/dist/lib/session/throughput.d.ts +1 -1
- package/dist/lib/session/throughput.js +1 -1
- package/dist/lib/session/types.d.ts +23 -1
- package/dist/lib/session/types.js +15 -0
- package/dist/lib/session/viewing-in.d.ts +1 -1
- package/dist/lib/session/viewing-in.js +1 -1
- package/dist/lib/session/watch.d.ts +101 -0
- package/dist/lib/session/watch.js +242 -0
- package/dist/lib/share/config.d.ts +4 -0
- package/dist/lib/share/config.js +1 -0
- package/dist/lib/share/provision.d.ts +38 -0
- package/dist/lib/share/provision.js +50 -0
- package/dist/lib/share/publish.d.ts +40 -1
- package/dist/lib/share/publish.js +116 -3
- package/dist/lib/share/worker-template.js +71 -6
- package/dist/lib/shims.d.ts +17 -3
- package/dist/lib/shims.js +98 -26
- package/dist/lib/signin-badge.js +3 -2
- package/dist/lib/skills.js +2 -0
- package/dist/lib/smart-launch.d.ts +8 -8
- package/dist/lib/smart-launch.js +80 -59
- package/dist/lib/ssh-exec.d.ts +1 -1
- package/dist/lib/ssh-exec.js +1 -1
- package/dist/lib/staleness/prune.d.ts +88 -0
- package/dist/lib/staleness/prune.js +69 -0
- package/dist/lib/staleness/registry.d.ts +1 -1
- package/dist/lib/staleness/writers/commands.js +34 -2
- package/dist/lib/staleness/writers/hooks.js +16 -0
- package/dist/lib/staleness/writers/kinds.d.ts +6 -0
- package/dist/lib/staleness/writers/skills.js +19 -0
- package/dist/lib/staleness/writers/subagents.d.ts +2 -3
- package/dist/lib/staleness/writers/subagents.js +0 -4
- package/dist/lib/staleness/writers/types.d.ts +32 -0
- package/dist/lib/startup/command-registry.d.ts +6 -12
- package/dist/lib/startup/command-registry.js +11 -31
- package/dist/lib/startup/dev-build.d.ts +10 -0
- package/dist/lib/startup/dev-build.js +13 -1
- package/dist/lib/startup/spellcheck.d.ts +18 -0
- package/dist/lib/startup/spellcheck.js +45 -0
- package/dist/lib/state.d.ts +36 -9
- package/dist/lib/state.js +96 -46
- package/dist/lib/subagents-registry.d.ts +0 -7
- package/dist/lib/subagents-registry.js +38 -65
- package/dist/lib/subagents.d.ts +0 -33
- package/dist/lib/subagents.js +0 -75
- package/dist/lib/sync-umbrella.js +1 -1
- package/dist/lib/teams/agents.d.ts +41 -1
- package/dist/lib/teams/agents.js +187 -9
- package/dist/lib/teams/api.d.ts +11 -1
- package/dist/lib/teams/api.js +11 -2
- package/dist/lib/teams/delivery.d.ts +41 -0
- package/dist/lib/teams/delivery.js +60 -0
- package/dist/lib/teams/registry.d.ts +14 -0
- package/dist/lib/teams/registry.js +51 -1
- package/dist/lib/teams/remoteWorktree.d.ts +19 -0
- package/dist/lib/teams/remoteWorktree.js +29 -0
- package/dist/lib/teams/scheduler.d.ts +1 -1
- package/dist/lib/teams/scheduler.js +1 -1
- package/dist/lib/teams/worktree.d.ts +24 -0
- package/dist/lib/teams/worktree.js +42 -0
- package/dist/lib/terminal/backends/vscodium-agent.d.ts +10 -1
- package/dist/lib/terminal/backends/vscodium-agent.js +16 -5
- package/dist/lib/terminal/engine.d.ts +4 -0
- package/dist/lib/terminal/engine.js +8 -2
- package/dist/lib/terminal/types.d.ts +21 -2
- package/dist/lib/testdata/refresh-coordinator-worker.d.ts +1 -0
- package/dist/lib/testdata/refresh-coordinator-worker.js +23 -0
- package/dist/lib/tickets/list.d.ts +53 -0
- package/dist/lib/tickets/list.js +153 -0
- package/dist/lib/tmux/orphan-reap.d.ts +320 -0
- package/dist/lib/tmux/orphan-reap.js +644 -0
- package/dist/lib/tmux/session.d.ts +68 -3
- package/dist/lib/tmux/session.js +95 -5
- package/dist/lib/triggers/handlers.d.ts +20 -0
- package/dist/lib/triggers/handlers.js +6 -1
- package/dist/lib/types.d.ts +46 -34
- package/dist/lib/usage-fleet.d.ts +32 -0
- package/dist/lib/usage-fleet.js +125 -0
- package/dist/lib/usage-refresh.d.ts +3 -3
- package/dist/lib/usage-refresh.js +19 -16
- package/dist/lib/usage.d.ts +51 -47
- package/dist/lib/usage.js +145 -153
- package/dist/lib/version-duplicates.js +2 -2
- package/dist/lib/versions.d.ts +56 -13
- package/dist/lib/versions.js +195 -45
- package/dist/lib/watchdog/log.d.ts +1 -1
- package/dist/lib/watchdog/log.js +3 -3
- package/dist/lib/watchdog/rotate.d.ts +6 -6
- package/dist/lib/watchdog/rotate.js +6 -6
- package/dist/lib/workflows.d.ts +1 -1
- package/dist/lib/yaml-io.d.ts +47 -0
- package/dist/lib/yaml-io.js +55 -0
- package/package.json +2 -1
- package/scripts/install-helper.js +2 -2
- package/scripts/postinstall.js +23 -20
- package/dist/commands/defaults.d.ts +0 -7
- package/dist/commands/defaults.js +0 -107
- package/dist/commands/export.d.ts +0 -11
- package/dist/commands/export.js +0 -215
- package/dist/commands/helper.d.ts +0 -12
- package/dist/commands/helper.js +0 -87
- package/dist/commands/hosts.d.ts +0 -11
- package/dist/commands/hosts.js +0 -330
- package/dist/commands/lock.d.ts +0 -12
- package/dist/commands/lock.js +0 -70
- package/dist/commands/pull.d.ts +0 -17
- package/dist/commands/pull.js +0 -39
- package/dist/commands/push.d.ts +0 -14
- package/dist/commands/push.js +0 -30
- package/dist/commands/wallet.d.ts +0 -20
- package/dist/commands/wallet.js +0 -216
- package/dist/commands/worktree.d.ts +0 -19
- package/dist/commands/worktree.js +0 -272
- package/dist/lib/account-labels.d.ts +0 -24
- package/dist/lib/account-labels.js +0 -72
- package/dist/lib/auto-dispatch-linear.d.ts +0 -18
- package/dist/lib/auto-dispatch-linear.js +0 -107
- package/dist/lib/auto-dispatch-provider.d.ts +0 -10
- package/dist/lib/auto-dispatch-provider.js +0 -30
- package/dist/lib/auto-dispatch.d.ts +0 -93
- package/dist/lib/auto-dispatch.js +0 -128
- package/dist/lib/export.d.ts +0 -72
- package/dist/lib/export.js +0 -269
- package/dist/lib/lock.d.ts +0 -93
- package/dist/lib/lock.js +0 -207
- package/dist/lib/wallet/index.d.ts +0 -78
- package/dist/lib/wallet/index.js +0 -253
package/dist/lib/daemon.js
CHANGED
|
@@ -10,7 +10,7 @@ import { spawn, execFileSync } from 'child_process';
|
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import * as path from 'path';
|
|
12
12
|
import * as os from 'os';
|
|
13
|
-
import { getDaemonDir
|
|
13
|
+
import { getDaemonDir } from './state.js';
|
|
14
14
|
import { isAlive, killTree, backgroundSpawnOptions, waitForExit } from './platform/index.js';
|
|
15
15
|
import { listJobs as listAllJobs } from './routines.js';
|
|
16
16
|
import { syncAllProjectRoutines } from './routines-project.js';
|
|
@@ -26,9 +26,13 @@ import { BrowserIPCServer, getSocketPath as getBrowserIpcSocketPath } from './br
|
|
|
26
26
|
import { secretsBrokerSocketPath, brokerPidAlive } from './secrets/agent.js';
|
|
27
27
|
import { redactSecrets } from './redact.js';
|
|
28
28
|
import { getAgentsBinPath, getCliLaunch, BUN_VIRTUAL_ROOT } from './cli-entry.js';
|
|
29
|
-
import { isSchedulerEnabled, assertSchedulerEnabled, isDaemonEnabled } from './device-config.js';
|
|
29
|
+
import { isSchedulerEnabled, assertSchedulerEnabled, isDaemonEnabled, getConfigValue } from './device-config.js';
|
|
30
30
|
import { reapTerminalRoutineProcesses } from './routine-process-cleanup.js';
|
|
31
|
-
import { recordSubsystemOk, recordSubsystemError, SUBSYSTEM_SECRETS_BROKER, SUBSYSTEM_BROWSER_IPC } from './daemon-health.js';
|
|
31
|
+
import { recordSubsystemOk, recordSubsystemError, recordSubsystemErrorReason, readSubsystemHealth, SUBSYSTEM_SECRETS_BROKER, SUBSYSTEM_BROWSER_IPC, SUBSYSTEM_DAEMON_START } from './daemon-health.js';
|
|
32
|
+
import { startAccountStateService } from './account-state-service.js';
|
|
33
|
+
import { runActiveSessionsWarmTick, runFleetCacheWarmTick, runUsageRefreshTick } from './daemon-ticks.js';
|
|
34
|
+
import { emit, emitRoutineEnd } from './events.js';
|
|
35
|
+
import { readDaemonServicesConfig } from './daemon-services.js';
|
|
32
36
|
const PID_FILE = 'daemon.pid';
|
|
33
37
|
const LIFETIME_FILE = 'daemon.lifetime';
|
|
34
38
|
const LOCK_FILE = 'daemon.lock';
|
|
@@ -47,7 +51,68 @@ const MONITOR_TICK_MS = 60_000;
|
|
|
47
51
|
* wedge or an OS suspend the process survived.
|
|
48
52
|
*/
|
|
49
53
|
const CATCHUP_TICK_MS = 5 * 60_000;
|
|
54
|
+
/**
|
|
55
|
+
* Cadences for the in-process background ticks, named here beside the other
|
|
56
|
+
* tick constants rather than left as inline literals at their `setInterval`
|
|
57
|
+
* (RUSH-2423). Each is a deliberate trade, not a round number:
|
|
58
|
+
*
|
|
59
|
+
* - **Self-heal** repairs slow rot (a stale non-default version, an invalid
|
|
60
|
+
* plugin manifest), so it is cheap to be late and expensive to run often —
|
|
61
|
+
* hence 6h, plus one staggered kickoff shortly after start so shims and PATH
|
|
62
|
+
* settle without making launch itself busy.
|
|
63
|
+
* - **Broker self-heal** is a bare `agentPing`, and the failure it recovers
|
|
64
|
+
* from wedges every keychain-backed secret on the box, so it runs minutely.
|
|
65
|
+
* - **Keychain reap** shells `ps` once per pass; 5 min bounds that cost while
|
|
66
|
+
* still clearing orphans well inside a human's attention span.
|
|
67
|
+
* - **State-dir self-check** is two `fs` reads. Its env override exists for
|
|
68
|
+
* tests, which cannot wait a minute to observe the self-terminate guard.
|
|
69
|
+
*/
|
|
70
|
+
const SELF_HEAL_TICK_MS = 6 * 60 * 60_000;
|
|
71
|
+
const SELF_HEAL_KICKOFF_MS = 30_000;
|
|
72
|
+
const BROKER_SELF_HEAL_TICK_MS = 60_000;
|
|
73
|
+
const KEYCHAIN_REAP_TICK_MS = 5 * 60_000;
|
|
74
|
+
// RUSH-2501: reap tmux sessions whose panes are all dead every 5 minutes.
|
|
75
|
+
const DEAD_PANE_REAP_TICK_MS = 5 * 60_000;
|
|
76
|
+
const STATE_DIR_CHECK_TICK_MS = 60_000;
|
|
77
|
+
// Watchdog nudges this host's own stalled sessions; device-probe refreshes
|
|
78
|
+
// registered devices' reachability and surfaces newly appeared tailnet nodes.
|
|
79
|
+
// Both are daemon-owned housekeeping timers, NOT routines (RUSH-2495) — plain
|
|
80
|
+
// in-process intervals the daemon holds directly, same cadence the old timers ran.
|
|
81
|
+
const WATCHDOG_TICK_MS = 3 * 60_000;
|
|
82
|
+
const DEVICE_PROBE_TICK_MS = 3 * 60_000;
|
|
83
|
+
// Keep the active-session cache/journal fresh for sessions watch + Factory.
|
|
84
|
+
// Matches DEFAULT_ACTIVE_CACHE_MAX_AGE_MS (15s) so long-lived watchers never
|
|
85
|
+
// outlive the publisher (RUSH-2484 — CLI-owned continuous publish).
|
|
86
|
+
const ACTIVE_SESSIONS_WARM_TICK_MS = 15_000;
|
|
50
87
|
const WEDGE_THRESHOLD_TICKS = 3;
|
|
88
|
+
/**
|
|
89
|
+
* Crash-loop prevention (RUSH-2418). Three layers, because none of them alone
|
|
90
|
+
* bounds a daemon that dies during startup:
|
|
91
|
+
*
|
|
92
|
+
* 1. **The OS supervisor paces the respawn.** `KeepAlive` with no
|
|
93
|
+
* `ThrottleInterval` lets launchd relaunch on its ~10s default, so a daemon
|
|
94
|
+
* that dies while booting is restarted six times a minute forever — the exact
|
|
95
|
+
* failure the menu-bar helper hit (`menubar/install-menubar.ts`: 38 orphaned
|
|
96
|
+
* `agents doctor` children, load average 490). systemd's `Restart=always`
|
|
97
|
+
* with no `StartLimit*` is the same uncapped loop.
|
|
98
|
+
* 2. **`StartLimitBurst` gives systemd a real cap** — after this many starts
|
|
99
|
+
* inside the interval the unit is put in `failed` and stops respawning, so a
|
|
100
|
+
* genuinely broken install stops burning the box and `systemctl --user status`
|
|
101
|
+
* names it. launchd has no burst equivalent; the throttle is its whole answer.
|
|
102
|
+
* 3. **The application-level circuit breaker** below stops *auto*-starts from
|
|
103
|
+
* re-entering the loop from the other direction — a foreground command that
|
|
104
|
+
* calls `ensureDaemonStarted()` on every invocation.
|
|
105
|
+
*/
|
|
106
|
+
const DAEMON_THROTTLE_SECONDS = 30;
|
|
107
|
+
const DAEMON_START_LIMIT_INTERVAL_SECONDS = 300;
|
|
108
|
+
const DAEMON_START_LIMIT_BURST = 5;
|
|
109
|
+
/**
|
|
110
|
+
* How many consecutive failed daemon starts disable the *implicit* auto-start
|
|
111
|
+
* (`ensureDaemonStarted`). Matches `DAEMON_START_LIMIT_BURST` so the two layers
|
|
112
|
+
* give up together rather than one silently masking the other. `agents daemon
|
|
113
|
+
* start` is the deliberate override and is never gated by this.
|
|
114
|
+
*/
|
|
115
|
+
export const DAEMON_AUTOSTART_FAILURE_LIMIT = 5;
|
|
51
116
|
/**
|
|
52
117
|
* RUSH-1817: decide whether the daemon should (re)take over hosting the secrets
|
|
53
118
|
* broker. The startup host decision is one-shot; this drives the periodic
|
|
@@ -64,16 +129,44 @@ export function schedulerGateTransition(running, enabled) {
|
|
|
64
129
|
return enabled ? 'reload' : 'stop';
|
|
65
130
|
return enabled ? 'boot' : 'none';
|
|
66
131
|
}
|
|
67
|
-
|
|
68
|
-
|
|
132
|
+
/**
|
|
133
|
+
* Wrap an async routine so it runs AT MOST ONCE, however many callers fire it.
|
|
134
|
+
*
|
|
135
|
+
* Extracted rather than left as a `let shuttingDown = false` inside runDaemon so
|
|
136
|
+
* the property can actually be tested (RUSH-2423). The daemon's shutdown is
|
|
137
|
+
* reachable from SIGTERM, SIGINT, and the state-dir self-check, but a real
|
|
138
|
+
* shutdown completes in ~26ms, so the re-entrant window is not reachable from
|
|
139
|
+
* outside the process — an end-to-end "send three signals" test passes with the
|
|
140
|
+
* guard removed and proves nothing. The mechanism is what is testable, so the
|
|
141
|
+
* mechanism is what is separated out.
|
|
142
|
+
*
|
|
143
|
+
* The flag is set synchronously before the first `await`, which is what makes
|
|
144
|
+
* this safe: two callers in the same tick cannot both get past it.
|
|
145
|
+
*
|
|
146
|
+
* A rejected `fn` leaves the guard SET — one attempt is all there is, and the
|
|
147
|
+
* rejection propagates to the caller that made it. That is right for shutdown
|
|
148
|
+
* (a failed shutdown must not be silently retried by the next signal) but is
|
|
149
|
+
* the thing to re-examine before giving this a second consumer.
|
|
150
|
+
*/
|
|
151
|
+
export function singleShot(fn) {
|
|
152
|
+
let ran = false;
|
|
153
|
+
return async () => {
|
|
154
|
+
if (ran)
|
|
155
|
+
return;
|
|
156
|
+
ran = true;
|
|
157
|
+
await fn();
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
function ensureDaemonDir() {
|
|
161
|
+
const dir = getDaemonDir();
|
|
69
162
|
fs.mkdirSync(dir, { recursive: true });
|
|
70
163
|
return dir;
|
|
71
164
|
}
|
|
72
165
|
function getPidPath() {
|
|
73
|
-
return path.join(
|
|
166
|
+
return path.join(ensureDaemonDir(), PID_FILE);
|
|
74
167
|
}
|
|
75
168
|
function getLockPath() {
|
|
76
|
-
return path.join(
|
|
169
|
+
return path.join(ensureDaemonDir(), LOCK_FILE);
|
|
77
170
|
}
|
|
78
171
|
/**
|
|
79
172
|
* Acquire an exclusive start lock. Returns a release function on success,
|
|
@@ -116,8 +209,16 @@ function acquireStartLock() {
|
|
|
116
209
|
throw err;
|
|
117
210
|
}
|
|
118
211
|
}
|
|
119
|
-
|
|
120
|
-
|
|
212
|
+
/**
|
|
213
|
+
* Absolute path to the daemon's structured log.
|
|
214
|
+
*
|
|
215
|
+
* Exported because two commands rebuilt the same path from a hardcoded
|
|
216
|
+
* `'logs.jsonl'` literal (`commands/daemon.ts`, `commands/routines.ts`), so
|
|
217
|
+
* renaming the file would have silently pointed them at nothing (RUSH-2423).
|
|
218
|
+
* One definition, three callers.
|
|
219
|
+
*/
|
|
220
|
+
export function getDaemonLogPath() {
|
|
221
|
+
return path.join(ensureDaemonDir(), LOG_FILE);
|
|
121
222
|
}
|
|
122
223
|
function getLaunchdPlistPath() {
|
|
123
224
|
return path.join(os.homedir(), 'Library', 'LaunchAgents', `${PLIST_NAME}.plist`);
|
|
@@ -150,7 +251,7 @@ export function removeDaemonPid() {
|
|
|
150
251
|
}
|
|
151
252
|
}
|
|
152
253
|
function getHeartbeatPath() {
|
|
153
|
-
return path.join(
|
|
254
|
+
return path.join(ensureDaemonDir(), HEARTBEAT_FILE);
|
|
154
255
|
}
|
|
155
256
|
export function writeHeartbeat(pid = process.pid) {
|
|
156
257
|
const hb = { lastTick: new Date().toISOString(), pid };
|
|
@@ -327,23 +428,23 @@ function evictIncumbentDaemon(pid) {
|
|
|
327
428
|
killTree(pid); // positive pid: SIGKILL reaches the daemon, not its job children
|
|
328
429
|
waitForExit(pid, STOP_KILL_GRACE_MS);
|
|
329
430
|
}
|
|
330
|
-
/** Directory that registers every live daemon of THIS device (one per
|
|
431
|
+
/** Directory that registers every live daemon of THIS device (one per state dir). */
|
|
331
432
|
function getDaemonInstancesDir() {
|
|
332
|
-
return path.join(
|
|
433
|
+
return path.join(getDaemonDir(), 'instances');
|
|
333
434
|
}
|
|
334
435
|
/**
|
|
335
436
|
* Record this daemon in the device's instance registry — a marker file named by
|
|
336
437
|
* pid under `<daemonDir>/instances/`. The registry, not a process scan, is how
|
|
337
438
|
* the reaper enumerates the device singleton: because the dir lives INSIDE the
|
|
338
|
-
*
|
|
439
|
+
* state dir (`AGENTS_DAEMON_DIR` ?? `<HOME>/.agents/.cache/helpers/daemon`), every
|
|
339
440
|
* daemon of one device — however it was launched — registers in the same place,
|
|
340
441
|
* while a genuinely separate install/home or a test fixture registers under its
|
|
341
|
-
* own
|
|
442
|
+
* own state dir and is invisible here. This is what fixes the two-entry pile-up:
|
|
342
443
|
* the compiled `dist/bin/agents` binary and the `node <shim>` JS entry have
|
|
343
444
|
* different `process.argv[1]`, so the old launch-entry-scoped `ps` match never
|
|
344
445
|
* reaped across them and duplicates accumulated (78 observed on one box), every
|
|
345
446
|
* routine double-firing. Best-effort — the reaper self-heals a missing/stale
|
|
346
|
-
* marker, and reading another process's ENV to key on the
|
|
447
|
+
* marker, and reading another process's ENV to key on the state dir directly is
|
|
347
448
|
* not portable (hardened macOS hides it from `ps`), so identity rides the shared
|
|
348
449
|
* on-disk registry instead. No-op on Windows (POSIX-only reaper).
|
|
349
450
|
*/
|
|
@@ -461,7 +562,7 @@ function rotateLogsIfNeeded(logPath) {
|
|
|
461
562
|
}
|
|
462
563
|
/** Append a JSONL log entry to the daemon log file (owner-only permissions). */
|
|
463
564
|
export function log(level, message) {
|
|
464
|
-
const logPath =
|
|
565
|
+
const logPath = getDaemonLogPath();
|
|
465
566
|
rotateLogsIfNeeded(logPath);
|
|
466
567
|
const entry = { ts: new Date().toISOString(), level: level.toUpperCase(), message: redactSecrets(message) };
|
|
467
568
|
fs.appendFileSync(logPath, JSON.stringify(entry) + '\n', 'utf-8');
|
|
@@ -469,6 +570,22 @@ export function log(level, message) {
|
|
|
469
570
|
fs.chmodSync(logPath, 0o600);
|
|
470
571
|
}
|
|
471
572
|
catch { /* best effort */ }
|
|
573
|
+
// Mirror into the unified event stream so `agents events --module daemon` sees
|
|
574
|
+
// always-on process lifecycle the same way secrets/browser/computer do.
|
|
575
|
+
// Fail soft — the daemon log file remains the primary sink for `daemon logs`.
|
|
576
|
+
try {
|
|
577
|
+
const lvl = level.toUpperCase();
|
|
578
|
+
const event = lvl === 'ERROR' || lvl === 'FATAL' ? 'daemon.error'
|
|
579
|
+
: lvl === 'START' || /starting|started/i.test(message) ? 'daemon.start'
|
|
580
|
+
: lvl === 'STOP' || /stopping|stopped|shutting down/i.test(message) ? 'daemon.stop'
|
|
581
|
+
: 'daemon.info';
|
|
582
|
+
emit(event, {
|
|
583
|
+
module: 'daemon',
|
|
584
|
+
detail: redactSecrets(message).slice(0, 500),
|
|
585
|
+
status: lvl,
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
catch { /* never crash the daemon on event-log failure */ }
|
|
472
589
|
}
|
|
473
590
|
/** Main daemon loop: load jobs, schedule crons, monitor runs, and handle signals. */
|
|
474
591
|
/**
|
|
@@ -532,6 +649,129 @@ export function warnEphemeralDaemonRoot(resolveBin = getAgentsBinPath) {
|
|
|
532
649
|
return null;
|
|
533
650
|
}
|
|
534
651
|
}
|
|
652
|
+
// ---------------------------------------------------------------------------
|
|
653
|
+
// Module-level periodic maintenance helpers (RUSH-2422)
|
|
654
|
+
//
|
|
655
|
+
// These were inline closures inside runDaemon(). Moved here so they are
|
|
656
|
+
// named in stack traces, readable without scrolling through runDaemon's
|
|
657
|
+
// 500-line body, and not recreated on every function invocation.
|
|
658
|
+
//
|
|
659
|
+
// Two functions (runBrokerSelfHeal, runStateDirSelfCheck) remain inline:
|
|
660
|
+
// • runBrokerSelfHeal closes over `hostedBroker` — a mutable runDaemon-
|
|
661
|
+
// local var also used by handleShutdown; lifting it requires both vars to
|
|
662
|
+
// be module-level, which is wider scope than this ticket.
|
|
663
|
+
// • runStateDirSelfCheck closes over `lifetimePath` and `lifetimeToken` —
|
|
664
|
+
// per-boot constants that cannot be pre-computed at module load time.
|
|
665
|
+
// ---------------------------------------------------------------------------
|
|
666
|
+
// In-flight guards: each flag prevents a slow tick from being re-entered
|
|
667
|
+
// by the next timer fire before the previous one finishes.
|
|
668
|
+
let healing = false;
|
|
669
|
+
let reapingKeychain = false;
|
|
670
|
+
let reapingDeadPanes = false;
|
|
671
|
+
/**
|
|
672
|
+
* Resource self-heal: fill missing resources, repair invalid manifests, and
|
|
673
|
+
* fast-forward pristine stale plugins. Conservative 'safe' mode: never
|
|
674
|
+
* overwrites hand-edited content. Runs ~every 6h plus once ~30s after startup.
|
|
675
|
+
*
|
|
676
|
+
* Does not run when the daemon's state directory no longer exists — that is the
|
|
677
|
+
* self-terminate guard's signal to shut down; background maintenance must not
|
|
678
|
+
* recreate the tree while it is mid-exit.
|
|
679
|
+
*/
|
|
680
|
+
async function runHealCheck() {
|
|
681
|
+
if (healing)
|
|
682
|
+
return;
|
|
683
|
+
if (!fs.existsSync(getDaemonDir()))
|
|
684
|
+
return;
|
|
685
|
+
healing = true;
|
|
686
|
+
try {
|
|
687
|
+
const { runSelfHeal, selfHealChangedAnything, selfHealNeedsAttention, summarizeSelfHeal } = await import('./self-heal/registry.js');
|
|
688
|
+
if (!fs.existsSync(getDaemonDir()))
|
|
689
|
+
return;
|
|
690
|
+
const report = await runSelfHeal({ mode: 'safe' });
|
|
691
|
+
if (selfHealChangedAnything(report) || selfHealNeedsAttention(report)) {
|
|
692
|
+
log('INFO', `self-heal: ${summarizeSelfHeal(report)}`);
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
catch (err) {
|
|
696
|
+
log('ERROR', `self-heal check failed: ${err.message}`);
|
|
697
|
+
}
|
|
698
|
+
finally {
|
|
699
|
+
healing = false;
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* Keychain orphan reaper: kill stuck keychain helper and `agents` processes
|
|
704
|
+
* whose keychain call never returned. Runs every 5 min. Single-executor: only
|
|
705
|
+
* the daemon runs this, so no cross-device race.
|
|
706
|
+
*/
|
|
707
|
+
async function runKeychainReap() {
|
|
708
|
+
if (reapingKeychain)
|
|
709
|
+
return;
|
|
710
|
+
reapingKeychain = true;
|
|
711
|
+
try {
|
|
712
|
+
const { reapOrphanedKeychainProcesses } = await import('./secrets/reaper.js');
|
|
713
|
+
const result = reapOrphanedKeychainProcesses();
|
|
714
|
+
if (result.reaped > 0) {
|
|
715
|
+
log('WARN', `Reaped ${result.reaped} keychain orphan/stuck process(es)`);
|
|
716
|
+
for (const d of result.details)
|
|
717
|
+
log('WARN', ` ${d}`);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
catch (err) {
|
|
721
|
+
log('ERROR', `Keychain reaper failed: ${err.message}`);
|
|
722
|
+
}
|
|
723
|
+
finally {
|
|
724
|
+
reapingKeychain = false;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* RUSH-2501: kill tmux sessions on the helper socket whose panes are ALL dead.
|
|
729
|
+
* RUSH-2521: and terminate the helper processes (MCP servers, harness background
|
|
730
|
+
* daemons) whose owning agent has exited — killing the pane only SIGHUPs its
|
|
731
|
+
* foreground process group, so anything that left that group outlives it.
|
|
732
|
+
*
|
|
733
|
+
* Runs every DEAD_PANE_REAP_TICK_MS (5 min). The daemon is the single executor
|
|
734
|
+
* so no UI surface can race it.
|
|
735
|
+
*/
|
|
736
|
+
async function runDeadPaneReap() {
|
|
737
|
+
if (reapingDeadPanes)
|
|
738
|
+
return;
|
|
739
|
+
reapingDeadPanes = true;
|
|
740
|
+
try {
|
|
741
|
+
const { reapDeadTmuxPanes } = await import('./tmux/session.js');
|
|
742
|
+
const { getDefaultSocketPath } = await import('./tmux/paths.js');
|
|
743
|
+
const result = await reapDeadTmuxPanes(getDefaultSocketPath());
|
|
744
|
+
for (const w of result.warnings)
|
|
745
|
+
log('WARN', `Dead-pane reaper: ${w}`);
|
|
746
|
+
if (result.processes > 0) {
|
|
747
|
+
log('INFO', `Dead-pane reaper: terminated ${result.processes} orphaned helper process(es)`);
|
|
748
|
+
for (const d of result.processDetails)
|
|
749
|
+
log('INFO', ` ${d}`);
|
|
750
|
+
}
|
|
751
|
+
if (result.reaped > 0) {
|
|
752
|
+
log('INFO', `Dead-pane reaper: reaped ${result.reaped} session(s)`);
|
|
753
|
+
for (const d of result.details)
|
|
754
|
+
log('INFO', ` killed ${d}`);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
catch (err) {
|
|
758
|
+
log('ERROR', `Dead-pane reaper failed: ${err.message}`);
|
|
759
|
+
}
|
|
760
|
+
finally {
|
|
761
|
+
reapingDeadPanes = false;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
/**
|
|
765
|
+
* One monitor tick: write heartbeat + drain the running-job monitor + reap
|
|
766
|
+
* stale terminal routine process groups. Runs on MONITOR_TICK_MS.
|
|
767
|
+
*/
|
|
768
|
+
function runMonitorTick() {
|
|
769
|
+
writeHeartbeat();
|
|
770
|
+
monitorRunningJobs();
|
|
771
|
+
const reaped = reapTerminalRoutineProcesses();
|
|
772
|
+
if (reaped.length > 0)
|
|
773
|
+
log('WARN', `Reaped ${reaped.length} terminal routine process group(s): ${reaped.join(', ')}`);
|
|
774
|
+
}
|
|
535
775
|
export async function runDaemon() {
|
|
536
776
|
// Single-instance guard (last-wins, SING-11): a direct `agents __daemon-run`
|
|
537
777
|
// (manual, or a service-manager restart racing a live predecessor) EVICTS the
|
|
@@ -546,13 +786,29 @@ export async function runDaemon() {
|
|
|
546
786
|
}
|
|
547
787
|
// Unlike the pid and heartbeat files, this marker is written exactly once
|
|
548
788
|
// for this daemon lifetime. Status probes deliberately repair those other
|
|
549
|
-
// files, so they cannot prove that the original state
|
|
550
|
-
const lifetimePath = path.join(
|
|
789
|
+
// files, so they cannot prove that the original state dir still exists.
|
|
790
|
+
const lifetimePath = path.join(getDaemonDir(), LIFETIME_FILE);
|
|
551
791
|
const lifetimeToken = `${process.pid}:${Date.now()}`;
|
|
552
792
|
fs.writeFileSync(lifetimePath, lifetimeToken, 'utf-8');
|
|
553
793
|
log('INFO', `Daemon started (PID: ${process.pid})`);
|
|
554
794
|
anchorDaemonCwd();
|
|
555
795
|
warnEphemeralDaemonRoot();
|
|
796
|
+
// Fold legacy per-device config stores into the central
|
|
797
|
+
// fleet.devices.<name>.config block (idempotent, cheap no-op once folded).
|
|
798
|
+
// The daemon boots via `agents __daemon-run`, which bypasses bootstrap's
|
|
799
|
+
// migration sentinel — so the daemon runs this itself so its scheduler/
|
|
800
|
+
// watchdog gates read the converged store.
|
|
801
|
+
try {
|
|
802
|
+
const { migrateDeviceConfigToCentral } = await import('./devices/config-migration.js');
|
|
803
|
+
migrateDeviceConfigToCentral();
|
|
804
|
+
}
|
|
805
|
+
catch (err) {
|
|
806
|
+
log('WARN', `device config migration failed: ${err.message}`);
|
|
807
|
+
}
|
|
808
|
+
// Per-service toggles live outside device-config so they can be checked by
|
|
809
|
+
// service clients (e.g. secrets) without loading the whole config stack.
|
|
810
|
+
let servicesConfig = readDaemonServicesConfig();
|
|
811
|
+
const isEnabled = (id) => servicesConfig.services[id] !== false;
|
|
556
812
|
// The daemon holds NO Claude credential of its own. Routine runs authenticate
|
|
557
813
|
// exactly like an interactive `agents run`: through the per-account
|
|
558
814
|
// CLAUDE_CONFIG_DIR login on this device (its own auto-refreshing
|
|
@@ -584,22 +840,27 @@ export async function runDaemon() {
|
|
|
584
840
|
// here must not stop the daemon. Retiring the standalone launchd service is
|
|
585
841
|
// the follow-on (#416 step 2 / #417).
|
|
586
842
|
let hostedBroker = null;
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
843
|
+
if (isEnabled('secrets-broker')) {
|
|
844
|
+
try {
|
|
845
|
+
const { agentPing, startHostedBroker } = await import('./secrets/agent.js');
|
|
846
|
+
if ((await agentPing()).reachable) {
|
|
847
|
+
log('INFO', 'Secrets broker already running (standalone); daemon not hosting it');
|
|
848
|
+
}
|
|
849
|
+
else {
|
|
850
|
+
hostedBroker = await startHostedBroker();
|
|
851
|
+
if (hostedBroker)
|
|
852
|
+
log('INFO', 'Secrets broker hosted in daemon (socket-first)');
|
|
853
|
+
}
|
|
854
|
+
recordSubsystemOk(SUBSYSTEM_SECRETS_BROKER);
|
|
591
855
|
}
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
856
|
+
catch (err) {
|
|
857
|
+
const message = err.message;
|
|
858
|
+
log('WARN', `Secrets broker host skipped: ${message}`);
|
|
859
|
+
recordSubsystemError(SUBSYSTEM_SECRETS_BROKER, message);
|
|
596
860
|
}
|
|
597
|
-
recordSubsystemOk(SUBSYSTEM_SECRETS_BROKER);
|
|
598
861
|
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
log('WARN', `Secrets broker host skipped: ${message}`);
|
|
602
|
-
recordSubsystemError(SUBSYSTEM_SECRETS_BROKER, message);
|
|
862
|
+
else {
|
|
863
|
+
log('INFO', 'Secrets broker service disabled; daemon not hosting it');
|
|
603
864
|
}
|
|
604
865
|
// scheduler.enabled=false in this machine's device doc means NO routines fire
|
|
605
866
|
// here — the scheduler and its catchup recovery simply never start, while the
|
|
@@ -608,7 +869,9 @@ export async function runDaemon() {
|
|
|
608
869
|
// (`routines add` auto-start, manual `routines start`) raise. The gate is
|
|
609
870
|
// re-evaluated on every SIGHUP reload (handleReload below) via
|
|
610
871
|
// schedulerGateTransition, so flipping the key never needs a daemon restart.
|
|
611
|
-
|
|
872
|
+
// Also honour the daemon-services toggle so `agents daemon services disable scheduler`
|
|
873
|
+
// has a single, obvious effect.
|
|
874
|
+
const schedulerEnabledAtBoot = isSchedulerEnabled() && isEnabled('scheduler');
|
|
612
875
|
if (!schedulerEnabledAtBoot) {
|
|
613
876
|
try {
|
|
614
877
|
assertSchedulerEnabled();
|
|
@@ -616,6 +879,8 @@ export async function runDaemon() {
|
|
|
616
879
|
catch (err) {
|
|
617
880
|
log('WARN', err.message);
|
|
618
881
|
}
|
|
882
|
+
if (!isEnabled('scheduler'))
|
|
883
|
+
log('INFO', 'Scheduler service disabled; no routines will fire');
|
|
619
884
|
}
|
|
620
885
|
const triggerJob = async (config, ctx) => {
|
|
621
886
|
const jobLabel = config.command
|
|
@@ -624,6 +889,13 @@ export async function runDaemon() {
|
|
|
624
889
|
? `workflow: ${config.workflow}`
|
|
625
890
|
: `agent: ${config.agent}`;
|
|
626
891
|
log('INFO', `Triggering job '${config.name}' (${jobLabel})`);
|
|
892
|
+
emit('routine.start', {
|
|
893
|
+
module: 'routine',
|
|
894
|
+
name: config.name,
|
|
895
|
+
kind: config.command ? 'command' : config.workflow ? 'workflow' : 'agent',
|
|
896
|
+
...(config.agent ? { agent: config.agent } : {}),
|
|
897
|
+
...(config.workflow ? { workflow: config.workflow } : {}),
|
|
898
|
+
});
|
|
627
899
|
// RUSH-2030: branded desktop notification on start (agent/workflow routines;
|
|
628
900
|
// suppressed for command housekeeping). Finish/output is fired from the
|
|
629
901
|
// onFinish hook below — executeJobDetached finalizes the run in-process, so
|
|
@@ -636,6 +908,7 @@ export async function runDaemon() {
|
|
|
636
908
|
try {
|
|
637
909
|
const meta = await executeJobDetached(config, {
|
|
638
910
|
onFinish: (final) => {
|
|
911
|
+
emitRoutineEnd(final);
|
|
639
912
|
try {
|
|
640
913
|
notifyRoutineFinish(final);
|
|
641
914
|
}
|
|
@@ -657,6 +930,11 @@ export async function runDaemon() {
|
|
|
657
930
|
catch (err) {
|
|
658
931
|
const message = err.message;
|
|
659
932
|
log('ERROR', `Job '${config.name}' failed to spawn: ${message}`);
|
|
933
|
+
emitRoutineEnd({
|
|
934
|
+
jobName: config.name,
|
|
935
|
+
status: 'failed',
|
|
936
|
+
detail: redactSecrets(message).slice(0, 500),
|
|
937
|
+
});
|
|
660
938
|
// RUSH-2030: the START ping already fired unconditionally above. A pre-spawn
|
|
661
939
|
// failure produces no run record and thus no onFinish, so send a synthetic
|
|
662
940
|
// "failed to start" finish here — otherwise the user is left with an orphaned
|
|
@@ -718,22 +996,137 @@ export async function runDaemon() {
|
|
|
718
996
|
}
|
|
719
997
|
if (schedulerEnabledAtBoot)
|
|
720
998
|
bootScheduler();
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
//
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
999
|
+
// Usage and authentication are first-party device state, so the daemon owns
|
|
1000
|
+
// their timers directly rather than scheduling them as built-in routines.
|
|
1001
|
+
// Explicit CLI refreshes converge on the same cross-process refresh leases.
|
|
1002
|
+
const accountStateService = isEnabled('account-state')
|
|
1003
|
+
? startAccountStateService({
|
|
1004
|
+
refreshUsage: runUsageRefreshTick,
|
|
1005
|
+
refreshAuth: runFleetCacheWarmTick,
|
|
1006
|
+
onError: (area, error) => log('WARN', `${area} state refresh failed: ${error.message}`),
|
|
1007
|
+
})
|
|
1008
|
+
: null;
|
|
1009
|
+
if (!accountStateService)
|
|
1010
|
+
log('INFO', 'Account-state service disabled');
|
|
1011
|
+
// Active-sessions publish-own: continuous journal writer for `sessions watch`.
|
|
1012
|
+
// Fire once immediately so a cold daemon does not leave watchers on
|
|
1013
|
+
// "awaiting publisher" for a full tick (RUSH-2484).
|
|
1014
|
+
let activeSessionsWarmInFlight = false;
|
|
1015
|
+
const runActiveSessionsWarm = async () => {
|
|
1016
|
+
if (activeSessionsWarmInFlight)
|
|
1017
|
+
return;
|
|
1018
|
+
activeSessionsWarmInFlight = true;
|
|
1019
|
+
try {
|
|
1020
|
+
await runActiveSessionsWarmTick();
|
|
1021
|
+
}
|
|
1022
|
+
catch (err) {
|
|
1023
|
+
log('WARN', `active-sessions warm failed: ${err.message}`);
|
|
1024
|
+
}
|
|
1025
|
+
finally {
|
|
1026
|
+
activeSessionsWarmInFlight = false;
|
|
1027
|
+
}
|
|
1028
|
+
};
|
|
1029
|
+
void runActiveSessionsWarm();
|
|
1030
|
+
const activeSessionsWarmInterval = setInterval(() => { void runActiveSessionsWarm(); }, ACTIVE_SESSIONS_WARM_TICK_MS);
|
|
1031
|
+
// Watchdog: nudge this host's own stalled agent sessions. Gated on the
|
|
1032
|
+
// `watchdog.enabled` device-config flag (`agents watchdog enable`), so the
|
|
1033
|
+
// timer always fires but only does work when the user opted in. Overlap-safe
|
|
1034
|
+
// via the in-flight guard (a slow pass never overlaps the next tick).
|
|
1035
|
+
let watchdogInterval;
|
|
1036
|
+
if (isEnabled('watchdog')) {
|
|
1037
|
+
let watchdogInFlight = false;
|
|
1038
|
+
const runWatchdogTick = async () => {
|
|
1039
|
+
if (watchdogInFlight)
|
|
1040
|
+
return;
|
|
1041
|
+
watchdogInFlight = true;
|
|
1042
|
+
try {
|
|
1043
|
+
if (getConfigValue('watchdog.enabled').value !== true)
|
|
1044
|
+
return;
|
|
1045
|
+
const { runWatchdogPass } = await import('./watchdog/service.js');
|
|
1046
|
+
const result = await runWatchdogPass({ nudge: true });
|
|
1047
|
+
log('INFO', `watchdog: ${result.counts.total} live, ${result.counts.stalled} stalled, ${result.counts.nudged} nudged`);
|
|
1048
|
+
emit('watchdog.action', {
|
|
1049
|
+
module: 'watchdog',
|
|
1050
|
+
total: result.counts.total,
|
|
1051
|
+
stalled: result.counts.stalled,
|
|
1052
|
+
nudged: result.counts.nudged,
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
catch (err) {
|
|
1056
|
+
log('WARN', `watchdog tick failed: ${err.message}`);
|
|
1057
|
+
}
|
|
1058
|
+
finally {
|
|
1059
|
+
watchdogInFlight = false;
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
watchdogInterval = setInterval(() => { void runWatchdogTick(); }, WATCHDOG_TICK_MS);
|
|
1063
|
+
}
|
|
1064
|
+
else {
|
|
1065
|
+
log('INFO', 'Watchdog service disabled');
|
|
1066
|
+
}
|
|
1067
|
+
// Device probe: refresh registered devices' reachability and detect newly
|
|
1068
|
+
// appeared tailnet nodes, dropping a sentinel per pending device so the
|
|
1069
|
+
// menu-bar helper can surface "NEW DEVICES → Register / Ignore". Refresh mode
|
|
1070
|
+
// never auto-registers a newcomer; a machine without tailscale is a clean
|
|
1071
|
+
// no-op. `reconcilePendingSentinels` re-subtracts the ignore-list AND the
|
|
1072
|
+
// registered roster, so a dismissed or already-known device is never
|
|
1073
|
+
// re-surfaced (RUSH-2495 + registry-empty pollution). On soft-fail (no
|
|
1074
|
+
// tailscale) we still prune registered/ignored sentinels so a hermetic test
|
|
1075
|
+
// leak cannot leave fleet boxes in NEW DEVICES forever.
|
|
1076
|
+
let deviceProbeInterval;
|
|
1077
|
+
if (isEnabled('device-probe')) {
|
|
1078
|
+
let deviceProbeInFlight = false;
|
|
1079
|
+
const runDeviceProbeTick = async () => {
|
|
1080
|
+
if (deviceProbeInFlight)
|
|
1081
|
+
return;
|
|
1082
|
+
deviceProbeInFlight = true;
|
|
1083
|
+
try {
|
|
1084
|
+
const { runDeviceSync } = await import('./devices/sync.js');
|
|
1085
|
+
const { reconcilePendingSentinels, pruneDismissedPendingSentinels, } = await import('./devices/pending.js');
|
|
1086
|
+
const dev = await runDeviceSync({ soft: true, mode: 'refresh' });
|
|
1087
|
+
if (!dev.ok) {
|
|
1088
|
+
await pruneDismissedPendingSentinels();
|
|
1089
|
+
if (dev.reason)
|
|
1090
|
+
log('WARN', `device probe soft-fail: ${dev.reason}`);
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
await reconcilePendingSentinels(dev.pending);
|
|
1094
|
+
if (dev.pending.length) {
|
|
1095
|
+
log('INFO', `devices: ${dev.pending.length} new pending (${dev.pending.map((p) => p.name).join(', ')})`);
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
catch (err) {
|
|
1099
|
+
log('WARN', `device probe tick failed: ${err.message}`);
|
|
1100
|
+
}
|
|
1101
|
+
finally {
|
|
1102
|
+
deviceProbeInFlight = false;
|
|
1103
|
+
}
|
|
1104
|
+
};
|
|
1105
|
+
// Fire once on start so a leftover pollution set is cleared without waiting
|
|
1106
|
+
// for the first interval tick (the 3-minute lag is how the menubar sat on
|
|
1107
|
+
// 20 phantom NEW DEVICES after a hermetic leak).
|
|
1108
|
+
void runDeviceProbeTick();
|
|
1109
|
+
deviceProbeInterval = setInterval(() => { void runDeviceProbeTick(); }, DEVICE_PROBE_TICK_MS);
|
|
1110
|
+
}
|
|
1111
|
+
else {
|
|
1112
|
+
log('INFO', 'Device-probe service disabled');
|
|
1113
|
+
}
|
|
728
1114
|
// Monitor engine: event-triggered watchers, beside the cron scheduler. Same
|
|
729
1115
|
// daemon, same dispatch seam — a monitor is a routine whose trigger is a
|
|
730
1116
|
// watched source instead of a clock. Reloads on SIGHUP alongside the scheduler.
|
|
731
|
-
const monitorEngine =
|
|
732
|
-
|
|
733
|
-
|
|
1117
|
+
const monitorEngine = isEnabled('monitors')
|
|
1118
|
+
? new MonitorEngine((level, message) => log(level, message))
|
|
1119
|
+
: null;
|
|
1120
|
+
if (monitorEngine) {
|
|
1121
|
+
try {
|
|
1122
|
+
monitorEngine.start();
|
|
1123
|
+
}
|
|
1124
|
+
catch (err) {
|
|
1125
|
+
log('ERROR', `Monitor engine failed to start: ${err.message}`);
|
|
1126
|
+
}
|
|
734
1127
|
}
|
|
735
|
-
|
|
736
|
-
log('
|
|
1128
|
+
else {
|
|
1129
|
+
log('INFO', 'Monitor engine disabled');
|
|
737
1130
|
}
|
|
738
1131
|
// Backlog recovery: any enabled recurring job whose most-recent expected fire
|
|
739
1132
|
// is older than its most-recent recorded run was missed — the laptop slept,
|
|
@@ -744,7 +1137,7 @@ export async function runDaemon() {
|
|
|
744
1137
|
// `catchup: false`, RUN late. Runs on a timer as well as at startup: a startup
|
|
745
1138
|
// pass alone misses a fire lost while the daemon stayed up but its event loop
|
|
746
1139
|
// was wedged, or one lost across an OS suspend that the process survived.
|
|
747
|
-
// Overlap guard, same shape as runHealCheck
|
|
1140
|
+
// Overlap guard, same shape as runHealCheck (above). A pass
|
|
748
1141
|
// awaits executeJobDetached per job and an off-box (host/cloud) dispatch can
|
|
749
1142
|
// block for a while, so a slow pass could still be working when the next tick
|
|
750
1143
|
// fires. Both passes would then see a job the first has not yet reached as
|
|
@@ -822,66 +1215,36 @@ export async function runDaemon() {
|
|
|
822
1215
|
catch (err) {
|
|
823
1216
|
log('ERROR', `Orphan reaper failed: ${err.message}`);
|
|
824
1217
|
}
|
|
825
|
-
const browserService = new BrowserService();
|
|
826
|
-
const browserIPC = new BrowserIPCServer(browserService);
|
|
827
|
-
|
|
828
|
-
await browserIPC.start();
|
|
829
|
-
log('INFO', 'Browser IPC server started');
|
|
830
|
-
recordSubsystemOk(SUBSYSTEM_BROWSER_IPC);
|
|
831
|
-
}
|
|
832
|
-
catch (err) {
|
|
833
|
-
const message = err.message;
|
|
834
|
-
log('ERROR', `Browser IPC failed to start: ${message}`);
|
|
835
|
-
recordSubsystemError(SUBSYSTEM_BROWSER_IPC, message);
|
|
836
|
-
}
|
|
837
|
-
writeHeartbeat();
|
|
838
|
-
const monitorInterval = setInterval(() => {
|
|
839
|
-
writeHeartbeat();
|
|
840
|
-
monitorRunningJobs();
|
|
841
|
-
const reaped = reapTerminalRoutineProcesses();
|
|
842
|
-
if (reaped.length > 0)
|
|
843
|
-
log('WARN', `Reaped ${reaped.length} terminal routine process group(s): ${reaped.join(', ')}`);
|
|
844
|
-
}, MONITOR_TICK_MS);
|
|
845
|
-
// Resource safety check: heal gaps between what DotAgents repos define and
|
|
846
|
-
// what's actually installed in each agent home — the slow rot that nothing
|
|
847
|
-
// else catches (a non-default version left stale, a Claude-invalid plugin
|
|
848
|
-
// manifest silently rejecting a whole plugin). Conservative 'safe' mode: it
|
|
849
|
-
// fills missing resources, repairs invalid manifests, and fast-forwards
|
|
850
|
-
// provably-unmodified stale plugins, but never overwrites hand-edited content
|
|
851
|
-
// or a plugin it can't prove is pristine — those it reports for `doctor --fix`.
|
|
852
|
-
// Runs ~every 6h plus once ~30s after startup (staggered so launch isn't busy).
|
|
853
|
-
let healing = false;
|
|
854
|
-
const runHealCheck = async () => {
|
|
855
|
-
if (healing)
|
|
856
|
-
return;
|
|
857
|
-
// The daemon's state directory is its liveness boundary. Once that tree is
|
|
858
|
-
// removed, background maintenance must not recreate it while the
|
|
859
|
-
// self-terminate guard is shutting the process down.
|
|
860
|
-
if (!fs.existsSync(getDaemonDirRoot()))
|
|
861
|
-
return;
|
|
862
|
-
healing = true;
|
|
1218
|
+
const browserService = isEnabled('browser-ipc') ? new BrowserService() : null;
|
|
1219
|
+
const browserIPC = browserService ? new BrowserIPCServer(browserService) : null;
|
|
1220
|
+
if (browserIPC) {
|
|
863
1221
|
try {
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
// Background heal is conservative (mode: 'safe'): fixes low-risk drift (shims,
|
|
868
|
-
// symlink adoption, PATH, missing resources) and only reports risky ones. The
|
|
869
|
-
// 30s kickoff means shims/PATH settle shortly after the daemon starts. No
|
|
870
|
-
// desktop toast here — background heal is silent by design; the log is the record.
|
|
871
|
-
const report = await runSelfHeal({ mode: 'safe' });
|
|
872
|
-
if (selfHealChangedAnything(report) || selfHealNeedsAttention(report)) {
|
|
873
|
-
log('INFO', `self-heal: ${summarizeSelfHeal(report)}`);
|
|
874
|
-
}
|
|
1222
|
+
await browserIPC.start();
|
|
1223
|
+
log('INFO', 'Browser IPC server started');
|
|
1224
|
+
recordSubsystemOk(SUBSYSTEM_BROWSER_IPC);
|
|
875
1225
|
}
|
|
876
1226
|
catch (err) {
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
healing = false;
|
|
1227
|
+
const message = err.message;
|
|
1228
|
+
log('ERROR', `Browser IPC failed to start: ${message}`);
|
|
1229
|
+
recordSubsystemError(SUBSYSTEM_BROWSER_IPC, message);
|
|
881
1230
|
}
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
|
|
1231
|
+
}
|
|
1232
|
+
else {
|
|
1233
|
+
log('INFO', 'Browser IPC service disabled');
|
|
1234
|
+
}
|
|
1235
|
+
runMonitorTick();
|
|
1236
|
+
const monitorInterval = setInterval(runMonitorTick, MONITOR_TICK_MS);
|
|
1237
|
+
// Resource safety check: see runHealCheck above. Runs ~every 6h plus once
|
|
1238
|
+
// ~30s after startup so shims/PATH settle shortly after the daemon starts.
|
|
1239
|
+
let healInterval;
|
|
1240
|
+
let healKickoff;
|
|
1241
|
+
if (isEnabled('self-heal')) {
|
|
1242
|
+
healInterval = setInterval(() => { void runHealCheck(); }, SELF_HEAL_TICK_MS);
|
|
1243
|
+
healKickoff = setTimeout(() => { void runHealCheck(); }, SELF_HEAL_KICKOFF_MS);
|
|
1244
|
+
}
|
|
1245
|
+
else {
|
|
1246
|
+
log('INFO', 'Self-heal service disabled');
|
|
1247
|
+
}
|
|
885
1248
|
// RUSH-1817: the startup host decision above is one-shot. If a standalone
|
|
886
1249
|
// broker answered agentPing() at daemon start, the daemon declined to host —
|
|
887
1250
|
// but should that standalone later die or crash-loop, nothing takes over and
|
|
@@ -890,93 +1253,121 @@ export async function runDaemon() {
|
|
|
890
1253
|
// Re-probe on a cadence: whenever the daemon is NOT itself hosting AND no
|
|
891
1254
|
// healthy broker answers a ping, take over hosting. startHostedBroker binds
|
|
892
1255
|
// the socket only when it is free, so a take-over never races a live broker.
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
if (!shouldTakeOverBroker(hostedBroker != null, reachable))
|
|
1256
|
+
// Inline (not module-level): closes over `hostedBroker`, a mutable runDaemon-
|
|
1257
|
+
// local var also closed over by handleShutdown. Lifting it would require
|
|
1258
|
+
// hostedBroker to be module-level state too — wider than this refactor's scope.
|
|
1259
|
+
let brokerSelfHealInterval;
|
|
1260
|
+
if (isEnabled('secrets-broker')) {
|
|
1261
|
+
let selfHealingBroker = false;
|
|
1262
|
+
const runBrokerSelfHeal = async () => {
|
|
1263
|
+
if (selfHealingBroker)
|
|
902
1264
|
return;
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
1265
|
+
selfHealingBroker = true;
|
|
1266
|
+
try {
|
|
1267
|
+
const { agentPing, startHostedBroker } = await import('./secrets/agent.js');
|
|
1268
|
+
const reachable = (await agentPing()).reachable;
|
|
1269
|
+
if (!shouldTakeOverBroker(hostedBroker != null, reachable))
|
|
1270
|
+
return;
|
|
1271
|
+
hostedBroker = await startHostedBroker();
|
|
1272
|
+
if (hostedBroker) {
|
|
1273
|
+
log('WARN', 'Secrets broker was unreachable; daemon took over hosting (self-heal)');
|
|
1274
|
+
}
|
|
906
1275
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
|
|
1276
|
+
catch (err) {
|
|
1277
|
+
log('WARN', `Secrets broker self-heal skipped: ${err.message}`);
|
|
1278
|
+
}
|
|
1279
|
+
finally {
|
|
1280
|
+
selfHealingBroker = false;
|
|
1281
|
+
}
|
|
1282
|
+
};
|
|
1283
|
+
brokerSelfHealInterval = setInterval(() => { void runBrokerSelfHeal(); }, BROKER_SELF_HEAL_TICK_MS);
|
|
1284
|
+
}
|
|
916
1285
|
// RUSH-2232: reap orphaned keychain helpers and `agents` processes stuck on a
|
|
917
1286
|
// keychain call. Runs as a 5-min interval in the daemon (the single executor)
|
|
918
|
-
// so no UI surface can race it.
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
for (const d of result.details)
|
|
931
|
-
log('WARN', ` ${d}`);
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
catch (err) {
|
|
935
|
-
log('ERROR', `Keychain reaper failed: ${err.message}`);
|
|
936
|
-
}
|
|
937
|
-
finally {
|
|
938
|
-
reapingKeychain = false;
|
|
939
|
-
}
|
|
940
|
-
};
|
|
941
|
-
const keychainReapInterval = setInterval(() => { void runKeychainReap(); }, 5 * 60_000);
|
|
1287
|
+
// so no UI surface can race it. See runKeychainReap above.
|
|
1288
|
+
let keychainReapInterval;
|
|
1289
|
+
if (isEnabled('keychain-reap')) {
|
|
1290
|
+
keychainReapInterval = setInterval(() => { void runKeychainReap(); }, KEYCHAIN_REAP_TICK_MS);
|
|
1291
|
+
}
|
|
1292
|
+
else {
|
|
1293
|
+
log('INFO', 'Keychain-reap service disabled');
|
|
1294
|
+
}
|
|
1295
|
+
// RUSH-2501: reap tmux sessions whose panes are all dead. Runs on the same
|
|
1296
|
+
// 5-min cadence as the keychain reaper. Daemon-only (single executor).
|
|
1297
|
+
void runDeadPaneReap(); // kick-off on startup so the backlog clears immediately
|
|
1298
|
+
const deadPaneReapInterval = setInterval(() => { void runDeadPaneReap(); }, DEAD_PANE_REAP_TICK_MS);
|
|
942
1299
|
// RUSH-2367: self-terminate if this daemon's own state dir has been removed
|
|
943
1300
|
// out from under it — the shape of a leaked test-fixture daemon whose /tmp
|
|
944
1301
|
// HOME was deleted by its test's own cleanup while the process itself
|
|
945
1302
|
// somehow survived (lost the SIGTERM/SIGKILL race, or outlived a killed
|
|
946
1303
|
// test runner before its `finally` ever ran). Nothing else can reach a
|
|
947
1304
|
// daemon in that state: no `agents daemon` command targets it, since a
|
|
948
|
-
// different HOME resolves a different
|
|
1305
|
+
// different HOME resolves a different ensureDaemonDir() and therefore a
|
|
949
1306
|
// different instance registry — without this it runs forever. Reads
|
|
950
|
-
//
|
|
1307
|
+
// the lifetime marker directly, never the local ensureDaemonDir() wrapper,
|
|
951
1308
|
// which recreates the directory as a side effect and would defeat the check.
|
|
952
1309
|
// Heartbeat/status paths may recreate the directory and pid file after a
|
|
953
1310
|
// deletion; they never recreate this per-lifetime token.
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
1311
|
+
// Inline (not module-level): closes over `lifetimePath` and `lifetimeToken`,
|
|
1312
|
+
// per-boot constants computed once at runDaemon() start that cannot be
|
|
1313
|
+
// pre-computed at module load time.
|
|
1314
|
+
let stateDirCheckInterval;
|
|
1315
|
+
if (isEnabled('state-dir-check')) {
|
|
1316
|
+
let checkingStateDir = false;
|
|
1317
|
+
const runStateDirSelfCheck = () => {
|
|
1318
|
+
if (checkingStateDir)
|
|
1319
|
+
return;
|
|
1320
|
+
checkingStateDir = true;
|
|
961
1321
|
try {
|
|
962
|
-
markerMatches =
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
1322
|
+
let markerMatches = false;
|
|
1323
|
+
try {
|
|
1324
|
+
markerMatches = fs.readFileSync(lifetimePath, 'utf-8') === lifetimeToken;
|
|
1325
|
+
}
|
|
1326
|
+
catch {
|
|
1327
|
+
// A missing state dir or marker is the condition this guard detects.
|
|
1328
|
+
}
|
|
1329
|
+
if (!markerMatches) {
|
|
1330
|
+
log('WARN', `Daemon state dir ${getDaemonDir()} no longer exists; exiting (self-terminate guard)`);
|
|
1331
|
+
void handleShutdown();
|
|
1332
|
+
}
|
|
966
1333
|
}
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
void handleShutdown();
|
|
1334
|
+
finally {
|
|
1335
|
+
checkingStateDir = false;
|
|
970
1336
|
}
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1337
|
+
};
|
|
1338
|
+
const stateDirCheckMs = Number(process.env.AGENTS_DAEMON_STATE_DIR_CHECK_MS) || STATE_DIR_CHECK_TICK_MS;
|
|
1339
|
+
stateDirCheckInterval = setInterval(runStateDirSelfCheck, stateDirCheckMs);
|
|
1340
|
+
}
|
|
1341
|
+
else {
|
|
1342
|
+
log('INFO', 'State-dir self-check disabled');
|
|
1343
|
+
}
|
|
1344
|
+
// RUSH-2418: startup is over — the scheduler, browser IPC, broker decision,
|
|
1345
|
+
// monitor engine and every background tick are up. Only NOW does this daemon
|
|
1346
|
+
// clear the auto-start failure streak `ensureDaemonStarted` reads. Clearing it
|
|
1347
|
+
// at claim time instead would reset the breaker for a process that dies while
|
|
1348
|
+
// initializing a subsystem, which is exactly the crash loop it exists to stop.
|
|
1349
|
+
recordSubsystemOk(SUBSYSTEM_DAEMON_START);
|
|
978
1350
|
const handleReload = () => {
|
|
979
1351
|
log('INFO', 'Reloading jobs (SIGHUP)');
|
|
1352
|
+
// Re-read per-service toggles so `agents daemon services enable|disable`
|
|
1353
|
+
// followed by `agents daemon reload` is truthful. Most services require a
|
|
1354
|
+
// restart to start/stop safely; log those instead of pretending they applied.
|
|
1355
|
+
const reloadedConfig = readDaemonServicesConfig();
|
|
1356
|
+
const reloadedEnabled = (id) => reloadedConfig.services[id] !== false;
|
|
1357
|
+
for (const id of Object.keys(servicesConfig.services)) {
|
|
1358
|
+
const was = servicesConfig.services[id] !== false;
|
|
1359
|
+
const now = reloadedEnabled(id);
|
|
1360
|
+
if (was !== now) {
|
|
1361
|
+
// scheduler and monitors are re-evaluated live later in this handler,
|
|
1362
|
+
// so don't tell the user they need a restart for those.
|
|
1363
|
+
if (id === 'scheduler' || (id === 'monitors' && !now)) {
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
log('INFO', `Service '${id}' toggled ${now ? 'on' : 'off'} — restart daemon to apply`);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
// Remember the reloaded state so subsequent reloads log transitions truthfully.
|
|
1370
|
+
servicesConfig = reloadedConfig;
|
|
980
1371
|
// Refresh user-layer copies of opted-in project routines BEFORE the
|
|
981
1372
|
// scheduler reloads, so YAML edits under `<project>/.agents/routines/`
|
|
982
1373
|
// take effect on the next fire without a manual `routines sync`.
|
|
@@ -994,7 +1385,9 @@ export async function runDaemon() {
|
|
|
994
1385
|
// this reload, no daemon restart needed. A `routines add` on a re-enabled
|
|
995
1386
|
// box signals exactly this reload, which boots the scheduler — the
|
|
996
1387
|
// "Scheduler reloaded" it prints is then truthful, not a dead-end.
|
|
997
|
-
|
|
1388
|
+
// Also honour the daemon-services toggle.
|
|
1389
|
+
const schedulerEnabledNow = isSchedulerEnabled() && reloadedEnabled('scheduler');
|
|
1390
|
+
const transition = schedulerGateTransition(scheduler !== null, schedulerEnabledNow);
|
|
998
1391
|
if (transition === 'boot') {
|
|
999
1392
|
log('INFO', 'scheduler.enabled is now on — booting the scheduler');
|
|
1000
1393
|
bootScheduler();
|
|
@@ -1008,37 +1401,69 @@ export async function runDaemon() {
|
|
|
1008
1401
|
const reloaded = scheduler.listScheduled();
|
|
1009
1402
|
log('INFO', `Reloaded ${reloaded.length} jobs`);
|
|
1010
1403
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1404
|
+
// Monitor engine can be stopped on reload if disabled; starting it requires
|
|
1405
|
+
// a restart, so we only handle the off transition here.
|
|
1406
|
+
if (monitorEngine) {
|
|
1407
|
+
if (!reloadedEnabled('monitors')) {
|
|
1408
|
+
log('WARN', 'monitors service is now disabled — stopping monitor engine; restart daemon to re-enable');
|
|
1409
|
+
try {
|
|
1410
|
+
monitorEngine.stop();
|
|
1411
|
+
}
|
|
1412
|
+
catch { /* best-effort */ }
|
|
1413
|
+
}
|
|
1414
|
+
else {
|
|
1415
|
+
try {
|
|
1416
|
+
monitorEngine.reload();
|
|
1417
|
+
}
|
|
1418
|
+
catch (err) {
|
|
1419
|
+
log('ERROR', `Monitor engine reload failed: ${err.message}`);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1016
1422
|
}
|
|
1017
1423
|
};
|
|
1018
|
-
|
|
1424
|
+
// Structurally single-shot (RUSH-2423). Shutdown is reachable from SIGTERM,
|
|
1425
|
+
// SIGINT, and the state-dir self-check's independent `void handleShutdown()`,
|
|
1426
|
+
// and two of those can arrive together — a service manager that SIGTERMs a
|
|
1427
|
+
// daemon whose state dir was just removed. It was only INCIDENTALLY safe
|
|
1428
|
+
// before (every step inside happens to be idempotent); the guard makes
|
|
1429
|
+
// single-shot a property of the function rather than one that every step
|
|
1430
|
+
// added later has to re-earn.
|
|
1431
|
+
const handleShutdown = singleShot(async () => {
|
|
1019
1432
|
log('INFO', 'Daemon shutting down');
|
|
1433
|
+
accountStateService?.stop();
|
|
1434
|
+
clearInterval(activeSessionsWarmInterval);
|
|
1435
|
+
if (watchdogInterval)
|
|
1436
|
+
clearInterval(watchdogInterval);
|
|
1437
|
+
if (deviceProbeInterval)
|
|
1438
|
+
clearInterval(deviceProbeInterval);
|
|
1020
1439
|
stopScheduler();
|
|
1021
|
-
monitorEngine
|
|
1022
|
-
await browserIPC
|
|
1440
|
+
monitorEngine?.stop();
|
|
1441
|
+
await browserIPC?.stop();
|
|
1023
1442
|
clearInterval(monitorInterval);
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1443
|
+
if (healInterval)
|
|
1444
|
+
clearInterval(healInterval);
|
|
1445
|
+
if (healKickoff)
|
|
1446
|
+
clearTimeout(healKickoff);
|
|
1447
|
+
if (brokerSelfHealInterval)
|
|
1448
|
+
clearInterval(brokerSelfHealInterval);
|
|
1449
|
+
if (keychainReapInterval)
|
|
1450
|
+
clearInterval(keychainReapInterval);
|
|
1451
|
+
clearInterval(deadPaneReapInterval);
|
|
1452
|
+
if (stateDirCheckInterval)
|
|
1453
|
+
clearInterval(stateDirCheckInterval);
|
|
1029
1454
|
try {
|
|
1030
1455
|
if (fs.readFileSync(lifetimePath, 'utf-8') === lifetimeToken)
|
|
1031
1456
|
fs.unlinkSync(lifetimePath);
|
|
1032
1457
|
}
|
|
1033
1458
|
catch {
|
|
1034
|
-
// Already removed with the state
|
|
1459
|
+
// Already removed with the state dir, or replaced by a newer owner.
|
|
1035
1460
|
}
|
|
1036
1461
|
hostedBroker?.close();
|
|
1037
1462
|
removeDaemonPid();
|
|
1038
1463
|
removeHeartbeat();
|
|
1039
1464
|
unregisterDaemonInstance();
|
|
1040
1465
|
process.exit(0);
|
|
1041
|
-
};
|
|
1466
|
+
});
|
|
1042
1467
|
process.on('SIGHUP', handleReload);
|
|
1043
1468
|
process.on('SIGTERM', () => handleShutdown());
|
|
1044
1469
|
process.on('SIGINT', () => handleShutdown());
|
|
@@ -1078,7 +1503,7 @@ export function writeOwnerOnlyServiceManifest(filePath, content) {
|
|
|
1078
1503
|
*/
|
|
1079
1504
|
export function generateLaunchdPlist(agentsBin = getAgentsBinPath()) {
|
|
1080
1505
|
const launch = getDaemonLaunch(agentsBin);
|
|
1081
|
-
const logPath =
|
|
1506
|
+
const logPath = getDaemonLogPath();
|
|
1082
1507
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
1083
1508
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
1084
1509
|
<plist version="1.0">
|
|
@@ -1093,6 +1518,8 @@ ${[launch.command, ...launch.args].map((arg) => ` <string>${xmlEscape(arg)}</
|
|
|
1093
1518
|
<true/>
|
|
1094
1519
|
<key>KeepAlive</key>
|
|
1095
1520
|
<true/>
|
|
1521
|
+
<key>ThrottleInterval</key>
|
|
1522
|
+
<integer>${DAEMON_THROTTLE_SECONDS}</integer>
|
|
1096
1523
|
<key>StandardOutPath</key>
|
|
1097
1524
|
<string>${logPath}</string>
|
|
1098
1525
|
<key>StandardErrorPath</key>
|
|
@@ -1123,12 +1550,14 @@ export function generateSystemdUnit(agentsBin = getAgentsBinPath()) {
|
|
|
1123
1550
|
return `[Unit]
|
|
1124
1551
|
Description=Agents Daemon - Scheduled Job Runner
|
|
1125
1552
|
After=network.target
|
|
1553
|
+
StartLimitIntervalSec=${DAEMON_START_LIMIT_INTERVAL_SECONDS}
|
|
1554
|
+
StartLimitBurst=${DAEMON_START_LIMIT_BURST}
|
|
1126
1555
|
|
|
1127
1556
|
[Service]
|
|
1128
1557
|
Type=simple
|
|
1129
1558
|
ExecStart=${execStart}
|
|
1130
1559
|
Restart=always
|
|
1131
|
-
RestartSec
|
|
1560
|
+
RestartSec=${DAEMON_THROTTLE_SECONDS}
|
|
1132
1561
|
Environment=PATH=${daemonPathValue(agentsBin, ['/usr/local/bin', '/usr/bin', '/bin'])}
|
|
1133
1562
|
|
|
1134
1563
|
[Install]
|
|
@@ -1187,13 +1616,44 @@ export function startDaemon(agentsBin) {
|
|
|
1187
1616
|
released = true;
|
|
1188
1617
|
releaseLock();
|
|
1189
1618
|
};
|
|
1619
|
+
// RUSH-2418: count starts PESSIMISTICALLY, and let a daemon that reaches
|
|
1620
|
+
// steady state clear the streak itself (`recordSubsystemOk` at the end of
|
|
1621
|
+
// runDaemon's startup). Recording a failure only on an observable error would
|
|
1622
|
+
// miss the crash loop entirely: a daemon that spawns and then dies returns a
|
|
1623
|
+
// perfectly real `child.pid`, so the launcher has no error to see. Every path
|
|
1624
|
+
// out of startDaemonLocked is either pid-truthy or a throw, so an
|
|
1625
|
+
// outcome-shaped check here can only ever catch an unspawnable binary — not
|
|
1626
|
+
// the failure this breaker exists for. Marking the attempt up front and
|
|
1627
|
+
// clearing on proven health inverts that: the streak grows exactly when
|
|
1628
|
+
// starts stop producing a daemon that lives.
|
|
1629
|
+
//
|
|
1630
|
+
// The no-launch returns above (`already-running`, `already-starting`) return
|
|
1631
|
+
// before this point on purpose — they attempted nothing, so they count nothing.
|
|
1632
|
+
recordSubsystemError(SUBSYSTEM_DAEMON_START, 'start issued; no daemon has reported healthy since');
|
|
1190
1633
|
try {
|
|
1191
1634
|
return startDaemonLocked(agentsBin ?? getAgentsBinPath(), releaseOnce);
|
|
1192
1635
|
}
|
|
1636
|
+
catch (err) {
|
|
1637
|
+
// Replace the provisional reason with the real one — the streak is already
|
|
1638
|
+
// counted, this just makes `agents daemon doctor` name the actual cause.
|
|
1639
|
+
recordSubsystemErrorReason(SUBSYSTEM_DAEMON_START, `start failed: ${err?.message ?? String(err)}`);
|
|
1640
|
+
throw err;
|
|
1641
|
+
}
|
|
1193
1642
|
finally {
|
|
1194
1643
|
releaseOnce();
|
|
1195
1644
|
}
|
|
1196
1645
|
}
|
|
1646
|
+
/**
|
|
1647
|
+
* Is the auto-start circuit breaker open (RUSH-2418)? True once
|
|
1648
|
+
* {@link DAEMON_AUTOSTART_FAILURE_LIMIT} consecutive starts have failed to
|
|
1649
|
+
* produce a daemon that reported healthy. Pure read of the persisted health
|
|
1650
|
+
* record, and `agents daemon doctor` reports the same record — the message the
|
|
1651
|
+
* breaker prints has to lead somewhere that can explain it.
|
|
1652
|
+
*/
|
|
1653
|
+
export function isDaemonAutostartCircuitOpen() {
|
|
1654
|
+
const health = readSubsystemHealth(SUBSYSTEM_DAEMON_START);
|
|
1655
|
+
return (health?.consecutiveFailures ?? 0) >= DAEMON_AUTOSTART_FAILURE_LIMIT;
|
|
1656
|
+
}
|
|
1197
1657
|
/**
|
|
1198
1658
|
* Bring the always-on daemon up as a side effect of a background-adjacent
|
|
1199
1659
|
* command (secrets unlock, browser start, ...), not only from `routines add`.
|
|
@@ -1211,6 +1671,25 @@ export function ensureDaemonStarted() {
|
|
|
1211
1671
|
// and calls startDaemon() directly instead of going through this helper.
|
|
1212
1672
|
if (!isDaemonEnabled())
|
|
1213
1673
|
return null;
|
|
1674
|
+
// A live daemon is the answer whatever the failure history says — the breaker
|
|
1675
|
+
// gates LAUNCHING one, never reporting one that is already up. Checked first
|
|
1676
|
+
// so a stale failure streak can't make a healthy daemon read as absent to
|
|
1677
|
+
// callers that branch on this return (e.g. secrets/agent.ts).
|
|
1678
|
+
if (isDaemonRunning())
|
|
1679
|
+
return startDaemon();
|
|
1680
|
+
// RUSH-2418: the auto-start circuit breaker. A daemon that dies during
|
|
1681
|
+
// startup would otherwise be relaunched by EVERY foreground command that
|
|
1682
|
+
// wants one (secrets unlock, browser start, watchdog, ...) — an
|
|
1683
|
+
// application-level crash loop the OS supervisor's throttle cannot see,
|
|
1684
|
+
// because each attempt is a fresh service start rather than a respawn. After
|
|
1685
|
+
// DAEMON_AUTOSTART_FAILURE_LIMIT consecutive failures, refuse and say why.
|
|
1686
|
+
// Deliberately NOT applied in startDaemon(): `agents daemon start` is the
|
|
1687
|
+
// operator's override and must always be able to retry.
|
|
1688
|
+
if (isDaemonAutostartCircuitOpen()) {
|
|
1689
|
+
process.stderr.write(`[agents] daemon auto-start disabled after ${DAEMON_AUTOSTART_FAILURE_LIMIT} consecutive failed starts. ` +
|
|
1690
|
+
`Run 'agents daemon doctor' to diagnose, or 'agents daemon start' to retry anyway.\n`);
|
|
1691
|
+
return null;
|
|
1692
|
+
}
|
|
1214
1693
|
try {
|
|
1215
1694
|
return startDaemon();
|
|
1216
1695
|
}
|
|
@@ -1341,19 +1820,25 @@ function daemonNodeBinDir() {
|
|
|
1341
1820
|
return path.dirname(process.execPath);
|
|
1342
1821
|
}
|
|
1343
1822
|
/**
|
|
1344
|
-
* The full PATH value the daemon service manifest pins, in order: the
|
|
1345
|
-
*
|
|
1346
|
-
*
|
|
1347
|
-
*
|
|
1348
|
-
*
|
|
1349
|
-
* (`/bin/sh -c 'agents
|
|
1350
|
-
*
|
|
1351
|
-
*
|
|
1352
|
-
*
|
|
1353
|
-
*
|
|
1823
|
+
* The full PATH value the daemon service manifest pins, in order: the directory
|
|
1824
|
+
* of the `agents` shim itself FIRST, then the Node runtime dir, then the
|
|
1825
|
+
* platform's system dirs.
|
|
1826
|
+
*
|
|
1827
|
+
* The shim's own dir must lead so a scheduled `command` routine that shells out
|
|
1828
|
+
* to the bare name `agents` (`/bin/sh -c 'agents repo pull system'`) resolves the
|
|
1829
|
+
* SAME binary the daemon is running. When the Node runtime dir came first, a
|
|
1830
|
+
* stale `agents` install inside that dir (common with nvm or an npm global in the
|
|
1831
|
+
* same Node prefix) shadowed the current binary and routines failed with an
|
|
1832
|
+
* `unknown command` error against the wrong build.
|
|
1833
|
+
*
|
|
1834
|
+
* The Node runtime dir stays second so the shim's shebang (`#!/usr/bin/env node`)
|
|
1835
|
+
* still resolves the exact Node that installed the service — never an ancient
|
|
1836
|
+
* system node or a pruned nvm version. Deduped across the whole list, so a
|
|
1837
|
+
* Node/shim dir that already appears among the system dirs (e.g. a
|
|
1838
|
+
* `/usr/local/bin` install) never doubles.
|
|
1354
1839
|
*/
|
|
1355
1840
|
function daemonPathValue(agentsBin, systemDirs) {
|
|
1356
|
-
return [...new Set([
|
|
1841
|
+
return [...new Set([path.dirname(agentsBin), daemonNodeBinDir(), ...systemDirs])].join(':');
|
|
1357
1842
|
}
|
|
1358
1843
|
/**
|
|
1359
1844
|
* Build the argv to relaunch the `agents` CLI with the given subcommand args.
|
|
@@ -1409,7 +1894,7 @@ export function validateDaemonBinary(binPath) {
|
|
|
1409
1894
|
}
|
|
1410
1895
|
export function startDetached(opts = {}) {
|
|
1411
1896
|
const agentsBin = opts.agentsBin ?? getAgentsBinPath();
|
|
1412
|
-
const logPath = opts.logPath ??
|
|
1897
|
+
const logPath = opts.logPath ?? getDaemonLogPath();
|
|
1413
1898
|
const logFd = fs.openSync(logPath, 'a');
|
|
1414
1899
|
const { command, args } = getDaemonLaunch(agentsBin);
|
|
1415
1900
|
// fdStdio: the log-file fds make windowsHide inert (libuv skips
|
|
@@ -1447,10 +1932,90 @@ function waitForPid(timeoutMs) {
|
|
|
1447
1932
|
}
|
|
1448
1933
|
return readDaemonPid();
|
|
1449
1934
|
}
|
|
1935
|
+
/**
|
|
1936
|
+
* Read the pid a state file claims, or null when it is absent/unreadable/not
|
|
1937
|
+
* pid-shaped. The lifetime marker stores `<pid>:<epochMs>`; the heartbeat
|
|
1938
|
+
* stores JSON with a `pid`.
|
|
1939
|
+
*/
|
|
1940
|
+
function claimedPid(read) {
|
|
1941
|
+
try {
|
|
1942
|
+
return read();
|
|
1943
|
+
}
|
|
1944
|
+
catch {
|
|
1945
|
+
return null;
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
/**
|
|
1949
|
+
* The lifetime marker, heartbeat, and instance-registry entry, described so
|
|
1950
|
+
* {@link stopDaemon} can assert them the same way it asserts the two sockets.
|
|
1951
|
+
*
|
|
1952
|
+
* Ownership, not mere presence, decides: a file naming a pid that is alive and
|
|
1953
|
+
* is not the daemon we just stopped belongs to a DIFFERENT daemon (a successor
|
|
1954
|
+
* that started during the stop, or a peer serving this state dir), and deleting
|
|
1955
|
+
* it would break that live daemon — the same reasoning the broker-socket branch
|
|
1956
|
+
* above uses for a standalone owner. Everything else is residue from a provably
|
|
1957
|
+
* dead owner and is reclaimed.
|
|
1958
|
+
*/
|
|
1959
|
+
export function stopResidueArtifacts(stoppedPid, survivors = []) {
|
|
1960
|
+
const artifacts = [];
|
|
1961
|
+
const lifetimePath = path.join(getDaemonDir(), LIFETIME_FILE);
|
|
1962
|
+
const lifetimeOwner = claimedPid(() => {
|
|
1963
|
+
const raw = fs.readFileSync(lifetimePath, 'utf-8').split(':')[0];
|
|
1964
|
+
const n = parseInt(raw, 10);
|
|
1965
|
+
return isNaN(n) ? null : n;
|
|
1966
|
+
});
|
|
1967
|
+
artifacts.push({
|
|
1968
|
+
label: 'daemon lifetime marker',
|
|
1969
|
+
present: fs.existsSync(lifetimePath),
|
|
1970
|
+
ownedByLiveOther: lifetimeOwner !== null && lifetimeOwner !== stoppedPid && isAlive(lifetimeOwner),
|
|
1971
|
+
reclaim: () => { try {
|
|
1972
|
+
fs.unlinkSync(lifetimePath);
|
|
1973
|
+
}
|
|
1974
|
+
catch { /* raced with a fresh start */ } },
|
|
1975
|
+
stillPresent: () => fs.existsSync(lifetimePath),
|
|
1976
|
+
});
|
|
1977
|
+
const heartbeatPath = getHeartbeatPath();
|
|
1978
|
+
const hb = readHeartbeat();
|
|
1979
|
+
artifacts.push({
|
|
1980
|
+
label: 'daemon heartbeat',
|
|
1981
|
+
present: fs.existsSync(heartbeatPath),
|
|
1982
|
+
// A stale heartbeat is not cosmetic: resolveLiveDaemonPid() trusts a FRESH
|
|
1983
|
+
// one to re-adopt a daemon whose pid file was lost, so leaving one behind
|
|
1984
|
+
// can make a dead daemon read as running.
|
|
1985
|
+
ownedByLiveOther: hb !== null && hb.pid !== stoppedPid && isAlive(hb.pid),
|
|
1986
|
+
reclaim: () => removeHeartbeat(),
|
|
1987
|
+
stillPresent: () => fs.existsSync(heartbeatPath),
|
|
1988
|
+
});
|
|
1989
|
+
// POSIX-only, matching registerDaemonInstance/unregisterDaemonInstance.
|
|
1990
|
+
if (process.platform !== 'win32' && stoppedPid !== null) {
|
|
1991
|
+
const markerPath = path.join(getDaemonInstancesDir(), String(stoppedPid));
|
|
1992
|
+
artifacts.push({
|
|
1993
|
+
label: 'daemon instance registry entry',
|
|
1994
|
+
present: fs.existsSync(markerPath),
|
|
1995
|
+
// The marker is named by pid, so it is unambiguously this daemon's — but
|
|
1996
|
+
// "this daemon" is only residue once it is actually DEAD. If the kill did
|
|
1997
|
+
// not land, deleting the marker erases the very record
|
|
1998
|
+
// `findSurvivingStateDirDaemons` enumerates, so the next `agents daemon
|
|
1999
|
+
// stop` would find an empty registry and a cleared pid file and report
|
|
2000
|
+
// `ok: true` with the daemon still running.
|
|
2001
|
+
//
|
|
2002
|
+
// "Dead" is decided by the caller's OWN survivor scan, not by `isAlive`:
|
|
2003
|
+
// a SIGKILLed child is a zombie until its parent reaps it, and `kill(pid,
|
|
2004
|
+
// 0)` succeeds on a zombie. Keyed off `isAlive` this kept the entry of a
|
|
2005
|
+
// daemon that was already gone, so the stop stopped being able to report
|
|
2006
|
+
// its own state truthfully. The survivor scan matches a live
|
|
2007
|
+
// `__daemon-run`, which a zombie is not.
|
|
2008
|
+
ownedByLiveOther: survivors.includes(stoppedPid),
|
|
2009
|
+
reclaim: () => unregisterDaemonInstance(stoppedPid),
|
|
2010
|
+
stillPresent: () => fs.existsSync(markerPath),
|
|
2011
|
+
});
|
|
2012
|
+
}
|
|
2013
|
+
return artifacts;
|
|
2014
|
+
}
|
|
1450
2015
|
/**
|
|
1451
2016
|
* Live `__daemon-run` processes still registered in THIS state dir's instance
|
|
1452
2017
|
* registry, excluding `exclude`. State-dir-scoped by construction: the registry
|
|
1453
|
-
* lives inside this
|
|
2018
|
+
* lives inside this state dir, so a daemon serving a DIFFERENT state dir (a test
|
|
1454
2019
|
* fixture with its own HOME, a separate install/home) registers elsewhere and is
|
|
1455
2020
|
* invisible here — it is never a stop/takeover target. POSIX-only (the registry
|
|
1456
2021
|
* and its `ps` liveness probe are); `[]` on Windows.
|
|
@@ -1624,6 +2189,31 @@ export function stopDaemon() {
|
|
|
1624
2189
|
released.push('secrets broker socket (reclaimed)');
|
|
1625
2190
|
}
|
|
1626
2191
|
}
|
|
2192
|
+
// ── The state files a killed daemon cannot clean up itself (RUSH-2421) ─────
|
|
2193
|
+
// handleShutdown removes the lifetime marker, the heartbeat and this pid's
|
|
2194
|
+
// instance-registry entry — but it only runs on the GRACEFUL path. Every
|
|
2195
|
+
// escalation above (killTree, and the whole win32 branch) skips it, so those
|
|
2196
|
+
// three outlive the daemon and the stop reported `ok: true` while its state
|
|
2197
|
+
// dir still described a daemon that no longer exists. Each is stale metadata
|
|
2198
|
+
// with real consequences: a leftover heartbeat is what `resolveLiveDaemonPid`
|
|
2199
|
+
// consults to re-adopt a "live" daemon, and a leftover registry entry is what
|
|
2200
|
+
// `reapStrayDaemons` enumerates. Same shape as the sockets above — reclaim
|
|
2201
|
+
// what a provably dead owner left, never touch what a live one owns.
|
|
2202
|
+
for (const artifact of stopResidueArtifacts(pid, survivors)) {
|
|
2203
|
+
if (!artifact.present) {
|
|
2204
|
+
released.push(artifact.label);
|
|
2205
|
+
continue;
|
|
2206
|
+
}
|
|
2207
|
+
if (artifact.ownedByLiveOther) {
|
|
2208
|
+
released.push(`${artifact.label} (owned by a live daemon)`);
|
|
2209
|
+
continue;
|
|
2210
|
+
}
|
|
2211
|
+
artifact.reclaim();
|
|
2212
|
+
if (artifact.stillPresent())
|
|
2213
|
+
surviving.push(`${artifact.label} not released`);
|
|
2214
|
+
else
|
|
2215
|
+
released.push(`${artifact.label} (reclaimed)`);
|
|
2216
|
+
}
|
|
1627
2217
|
// In-flight detached routine children survive on purpose (SING-11a) — report,
|
|
1628
2218
|
// never kill: severing a live agent mid-run is worse than a daemon restart.
|
|
1629
2219
|
const detachedChildren = listLiveRoutineChildren();
|
|
@@ -1656,14 +2246,14 @@ export function getDaemonStatus() {
|
|
|
1656
2246
|
running,
|
|
1657
2247
|
pid,
|
|
1658
2248
|
jobCount,
|
|
1659
|
-
logPath:
|
|
2249
|
+
logPath: getDaemonLogPath(),
|
|
1660
2250
|
binaryPath,
|
|
1661
2251
|
heartbeat: readHeartbeat(),
|
|
1662
2252
|
};
|
|
1663
2253
|
}
|
|
1664
2254
|
/** Read the daemon log, optionally limited to the last N lines. */
|
|
1665
2255
|
export function readDaemonLog(lines) {
|
|
1666
|
-
const logPath =
|
|
2256
|
+
const logPath = getDaemonLogPath();
|
|
1667
2257
|
if (!fs.existsSync(logPath))
|
|
1668
2258
|
return '(no log file)';
|
|
1669
2259
|
const content = fs.readFileSync(logPath, 'utf-8');
|