@phnx-labs/agents-cli 1.22.66 → 1.22.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/CHANGELOG.md +56 -0
  2. package/README.md +21 -8
  3. package/dist/bootstrap.js +6 -3
  4. package/dist/commands/browser.js +46 -21
  5. package/dist/commands/daemon-test-harness.d.ts +1 -1
  6. package/dist/commands/daemon-test-harness.js +2 -2
  7. package/dist/commands/daemon.js +21 -22
  8. package/dist/commands/exec.js +50 -0
  9. package/dist/commands/feed.js +20 -7
  10. package/dist/commands/monitors.js +5 -2
  11. package/dist/commands/projects.d.ts +26 -6
  12. package/dist/commands/projects.js +55 -22
  13. package/dist/commands/repo.js +57 -19
  14. package/dist/commands/resume.d.ts +16 -0
  15. package/dist/commands/resume.js +41 -8
  16. package/dist/commands/routines.js +42 -21
  17. package/dist/commands/send.js +29 -2
  18. package/dist/commands/sessions-inject.d.ts +58 -0
  19. package/dist/commands/sessions-inject.js +143 -7
  20. package/dist/commands/sessions-optimize.js +1 -1
  21. package/dist/commands/sessions-picker.js +1 -0
  22. package/dist/commands/sessions.js +4 -11
  23. package/dist/commands/share.d.ts +5 -3
  24. package/dist/commands/share.js +73 -20
  25. package/dist/commands/ssh.js +205 -2
  26. package/dist/lib/accounting/account-pool-collect.d.ts +6 -4
  27. package/dist/lib/accounting/account-pool-collect.js +6 -4
  28. package/dist/lib/accounting/usage-ingest.js +4 -2
  29. package/dist/lib/accounting/usage-sync.d.ts +38 -93
  30. package/dist/lib/accounting/usage-sync.js +66 -210
  31. package/dist/lib/accounting/usage.d.ts +18 -5
  32. package/dist/lib/accounting/usage.js +165 -20
  33. package/dist/lib/auth-health.d.ts +8 -0
  34. package/dist/lib/auth-health.js +4 -4
  35. package/dist/lib/boot-profile.d.ts +14 -0
  36. package/dist/lib/boot-profile.js +66 -0
  37. package/dist/lib/browser/caller-identity.d.ts +12 -0
  38. package/dist/lib/browser/caller-identity.js +19 -0
  39. package/dist/lib/browser/ipc.d.ts +37 -32
  40. package/dist/lib/browser/ipc.js +146 -94
  41. package/dist/lib/browser/task-index.d.ts +10 -2
  42. package/dist/lib/browser/task-index.js +22 -3
  43. package/dist/lib/channels/providers/desktop.d.ts +5 -4
  44. package/dist/lib/channels/providers/desktop.js +5 -4
  45. package/dist/lib/claude-account-token.js +108 -4
  46. package/dist/lib/daemon/account-state-daemon-service.d.ts +49 -9
  47. package/dist/lib/daemon/account-state-daemon-service.js +81 -18
  48. package/dist/lib/daemon/auth-sync-service.d.ts +4 -4
  49. package/dist/lib/daemon/auth-sync-service.js +17 -6
  50. package/dist/lib/daemon/catchup-service.d.ts +51 -0
  51. package/dist/lib/daemon/catchup-service.js +51 -0
  52. package/dist/lib/daemon/daemon.d.ts +12 -22
  53. package/dist/lib/daemon/daemon.js +463 -176
  54. package/dist/lib/daemon/runner.js +2 -0
  55. package/dist/lib/daemon/service.d.ts +22 -4
  56. package/dist/lib/daemon/service.js +2 -2
  57. package/dist/lib/daemon/supervisor.d.ts +55 -15
  58. package/dist/lib/daemon/supervisor.js +119 -29
  59. package/dist/lib/daemon/usage-sync-service.d.ts +4 -6
  60. package/dist/lib/daemon/usage-sync-service.js +22 -18
  61. package/dist/lib/daemon-health.js +36 -31
  62. package/dist/lib/daemon-services.d.ts +1 -1
  63. package/dist/lib/daemon-services.js +12 -2
  64. package/dist/lib/daemon-ticks.d.ts +9 -6
  65. package/dist/lib/daemon-ticks.js +14 -8
  66. package/dist/lib/devices/health.d.ts +38 -2
  67. package/dist/lib/devices/health.js +43 -5
  68. package/dist/lib/devices/registry.js +2 -0
  69. package/dist/lib/devices/worker-pick.d.ts +1 -1
  70. package/dist/lib/devices/worker-pick.js +4 -1
  71. package/dist/lib/exec.js +15 -0
  72. package/dist/lib/feed/watch.d.ts +3 -0
  73. package/dist/lib/feed/watch.js +13 -3
  74. package/dist/lib/feed-broadcast.d.ts +64 -5
  75. package/dist/lib/feed-broadcast.js +124 -22
  76. package/dist/lib/fleet-shared-repo-sync.d.ts +36 -0
  77. package/dist/lib/fleet-shared-repo-sync.js +333 -0
  78. package/dist/lib/fleet-shared-state.d.ts +38 -0
  79. package/dist/lib/fleet-shared-state.js +105 -0
  80. package/dist/lib/hosts/remote-cmd.d.ts +2 -0
  81. package/dist/lib/hosts/remote-cmd.js +12 -3
  82. package/dist/lib/lock-compromise.d.ts +8 -0
  83. package/dist/lib/lock-compromise.js +12 -0
  84. package/dist/lib/monitors/engine.d.ts +2 -1
  85. package/dist/lib/monitors/engine.js +27 -2
  86. package/dist/lib/monitors/sources/command.js +13 -3
  87. package/dist/lib/monitors/sources/failure.d.ts +32 -0
  88. package/dist/lib/monitors/sources/failure.js +52 -0
  89. package/dist/lib/monitors/sources/types.d.ts +9 -0
  90. package/dist/lib/owner-message.d.ts +12 -0
  91. package/dist/lib/owner-message.js +44 -0
  92. package/dist/lib/refresh-coordinator.js +2 -0
  93. package/dist/lib/run-trace-sync.d.ts +28 -0
  94. package/dist/lib/run-trace-sync.js +99 -0
  95. package/dist/lib/secrets/filestore.d.ts +4 -0
  96. package/dist/lib/secrets/filestore.js +164 -3
  97. package/dist/lib/secrets/push.d.ts +10 -0
  98. package/dist/lib/secrets/push.js +86 -7
  99. package/dist/lib/secrets/remote.d.ts +18 -6
  100. package/dist/lib/secrets/remote.js +29 -4
  101. package/dist/lib/secrets/reserved-sync.d.ts +28 -27
  102. package/dist/lib/secrets/reserved-sync.js +119 -101
  103. package/dist/lib/session/active.d.ts +13 -1
  104. package/dist/lib/session/active.js +5 -0
  105. package/dist/lib/session/actor-sidecar.d.ts +12 -0
  106. package/dist/lib/session/actor-sidecar.js +2 -0
  107. package/dist/lib/session/db.d.ts +16 -2
  108. package/dist/lib/session/db.js +73 -22
  109. package/dist/lib/session/discover.d.ts +12 -3
  110. package/dist/lib/session/discover.js +187 -26
  111. package/dist/lib/session/linear.d.ts +13 -0
  112. package/dist/lib/session/linear.js +44 -0
  113. package/dist/lib/session/live-metadata.js +1 -0
  114. package/dist/lib/session/parse.js +2 -3
  115. package/dist/lib/session/prompt.d.ts +17 -0
  116. package/dist/lib/session/prompt.js +35 -0
  117. package/dist/lib/session/recovery.d.ts +43 -6
  118. package/dist/lib/session/recovery.js +80 -10
  119. package/dist/lib/session/remote/remote-list.d.ts +17 -1
  120. package/dist/lib/session/remote/remote-list.js +29 -4
  121. package/dist/lib/session/remote/watch.d.ts +25 -2
  122. package/dist/lib/session/remote/watch.js +188 -11
  123. package/dist/lib/session/session-cache.d.ts +2 -1
  124. package/dist/lib/session/session-cache.js +1 -0
  125. package/dist/lib/session/state.js +11 -13
  126. package/dist/lib/session/types.d.ts +2 -0
  127. package/dist/lib/share/backend.d.ts +2 -2
  128. package/dist/lib/share/backend.js +20 -9
  129. package/dist/lib/share/delete.d.ts +5 -1
  130. package/dist/lib/share/delete.js +7 -2
  131. package/dist/lib/share/http-error.d.ts +52 -0
  132. package/dist/lib/share/http-error.js +65 -0
  133. package/dist/lib/share/publish.d.ts +67 -11
  134. package/dist/lib/share/publish.js +98 -16
  135. package/dist/lib/share/worker-template.js +105 -9
  136. package/dist/lib/smart-launch.js +27 -4
  137. package/dist/lib/ssh-exec.d.ts +2 -0
  138. package/dist/lib/ssh-exec.js +20 -4
  139. package/dist/lib/storage/index.d.ts +14 -0
  140. package/dist/lib/storage/index.js +14 -0
  141. package/dist/lib/storage/selection.d.ts +48 -0
  142. package/dist/lib/storage/selection.js +39 -0
  143. package/dist/lib/storage/visibility.d.ts +82 -0
  144. package/dist/lib/storage/visibility.js +99 -0
  145. package/dist/lib/teams/agents.js +3 -1
  146. package/dist/lib/teams/placement-probe.js +1 -0
  147. package/dist/lib/teams/registry.js +2 -0
  148. package/dist/lib/teams/scheduler.d.ts +8 -1
  149. package/dist/lib/teams/scheduler.js +4 -1
  150. package/dist/lib/testdata/daemon-health-writer.d.ts +1 -0
  151. package/dist/lib/testdata/daemon-health-writer.js +8 -0
  152. package/dist/lib/traces/backend.js +13 -2
  153. package/dist/lib/traces/sync.d.ts +7 -0
  154. package/dist/lib/traces/sync.js +9 -0
  155. package/dist/lib/usage-refresh.d.ts +8 -2
  156. package/dist/lib/usage-refresh.js +3 -3
  157. package/dist/lib/worktree/held.d.ts +166 -0
  158. package/dist/lib/worktree/held.js +368 -0
  159. package/package.json +2 -2
  160. package/dist/lib/account-state-service.d.ts +0 -21
  161. package/dist/lib/account-state-service.js +0 -60
@@ -33,6 +33,7 @@ import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
33
33
  import { backgroundSpawnOptions, killTree } from '../platform/process.js';
34
34
  import lockfile from 'proper-lockfile';
35
35
  import { ensureLockTarget } from '../fs-atomic.js';
36
+ import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
36
37
  import { walkForFiles } from '../fs-walk.js';
37
38
  import { getBinaryPath, isVersionInstalled, resolveVersion, getVersionHomePath } from '../installations/versions.js';
38
39
  import { resolveClaudeSetupToken } from '../claude-account-token.js';
@@ -153,6 +154,7 @@ async function withRoutineLock(config, launch) {
153
154
  minTimeout: 100,
154
155
  maxTimeout: 100,
155
156
  },
157
+ onCompromised: logAndContinueOnLockCompromised('routines launch'),
156
158
  });
157
159
  try {
158
160
  return await launch();
@@ -41,7 +41,14 @@ export interface DaemonService {
41
41
  /** A service the supervisor ticks on a fixed interval, under a hard per-tick deadline. */
42
42
  export interface PeriodicService extends DaemonService {
43
43
  readonly intervalMs: number;
44
- /** Hard cap per tick. An over-budget tick parks the service; its in-flight guard remains held until the real promise settles. */
44
+ /**
45
+ * Hard cap per tick. An over-budget tick is ABANDONED: the supervisor aborts
46
+ * the tick's {@link AbortSignal}, parks the service, and schedules a backoff
47
+ * restart immediately — it does NOT wait for the real promise to settle
48
+ * (PHNX-3608). A tick that awaits `signal` (or forwards it to its I/O) can
49
+ * observe the deadline and unwind; one that ignores it is left to drain in the
50
+ * background while the service is already being restarted.
51
+ */
45
52
  readonly deadlineMs: number;
46
53
  /**
47
54
  * Delay before the FIRST tick after `start()`, in ms. Every later tick
@@ -52,7 +59,13 @@ export interface PeriodicService extends DaemonService {
52
59
  * firing at daemon startup.
53
60
  */
54
61
  readonly startupDelayMs?: number;
55
- tick(ctx: DaemonContext): Promise<void>;
62
+ /**
63
+ * Run one tick. `signal` aborts when the supervisor's {@link deadlineMs}
64
+ * elapses (or the service is being stopped), so a well-behaved tick threads it
65
+ * into its awaits — `fetch(url, { signal })`, an ssh/exec spawn, a
66
+ * cancellable sleep — to bound its own I/O instead of leaking a runaway await.
67
+ */
68
+ tick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
56
69
  }
57
70
  export declare function isPeriodicService(service: DaemonService): service is PeriodicService;
58
71
  /**
@@ -93,10 +106,15 @@ export declare abstract class BasePeriodicService implements PeriodicService {
93
106
  private healthRecord;
94
107
  protected abstract onStart(ctx: DaemonContext): Promise<void>;
95
108
  protected abstract onStop(): Promise<void>;
96
- protected abstract onTick(ctx: DaemonContext): Promise<void>;
109
+ /**
110
+ * Run one tick. `signal` aborts at the supervisor's deadline (or on stop);
111
+ * thread it into the tick's own awaits to bound its I/O. A subclass that does
112
+ * no cancellable work may implement `onTick(ctx)` and ignore the second arg.
113
+ */
114
+ protected abstract onTick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
97
115
  start(ctx: DaemonContext): Promise<void>;
98
116
  stop(): Promise<void>;
99
117
  restart(): Promise<void>;
100
- tick(ctx: DaemonContext): Promise<void>;
118
+ tick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
101
119
  health(): ServiceHealth;
102
120
  }
@@ -81,9 +81,9 @@ export class BasePeriodicService {
81
81
  await this.stop();
82
82
  await this.start(ctx);
83
83
  }
84
- async tick(ctx) {
84
+ async tick(ctx, signal) {
85
85
  try {
86
- await this.onTick(ctx);
86
+ await this.onTick(ctx, signal);
87
87
  this.healthRecord = { ...this.healthRecord, lastRunMs: Date.now(), consecutiveFailures: 0, lastError: undefined };
88
88
  }
89
89
  catch (err) {
@@ -12,14 +12,23 @@
12
12
  * - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
13
13
  * local in-flight guard `true` forever, silently freezing that one service
14
14
  * for the daemon's life (observed ~51h). Here, every tick races a
15
- * `deadlineMs` timeout; when the deadline wins, that service is parked and
16
- * its in-flight guard stays held until the real promise settles. This keeps
17
- * a timed-out tick from overlapping another tick or lifecycle transition.
15
+ * `deadlineMs` timeout; when the deadline wins, the service is ABANDONED
16
+ * (PHNX-3608): its `AbortSignal` is aborted so a cooperating tick can unwind
17
+ * its own I/O, the in-flight guard is released immediately, and the backoff
18
+ * restart is scheduled right away — it is NOT gated on the runaway promise
19
+ * settling. An earlier revision kept the guard held until the real promise
20
+ * settled, which meant a tick that never settles parked the service forever
21
+ * and blocked backoff restart, `daemon services restart`, and SIGHUP reload —
22
+ * the exact 12h-usage-dark class this exists to prevent. The orphaned promise
23
+ * is drained separately in the background; `finishTick` is version-guarded so
24
+ * its late settlement can never disturb the fresh tick the restart started.
18
25
  *
19
26
  * Repeated failures (thrown or timed-out) open a circuit breaker: the service
20
27
  * is parked (its timer stopped) and retried on exponential backoff via its own
21
28
  * `restart()`, while every sibling service keeps ticking on its own timer,
22
- * unaffected.
29
+ * unaffected. `start()`/`stop()`/`restart()` are themselves bounded by
30
+ * {@link ServiceSupervisorOptions.lifecycleDeadlineMs} so a wedged bind or close
31
+ * cannot stall daemon startup or shutdown.
23
32
  */
24
33
  import type { DaemonServiceId } from '../daemon-services.js';
25
34
  import type { DaemonContext, DaemonService, ServiceHealth } from './service.js';
@@ -30,6 +39,18 @@ export interface ServiceSupervisorOptions {
30
39
  backoffBaseMs?: number;
31
40
  /** Backoff ceiling. Default 5 minutes. */
32
41
  backoffMaxMs?: number;
42
+ /**
43
+ * Hard cap on a service's `start()`/`stop()`/`restart()` call (PHNX-3608). A
44
+ * wedged bind/close would otherwise stall `startAll()` (which awaits each
45
+ * `startOne` in turn) or `stopAll()` at shutdown. A start/restart that
46
+ * breaches it is treated as a failure (parked + backoff); a stop that breaches
47
+ * it is logged and the service is left marked stopped. Default 30s.
48
+ */
49
+ lifecycleDeadlineMs?: number;
50
+ }
51
+ export interface RegisterServiceOptions {
52
+ /** Register the lifecycle owner but leave it stopped until a live enable. */
53
+ enabled?: boolean;
33
54
  }
34
55
  export declare class ServiceSupervisor {
35
56
  private readonly registry;
@@ -37,7 +58,16 @@ export declare class ServiceSupervisor {
37
58
  private readonly parkAfterFailures;
38
59
  private readonly backoffBaseMs;
39
60
  private readonly backoffMaxMs;
61
+ private readonly lifecycleDeadlineMs;
40
62
  constructor(opts?: ServiceSupervisorOptions);
63
+ /**
64
+ * Race `op` against a deadline. On breach the returned promise rejects with a
65
+ * labelled error while the real `op` is left to settle in the background —
66
+ * used to bound the lifecycle calls (`start`/`stop`/`restart`) so a wedged one
67
+ * cannot stall startup or shutdown. `Promise.race` cannot cancel `op`; the
68
+ * caller decides what a breach means for that service.
69
+ */
70
+ private withDeadline;
41
71
  /**
42
72
  * Register a service. Must be called before `startAll()`.
43
73
  *
@@ -46,14 +76,20 @@ export declare class ServiceSupervisor {
46
76
  * supervisor uses `isPeriodicService()` to decide whether to schedule a timer
47
77
  * for each registered entry.
48
78
  */
49
- register(service: DaemonService): void;
79
+ register(service: DaemonService, options?: RegisterServiceOptions): void;
80
+ /**
81
+ * Flip a tick's in-flight guard off and release every `awaitIdle()` waiter in
82
+ * the same synchronous step, so no waiter's continuation can run while
83
+ * `inFlight` is still true (PHNX-3608).
84
+ */
85
+ private clearInFlight;
50
86
  /** Start every registered service and begin ticking each on its own timer. */
51
87
  startAll(ctx: DaemonContext): Promise<void>;
52
88
  /** Stop every registered service and clear all timers. */
53
89
  stopAll(): Promise<void>;
54
90
  /** Force one service to restart right now, outside its normal backoff schedule. Drives `agents daemon services restart <id>` (RUSH-3193 P4). */
55
91
  restartOne(id: DaemonServiceId): Promise<void>;
56
- /** Whether `id` is registered on this supervisor. A service disabled at daemon boot is never registered — see {@link start}. */
92
+ /** Whether `id` has a lifecycle owner on this supervisor, running or stopped. */
57
93
  isRegistered(id: DaemonServiceId): boolean;
58
94
  /** Every currently registered service id. */
59
95
  registeredIds(): DaemonServiceId[];
@@ -66,9 +102,9 @@ export declare class ServiceSupervisor {
66
102
  /**
67
103
  * Start one registered service live — drives `agents daemon services enable
68
104
  * <id>` (RUSH-3193 P4). Only affects a service already registered on this
69
- * supervisor: one disabled at daemon boot was never constructed or
70
- * registered, so enabling it live is not possible without a daemon restart
71
- * (the CLI falls back to that advice — see `commands/daemon.ts`).
105
+ * supervisor. Callers that need live enable from a boot-disabled state must
106
+ * register the service with `{ enabled: false }`, which gives the supervisor
107
+ * ownership without running its startup side effects.
72
108
  */
73
109
  start(id: DaemonServiceId): Promise<void>;
74
110
  /** Stop one registered service live — drives `agents daemon services disable <id>` (RUSH-3193 P4). */
@@ -81,12 +117,16 @@ export declare class ServiceSupervisor {
81
117
  private stopOne;
82
118
  private scheduleTimer;
83
119
  /**
84
- * Run one tick under a hard deadline. The deadline is enforced with
85
- * `Promise.race`, not true cancellation — JS cannot abort an arbitrary
86
- * in-flight await. A timeout parks the service immediately but deliberately
87
- * keeps `inFlight` set until the REAL promise settles. Starting another tick
88
- * or tearing resources down underneath the abandoned one would trade a
89
- * visible parked service for duplicated side effects and lifecycle races.
120
+ * Run one tick under a hard deadline (PHNX-3608). The tick receives an
121
+ * `AbortSignal` that is aborted when the deadline elapses, so a cooperating
122
+ * tick can bound its own I/O and unwind. The deadline itself is still enforced
123
+ * with `Promise.race` — JS cannot forcibly cancel an arbitrary await — but on
124
+ * a breach the service is ABANDONED, not parked-and-held: the in-flight guard
125
+ * is released immediately and the backoff restart is scheduled right away
126
+ * (via `park()`), so a tick that never settles can no longer wedge backoff,
127
+ * `daemon services restart`, or SIGHUP reload. The runaway promise drains in
128
+ * the background; `finishTick` is version-guarded on `activeTick`, so its late
129
+ * settlement can never disturb the fresh tick a restart has since started.
90
130
  */
91
131
  private runTick;
92
132
  private park;
@@ -12,30 +12,62 @@
12
12
  * - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
13
13
  * local in-flight guard `true` forever, silently freezing that one service
14
14
  * for the daemon's life (observed ~51h). Here, every tick races a
15
- * `deadlineMs` timeout; when the deadline wins, that service is parked and
16
- * its in-flight guard stays held until the real promise settles. This keeps
17
- * a timed-out tick from overlapping another tick or lifecycle transition.
15
+ * `deadlineMs` timeout; when the deadline wins, the service is ABANDONED
16
+ * (PHNX-3608): its `AbortSignal` is aborted so a cooperating tick can unwind
17
+ * its own I/O, the in-flight guard is released immediately, and the backoff
18
+ * restart is scheduled right away — it is NOT gated on the runaway promise
19
+ * settling. An earlier revision kept the guard held until the real promise
20
+ * settled, which meant a tick that never settles parked the service forever
21
+ * and blocked backoff restart, `daemon services restart`, and SIGHUP reload —
22
+ * the exact 12h-usage-dark class this exists to prevent. The orphaned promise
23
+ * is drained separately in the background; `finishTick` is version-guarded so
24
+ * its late settlement can never disturb the fresh tick the restart started.
18
25
  *
19
26
  * Repeated failures (thrown or timed-out) open a circuit breaker: the service
20
27
  * is parked (its timer stopped) and retried on exponential backoff via its own
21
28
  * `restart()`, while every sibling service keeps ticking on its own timer,
22
- * unaffected.
29
+ * unaffected. `start()`/`stop()`/`restart()` are themselves bounded by
30
+ * {@link ServiceSupervisorOptions.lifecycleDeadlineMs} so a wedged bind or close
31
+ * cannot stall daemon startup or shutdown.
23
32
  */
24
33
  import { recordSubsystemOk, recordSubsystemError, recordSubsystemState } from '../daemon-health.js';
25
34
  import { isPeriodicService } from './service.js';
26
35
  const DEFAULT_PARK_AFTER_FAILURES = 3;
27
36
  const DEFAULT_BACKOFF_BASE_MS = 5_000;
28
37
  const DEFAULT_BACKOFF_MAX_MS = 5 * 60_000;
38
+ const DEFAULT_LIFECYCLE_DEADLINE_MS = 30_000;
29
39
  export class ServiceSupervisor {
30
40
  registry = new Map();
31
41
  ctx = null;
32
42
  parkAfterFailures;
33
43
  backoffBaseMs;
34
44
  backoffMaxMs;
45
+ lifecycleDeadlineMs;
35
46
  constructor(opts = {}) {
36
47
  this.parkAfterFailures = opts.parkAfterFailures ?? DEFAULT_PARK_AFTER_FAILURES;
37
48
  this.backoffBaseMs = opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
38
49
  this.backoffMaxMs = opts.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
50
+ this.lifecycleDeadlineMs = opts.lifecycleDeadlineMs ?? DEFAULT_LIFECYCLE_DEADLINE_MS;
51
+ }
52
+ /**
53
+ * Race `op` against a deadline. On breach the returned promise rejects with a
54
+ * labelled error while the real `op` is left to settle in the background —
55
+ * used to bound the lifecycle calls (`start`/`stop`/`restart`) so a wedged one
56
+ * cannot stall startup or shutdown. `Promise.race` cannot cancel `op`; the
57
+ * caller decides what a breach means for that service.
58
+ */
59
+ async withDeadline(op, ms, label) {
60
+ let timer;
61
+ const deadline = new Promise((_, reject) => {
62
+ timer = setTimeout(() => reject(new Error(`${label} exceeded deadline of ${ms}ms`)), ms);
63
+ });
64
+ try {
65
+ return await Promise.race([op(), deadline]);
66
+ }
67
+ finally {
68
+ if (timer)
69
+ clearTimeout(timer);
70
+ }
39
71
  }
40
72
  /**
41
73
  * Register a service. Must be called before `startAll()`.
@@ -45,24 +77,40 @@ export class ServiceSupervisor {
45
77
  * supervisor uses `isPeriodicService()` to decide whether to schedule a timer
46
78
  * for each registered entry.
47
79
  */
48
- register(service) {
80
+ register(service, options = {}) {
49
81
  if (this.registry.has(service.id))
50
82
  throw new Error(`service '${service.id}' is already registered`);
51
83
  this.registry.set(service.id, {
52
84
  service,
53
- state: 'idle',
85
+ state: options.enabled === false ? 'stopped' : 'idle',
54
86
  lastRunMs: 0,
55
87
  consecutiveFailures: 0,
56
88
  restartAttempts: 0,
57
89
  inFlight: false,
58
90
  everStarted: false,
91
+ idleWaiters: [],
59
92
  });
60
93
  }
94
+ /**
95
+ * Flip a tick's in-flight guard off and release every `awaitIdle()` waiter in
96
+ * the same synchronous step, so no waiter's continuation can run while
97
+ * `inFlight` is still true (PHNX-3608).
98
+ */
99
+ clearInFlight(entry) {
100
+ entry.inFlight = false;
101
+ const waiters = entry.idleWaiters;
102
+ entry.idleWaiters = [];
103
+ for (const resolve of waiters)
104
+ resolve();
105
+ }
61
106
  /** Start every registered service and begin ticking each on its own timer. */
62
107
  async startAll(ctx) {
63
108
  this.ctx = ctx;
64
- for (const id of this.registry.keys())
109
+ for (const [id, entry] of this.registry) {
110
+ if (entry.state === 'stopped')
111
+ continue;
65
112
  await this.startOne(id);
113
+ }
66
114
  }
67
115
  /** Stop every registered service and clear all timers. */
68
116
  async stopAll() {
@@ -98,7 +146,7 @@ export class ServiceSupervisor {
98
146
  recordSubsystemState(id, 'parked');
99
147
  await this.attemptRestart(id);
100
148
  }
101
- /** Whether `id` is registered on this supervisor. A service disabled at daemon boot is never registered — see {@link start}. */
149
+ /** Whether `id` has a lifecycle owner on this supervisor, running or stopped. */
102
150
  isRegistered(id) {
103
151
  return this.registry.has(id);
104
152
  }
@@ -115,17 +163,23 @@ export class ServiceSupervisor {
115
163
  const entry = this.registry.get(id);
116
164
  if (!entry)
117
165
  throw new Error(`service '${id}' is not registered`);
118
- const activeTick = entry.activeTick;
119
- if (!activeTick)
166
+ // Resolve from clearInFlight (which also flips `inFlight` false) rather than
167
+ // from the tick promise directly: awaiting the tick promise resumed this
168
+ // caller on a shorter microtask chain than the tick's own finally→finishTick,
169
+ // so a queued `.then(stop)` could run before `inFlight` was cleared and
170
+ // `stop()` would throw "cannot stop while a tick is still in flight"
171
+ // (PHNX-3608 — the SIGHUP live-disable regression). Waiting on the guard
172
+ // itself makes the ordering deterministic.
173
+ if (!entry.inFlight)
120
174
  return;
121
- await activeTick.catch(() => undefined);
175
+ await new Promise((resolve) => { entry.idleWaiters.push(resolve); });
122
176
  }
123
177
  /**
124
178
  * Start one registered service live — drives `agents daemon services enable
125
179
  * <id>` (RUSH-3193 P4). Only affects a service already registered on this
126
- * supervisor: one disabled at daemon boot was never constructed or
127
- * registered, so enabling it live is not possible without a daemon restart
128
- * (the CLI falls back to that advice — see `commands/daemon.ts`).
180
+ * supervisor. Callers that need live enable from a boot-disabled state must
181
+ * register the service with `{ enabled: false }`, which gives the supervisor
182
+ * ownership without running its startup side effects.
129
183
  */
130
184
  async start(id) {
131
185
  const entry = this.registry.get(id);
@@ -172,9 +226,12 @@ export class ServiceSupervisor {
172
226
  if (!entry || !ctx)
173
227
  return;
174
228
  try {
175
- await entry.service.start(ctx);
229
+ await this.withDeadline(() => entry.service.start(ctx), this.lifecycleDeadlineMs, `service '${id}' start`);
176
230
  }
177
231
  catch (err) {
232
+ // A thrown OR wedged start() parks the service and retries on backoff,
233
+ // rather than propagating out of startAll() and stalling the boot of
234
+ // every service after it (PHNX-3608).
178
235
  this.recordFailure(entry, id, err);
179
236
  this.park(id);
180
237
  return;
@@ -226,15 +283,26 @@ export class ServiceSupervisor {
226
283
  }
227
284
  entry.state = 'stopped';
228
285
  recordSubsystemState(id, 'stopped');
286
+ // Signal any in-flight tick to unwind — a cooperating tick threads this into
287
+ // its I/O and returns promptly at shutdown instead of blocking on it.
288
+ entry.activeController?.abort();
229
289
  // Whole-daemon shutdown must not hang forever on an unresolved tick, but it
230
290
  // also must not tear down resources the tick may still be using. Stop its
231
291
  // timers and mark it stopped; process exit owns final cleanup in this case.
232
- if (entry.inFlight && force)
292
+ if (entry.inFlight && force) {
293
+ // Release any awaitIdle waiter — the service is stopping, so no one should
294
+ // keep waiting on a tick that will now be abandoned at process exit.
295
+ const waiters = entry.idleWaiters;
296
+ entry.idleWaiters = [];
297
+ for (const resolve of waiters)
298
+ resolve();
233
299
  return;
300
+ }
234
301
  if (!entry.everStarted)
235
302
  return; // start() never succeeded — nothing to stop.
236
303
  try {
237
- await entry.service.stop();
304
+ // A wedged stop() must not stall stopAll() at shutdown (PHNX-3608).
305
+ await this.withDeadline(() => entry.service.stop(), this.lifecycleDeadlineMs, `service '${id}' stop`);
238
306
  }
239
307
  catch (err) {
240
308
  this.ctx?.log('WARN', `service '${id}' stop failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -247,12 +315,16 @@ export class ServiceSupervisor {
247
315
  entry.timer = setInterval(() => { void this.runTick(id); }, entry.service.intervalMs);
248
316
  }
249
317
  /**
250
- * Run one tick under a hard deadline. The deadline is enforced with
251
- * `Promise.race`, not true cancellation — JS cannot abort an arbitrary
252
- * in-flight await. A timeout parks the service immediately but deliberately
253
- * keeps `inFlight` set until the REAL promise settles. Starting another tick
254
- * or tearing resources down underneath the abandoned one would trade a
255
- * visible parked service for duplicated side effects and lifecycle races.
318
+ * Run one tick under a hard deadline (PHNX-3608). The tick receives an
319
+ * `AbortSignal` that is aborted when the deadline elapses, so a cooperating
320
+ * tick can bound its own I/O and unwind. The deadline itself is still enforced
321
+ * with `Promise.race` — JS cannot forcibly cancel an arbitrary await — but on
322
+ * a breach the service is ABANDONED, not parked-and-held: the in-flight guard
323
+ * is released immediately and the backoff restart is scheduled right away
324
+ * (via `park()`), so a tick that never settles can no longer wedge backoff,
325
+ * `daemon services restart`, or SIGHUP reload. The runaway promise drains in
326
+ * the background; `finishTick` is version-guarded on `activeTick`, so its late
327
+ * settlement can never disturb the fresh tick a restart has since started.
256
328
  */
257
329
  async runTick(id) {
258
330
  const entry = this.registry.get(id);
@@ -265,9 +337,11 @@ export class ServiceSupervisor {
265
337
  return;
266
338
  const periodicService = entry.service;
267
339
  entry.inFlight = true;
340
+ const controller = new AbortController();
341
+ entry.activeController = controller;
268
342
  let deadlineTimer;
269
343
  let timedOut = false;
270
- const tickPromise = periodicService.tick(ctx);
344
+ const tickPromise = periodicService.tick(ctx, controller.signal);
271
345
  entry.activeTick = tickPromise;
272
346
  try {
273
347
  const deadline = new Promise((_, reject) => {
@@ -291,9 +365,22 @@ export class ServiceSupervisor {
291
365
  if (deadlineTimer)
292
366
  clearTimeout(deadlineTimer);
293
367
  if (timedOut) {
294
- // Promise.race cannot cancel arbitrary work. Keep the service parked and
295
- // in-flight until the REAL tick settles; only then may backoff restart it.
296
- void tickPromise.then(() => this.finishTick(id, tickPromise), () => this.finishTick(id, tickPromise));
368
+ // Abandon the runaway tick: abort its signal so a cooperating tick can
369
+ // unwind, then release the in-flight guard NOW so the backoff restart
370
+ // park() scheduled is not blocked on a promise that may never settle.
371
+ // The real promise is drained separately and its result discarded — the
372
+ // version-guarded finishTick keeps that late settlement from touching a
373
+ // restart's fresh tick.
374
+ controller.abort();
375
+ entry.activeTick = undefined;
376
+ entry.activeController = undefined;
377
+ this.clearInFlight(entry);
378
+ // Re-read the entry so its `state` is the full union, not the `'running'`
379
+ // narrowing from this function's entry-guard — park() set it to 'parked'.
380
+ const current = this.registry.get(id);
381
+ if (current && current.state === 'parked' && !current.restartTimer)
382
+ this.scheduleRestart(id);
383
+ void tickPromise.then(() => undefined, () => undefined);
297
384
  }
298
385
  else {
299
386
  this.finishTick(id, tickPromise);
@@ -328,7 +415,9 @@ export class ServiceSupervisor {
328
415
  if (!entry || !ctx || entry.state === 'stopped')
329
416
  return;
330
417
  try {
331
- await entry.service.restart();
418
+ // A wedged restart() must not silently stall the circuit breaker — bound
419
+ // it and treat a breach as another failed restart, retried on backoff.
420
+ await this.withDeadline(() => entry.service.restart(), this.lifecycleDeadlineMs, `service '${id}' restart`);
332
421
  entry.everStarted = true; // restart() is stop()+start() on the service — a successful one means it is running again and owes a stop() at shutdown.
333
422
  entry.state = 'running';
334
423
  recordSubsystemState(id, 'running');
@@ -364,7 +453,8 @@ export class ServiceSupervisor {
364
453
  if (!entry || entry.activeTick !== tickPromise)
365
454
  return;
366
455
  entry.activeTick = undefined;
367
- entry.inFlight = false;
456
+ entry.activeController = undefined;
457
+ this.clearInFlight(entry);
368
458
  if (entry.state === 'parked' && !entry.restartTimer)
369
459
  this.scheduleRestart(id);
370
460
  }
@@ -1,12 +1,10 @@
1
1
  /**
2
2
  * Fleet usage-snapshot sync as a `PeriodicService` (PHNX-3392 usage-sync).
3
3
  *
4
- * On a headed box (personal/desktop) each tick pushes the local identity-keyed
5
- * Claude usage rows to worker peers that cannot read usage themselves. On a
6
- * worker whose cache is empty or stale, the same tick pulls those rows from the
7
- * primary headed device. Both drivers gate on the self role, so this service is
8
- * safe to register everywhere. Single-executor per destination: each daemon only
9
- * writes the DESTINATION's own cache, and the merge is newest-wins + idempotent.
4
+ * A headed box publishes its local identity-keyed Claude usage rows into its
5
+ * owned file in the fleet-synced user repo. The tick then runs one serialized,
6
+ * timeout-bounded git exchange; a worker reads the delivered peer snapshots
7
+ * and merges newest-wins. No tick opens a device-to-device SSH mesh.
10
8
  */
11
9
  import { BasePeriodicService, type DaemonContext } from './service.js';
12
10
  import type { DaemonServiceId } from '../daemon-services.js';
@@ -1,12 +1,10 @@
1
1
  /**
2
2
  * Fleet usage-snapshot sync as a `PeriodicService` (PHNX-3392 usage-sync).
3
3
  *
4
- * On a headed box (personal/desktop) each tick pushes the local identity-keyed
5
- * Claude usage rows to worker peers that cannot read usage themselves. On a
6
- * worker whose cache is empty or stale, the same tick pulls those rows from the
7
- * primary headed device. Both drivers gate on the self role, so this service is
8
- * safe to register everywhere. Single-executor per destination: each daemon only
9
- * writes the DESTINATION's own cache, and the merge is newest-wins + idempotent.
4
+ * A headed box publishes its local identity-keyed Claude usage rows into its
5
+ * owned file in the fleet-synced user repo. The tick then runs one serialized,
6
+ * timeout-bounded git exchange; a worker reads the delivered peer snapshots
7
+ * and merges newest-wins. No tick opens a device-to-device SSH mesh.
10
8
  */
11
9
  import { BasePeriodicService } from './service.js';
12
10
  const USAGE_SYNC_TICK_MS = 15 * 60_000;
@@ -24,19 +22,25 @@ export class UsageSyncService extends BasePeriodicService {
24
22
  // Nothing to release — the supervisor's timer teardown is the only cleanup.
25
23
  }
26
24
  async onTick(ctx) {
27
- const { pullUsageFromPrimary, syncFleetUsageSnapshots } = await import('../accounting/usage-sync.js');
28
- const pushResult = syncFleetUsageSnapshots();
29
- if (pushResult.pushed.length > 0) {
30
- ctx.log('INFO', `usage-sync: pushed usage to ${pushResult.pushed.join(', ')}`);
25
+ const { consumeUsageSnapshotsFromSharedStore, publishUsageSnapshotToSharedStore } = await import('../accounting/usage-sync.js');
26
+ const published = publishUsageSnapshotToSharedStore();
27
+ if (published.changed)
28
+ ctx.log('INFO', `usage-sync: published usage snapshot to ${published.path}`);
29
+ if (published.error)
30
+ ctx.log('WARN', `usage-sync: publish: ${published.error}`);
31
+ const { syncFleetSharedStateRepo } = await import('../fleet-shared-repo-sync.js');
32
+ const transport = await syncFleetSharedStateRepo();
33
+ if (transport.skipped)
34
+ ctx.log('WARN', `usage-sync: ${transport.skipped}`);
35
+ if (transport.error)
36
+ ctx.log('WARN', `usage-sync: shared-store transport: ${transport.error}`);
37
+ if (!transport.success)
38
+ return;
39
+ const consumed = consumeUsageSnapshotsFromSharedStore();
40
+ if (consumed.merged > 0) {
41
+ ctx.log('INFO', `usage-sync: merged ${consumed.merged} row(s) from ${consumed.sources.join(', ')}`);
31
42
  }
32
- for (const err of pushResult.errors) {
43
+ for (const err of consumed.errors)
33
44
  ctx.log('WARN', `usage-sync: ${err.device}: ${err.message}`);
34
- }
35
- const pullResult = pullUsageFromPrimary();
36
- if (pullResult.pulledFrom) {
37
- ctx.log('INFO', `usage-sync: pulled usage from ${pullResult.pulledFrom}; merged ${pullResult.merged} row(s)`);
38
- }
39
- if (pullResult.error)
40
- ctx.log('WARN', `usage-sync: pull: ${pullResult.error}`);
41
45
  }
42
46
  }