@phnx-labs/agents-cli 1.22.66 → 1.22.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +21 -8
- package/dist/bootstrap.js +6 -3
- package/dist/commands/browser.js +46 -21
- package/dist/commands/daemon-test-harness.d.ts +1 -1
- package/dist/commands/daemon-test-harness.js +2 -2
- package/dist/commands/daemon.js +21 -22
- package/dist/commands/exec.js +50 -0
- package/dist/commands/feed.js +20 -7
- package/dist/commands/monitors.js +5 -2
- package/dist/commands/projects.d.ts +26 -6
- package/dist/commands/projects.js +55 -22
- package/dist/commands/repo.js +57 -19
- package/dist/commands/resume.d.ts +16 -0
- package/dist/commands/resume.js +41 -8
- package/dist/commands/routines.js +42 -21
- package/dist/commands/send.js +29 -2
- package/dist/commands/sessions-inject.d.ts +58 -0
- package/dist/commands/sessions-inject.js +143 -7
- package/dist/commands/sessions-optimize.js +1 -1
- package/dist/commands/sessions-picker.js +1 -0
- package/dist/commands/sessions.js +4 -11
- package/dist/commands/share.d.ts +5 -3
- package/dist/commands/share.js +73 -20
- package/dist/commands/ssh.js +205 -2
- package/dist/lib/accounting/account-pool-collect.d.ts +6 -4
- package/dist/lib/accounting/account-pool-collect.js +6 -4
- package/dist/lib/accounting/usage-ingest.js +4 -2
- package/dist/lib/accounting/usage-sync.d.ts +38 -93
- package/dist/lib/accounting/usage-sync.js +66 -210
- package/dist/lib/accounting/usage.d.ts +18 -5
- package/dist/lib/accounting/usage.js +165 -20
- package/dist/lib/auth-health.d.ts +8 -0
- package/dist/lib/auth-health.js +4 -4
- package/dist/lib/boot-profile.d.ts +14 -0
- package/dist/lib/boot-profile.js +66 -0
- package/dist/lib/browser/caller-identity.d.ts +12 -0
- package/dist/lib/browser/caller-identity.js +19 -0
- package/dist/lib/browser/ipc.d.ts +37 -32
- package/dist/lib/browser/ipc.js +146 -94
- package/dist/lib/browser/task-index.d.ts +10 -2
- package/dist/lib/browser/task-index.js +22 -3
- package/dist/lib/channels/providers/desktop.d.ts +5 -4
- package/dist/lib/channels/providers/desktop.js +5 -4
- package/dist/lib/claude-account-token.js +108 -4
- package/dist/lib/daemon/account-state-daemon-service.d.ts +49 -9
- package/dist/lib/daemon/account-state-daemon-service.js +81 -18
- package/dist/lib/daemon/auth-sync-service.d.ts +4 -4
- package/dist/lib/daemon/auth-sync-service.js +17 -6
- package/dist/lib/daemon/catchup-service.d.ts +51 -0
- package/dist/lib/daemon/catchup-service.js +51 -0
- package/dist/lib/daemon/daemon.d.ts +12 -22
- package/dist/lib/daemon/daemon.js +463 -176
- package/dist/lib/daemon/runner.js +2 -0
- package/dist/lib/daemon/service.d.ts +22 -4
- package/dist/lib/daemon/service.js +2 -2
- package/dist/lib/daemon/supervisor.d.ts +55 -15
- package/dist/lib/daemon/supervisor.js +119 -29
- package/dist/lib/daemon/usage-sync-service.d.ts +4 -6
- package/dist/lib/daemon/usage-sync-service.js +22 -18
- package/dist/lib/daemon-health.js +36 -31
- package/dist/lib/daemon-services.d.ts +1 -1
- package/dist/lib/daemon-services.js +12 -2
- package/dist/lib/daemon-ticks.d.ts +9 -6
- package/dist/lib/daemon-ticks.js +14 -8
- package/dist/lib/devices/health.d.ts +38 -2
- package/dist/lib/devices/health.js +43 -5
- package/dist/lib/devices/registry.js +2 -0
- package/dist/lib/devices/worker-pick.d.ts +1 -1
- package/dist/lib/devices/worker-pick.js +4 -1
- package/dist/lib/exec.js +15 -0
- package/dist/lib/feed/watch.d.ts +3 -0
- package/dist/lib/feed/watch.js +13 -3
- package/dist/lib/feed-broadcast.d.ts +64 -5
- package/dist/lib/feed-broadcast.js +124 -22
- package/dist/lib/fleet-shared-repo-sync.d.ts +36 -0
- package/dist/lib/fleet-shared-repo-sync.js +333 -0
- package/dist/lib/fleet-shared-state.d.ts +38 -0
- package/dist/lib/fleet-shared-state.js +105 -0
- package/dist/lib/hosts/remote-cmd.d.ts +2 -0
- package/dist/lib/hosts/remote-cmd.js +12 -3
- package/dist/lib/lock-compromise.d.ts +8 -0
- package/dist/lib/lock-compromise.js +12 -0
- package/dist/lib/monitors/engine.d.ts +2 -1
- package/dist/lib/monitors/engine.js +27 -2
- package/dist/lib/monitors/sources/command.js +13 -3
- package/dist/lib/monitors/sources/failure.d.ts +32 -0
- package/dist/lib/monitors/sources/failure.js +52 -0
- package/dist/lib/monitors/sources/types.d.ts +9 -0
- package/dist/lib/owner-message.d.ts +12 -0
- package/dist/lib/owner-message.js +44 -0
- package/dist/lib/refresh-coordinator.js +2 -0
- package/dist/lib/run-trace-sync.d.ts +28 -0
- package/dist/lib/run-trace-sync.js +99 -0
- package/dist/lib/secrets/filestore.d.ts +4 -0
- package/dist/lib/secrets/filestore.js +164 -3
- package/dist/lib/secrets/push.d.ts +10 -0
- package/dist/lib/secrets/push.js +86 -7
- package/dist/lib/secrets/remote.d.ts +18 -6
- package/dist/lib/secrets/remote.js +29 -4
- package/dist/lib/secrets/reserved-sync.d.ts +28 -27
- package/dist/lib/secrets/reserved-sync.js +119 -101
- package/dist/lib/session/active.d.ts +13 -1
- package/dist/lib/session/active.js +5 -0
- package/dist/lib/session/actor-sidecar.d.ts +12 -0
- package/dist/lib/session/actor-sidecar.js +2 -0
- package/dist/lib/session/db.d.ts +16 -2
- package/dist/lib/session/db.js +73 -22
- package/dist/lib/session/discover.d.ts +12 -3
- package/dist/lib/session/discover.js +187 -26
- package/dist/lib/session/linear.d.ts +13 -0
- package/dist/lib/session/linear.js +44 -0
- package/dist/lib/session/live-metadata.js +1 -0
- package/dist/lib/session/parse.js +2 -3
- package/dist/lib/session/prompt.d.ts +17 -0
- package/dist/lib/session/prompt.js +35 -0
- package/dist/lib/session/recovery.d.ts +43 -6
- package/dist/lib/session/recovery.js +80 -10
- package/dist/lib/session/remote/remote-list.d.ts +17 -1
- package/dist/lib/session/remote/remote-list.js +29 -4
- package/dist/lib/session/remote/watch.d.ts +25 -2
- package/dist/lib/session/remote/watch.js +188 -11
- package/dist/lib/session/session-cache.d.ts +2 -1
- package/dist/lib/session/session-cache.js +1 -0
- package/dist/lib/session/state.js +11 -13
- package/dist/lib/session/types.d.ts +2 -0
- package/dist/lib/share/backend.d.ts +2 -2
- package/dist/lib/share/backend.js +20 -9
- package/dist/lib/share/delete.d.ts +5 -1
- package/dist/lib/share/delete.js +7 -2
- package/dist/lib/share/http-error.d.ts +52 -0
- package/dist/lib/share/http-error.js +65 -0
- package/dist/lib/share/publish.d.ts +67 -11
- package/dist/lib/share/publish.js +98 -16
- package/dist/lib/share/worker-template.js +105 -9
- package/dist/lib/smart-launch.js +27 -4
- package/dist/lib/ssh-exec.d.ts +2 -0
- package/dist/lib/ssh-exec.js +20 -4
- package/dist/lib/storage/index.d.ts +14 -0
- package/dist/lib/storage/index.js +14 -0
- package/dist/lib/storage/selection.d.ts +48 -0
- package/dist/lib/storage/selection.js +39 -0
- package/dist/lib/storage/visibility.d.ts +82 -0
- package/dist/lib/storage/visibility.js +99 -0
- package/dist/lib/teams/agents.js +3 -1
- package/dist/lib/teams/placement-probe.js +1 -0
- package/dist/lib/teams/registry.js +2 -0
- package/dist/lib/teams/scheduler.d.ts +8 -1
- package/dist/lib/teams/scheduler.js +4 -1
- package/dist/lib/testdata/daemon-health-writer.d.ts +1 -0
- package/dist/lib/testdata/daemon-health-writer.js +8 -0
- package/dist/lib/traces/backend.js +13 -2
- package/dist/lib/traces/sync.d.ts +7 -0
- package/dist/lib/traces/sync.js +9 -0
- package/dist/lib/usage-refresh.d.ts +8 -2
- package/dist/lib/usage-refresh.js +3 -3
- package/dist/lib/worktree/held.d.ts +166 -0
- package/dist/lib/worktree/held.js +368 -0
- package/package.json +2 -2
- package/dist/lib/account-state-service.d.ts +0 -21
- package/dist/lib/account-state-service.js +0 -60
|
@@ -33,6 +33,7 @@ import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
|
|
|
33
33
|
import { backgroundSpawnOptions, killTree } from '../platform/process.js';
|
|
34
34
|
import lockfile from 'proper-lockfile';
|
|
35
35
|
import { ensureLockTarget } from '../fs-atomic.js';
|
|
36
|
+
import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
|
|
36
37
|
import { walkForFiles } from '../fs-walk.js';
|
|
37
38
|
import { getBinaryPath, isVersionInstalled, resolveVersion, getVersionHomePath } from '../installations/versions.js';
|
|
38
39
|
import { resolveClaudeSetupToken } from '../claude-account-token.js';
|
|
@@ -153,6 +154,7 @@ async function withRoutineLock(config, launch) {
|
|
|
153
154
|
minTimeout: 100,
|
|
154
155
|
maxTimeout: 100,
|
|
155
156
|
},
|
|
157
|
+
onCompromised: logAndContinueOnLockCompromised('routines launch'),
|
|
156
158
|
});
|
|
157
159
|
try {
|
|
158
160
|
return await launch();
|
|
@@ -41,7 +41,14 @@ export interface DaemonService {
|
|
|
41
41
|
/** A service the supervisor ticks on a fixed interval, under a hard per-tick deadline. */
|
|
42
42
|
export interface PeriodicService extends DaemonService {
|
|
43
43
|
readonly intervalMs: number;
|
|
44
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Hard cap per tick. An over-budget tick is ABANDONED: the supervisor aborts
|
|
46
|
+
* the tick's {@link AbortSignal}, parks the service, and schedules a backoff
|
|
47
|
+
* restart immediately — it does NOT wait for the real promise to settle
|
|
48
|
+
* (PHNX-3608). A tick that awaits `signal` (or forwards it to its I/O) can
|
|
49
|
+
* observe the deadline and unwind; one that ignores it is left to drain in the
|
|
50
|
+
* background while the service is already being restarted.
|
|
51
|
+
*/
|
|
45
52
|
readonly deadlineMs: number;
|
|
46
53
|
/**
|
|
47
54
|
* Delay before the FIRST tick after `start()`, in ms. Every later tick
|
|
@@ -52,7 +59,13 @@ export interface PeriodicService extends DaemonService {
|
|
|
52
59
|
* firing at daemon startup.
|
|
53
60
|
*/
|
|
54
61
|
readonly startupDelayMs?: number;
|
|
55
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Run one tick. `signal` aborts when the supervisor's {@link deadlineMs}
|
|
64
|
+
* elapses (or the service is being stopped), so a well-behaved tick threads it
|
|
65
|
+
* into its awaits — `fetch(url, { signal })`, an ssh/exec spawn, a
|
|
66
|
+
* cancellable sleep — to bound its own I/O instead of leaking a runaway await.
|
|
67
|
+
*/
|
|
68
|
+
tick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
|
|
56
69
|
}
|
|
57
70
|
export declare function isPeriodicService(service: DaemonService): service is PeriodicService;
|
|
58
71
|
/**
|
|
@@ -93,10 +106,15 @@ export declare abstract class BasePeriodicService implements PeriodicService {
|
|
|
93
106
|
private healthRecord;
|
|
94
107
|
protected abstract onStart(ctx: DaemonContext): Promise<void>;
|
|
95
108
|
protected abstract onStop(): Promise<void>;
|
|
96
|
-
|
|
109
|
+
/**
|
|
110
|
+
* Run one tick. `signal` aborts at the supervisor's deadline (or on stop);
|
|
111
|
+
* thread it into the tick's own awaits to bound its I/O. A subclass that does
|
|
112
|
+
* no cancellable work may implement `onTick(ctx)` and ignore the second arg.
|
|
113
|
+
*/
|
|
114
|
+
protected abstract onTick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
|
|
97
115
|
start(ctx: DaemonContext): Promise<void>;
|
|
98
116
|
stop(): Promise<void>;
|
|
99
117
|
restart(): Promise<void>;
|
|
100
|
-
tick(ctx: DaemonContext): Promise<void>;
|
|
118
|
+
tick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
|
|
101
119
|
health(): ServiceHealth;
|
|
102
120
|
}
|
|
@@ -81,9 +81,9 @@ export class BasePeriodicService {
|
|
|
81
81
|
await this.stop();
|
|
82
82
|
await this.start(ctx);
|
|
83
83
|
}
|
|
84
|
-
async tick(ctx) {
|
|
84
|
+
async tick(ctx, signal) {
|
|
85
85
|
try {
|
|
86
|
-
await this.onTick(ctx);
|
|
86
|
+
await this.onTick(ctx, signal);
|
|
87
87
|
this.healthRecord = { ...this.healthRecord, lastRunMs: Date.now(), consecutiveFailures: 0, lastError: undefined };
|
|
88
88
|
}
|
|
89
89
|
catch (err) {
|
|
@@ -12,14 +12,23 @@
|
|
|
12
12
|
* - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
|
|
13
13
|
* local in-flight guard `true` forever, silently freezing that one service
|
|
14
14
|
* for the daemon's life (observed ~51h). Here, every tick races a
|
|
15
|
-
* `deadlineMs` timeout; when the deadline wins,
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* `deadlineMs` timeout; when the deadline wins, the service is ABANDONED
|
|
16
|
+
* (PHNX-3608): its `AbortSignal` is aborted so a cooperating tick can unwind
|
|
17
|
+
* its own I/O, the in-flight guard is released immediately, and the backoff
|
|
18
|
+
* restart is scheduled right away — it is NOT gated on the runaway promise
|
|
19
|
+
* settling. An earlier revision kept the guard held until the real promise
|
|
20
|
+
* settled, which meant a tick that never settles parked the service forever
|
|
21
|
+
* and blocked backoff restart, `daemon services restart`, and SIGHUP reload —
|
|
22
|
+
* the exact 12h-usage-dark class this exists to prevent. The orphaned promise
|
|
23
|
+
* is drained separately in the background; `finishTick` is version-guarded so
|
|
24
|
+
* its late settlement can never disturb the fresh tick the restart started.
|
|
18
25
|
*
|
|
19
26
|
* Repeated failures (thrown or timed-out) open a circuit breaker: the service
|
|
20
27
|
* is parked (its timer stopped) and retried on exponential backoff via its own
|
|
21
28
|
* `restart()`, while every sibling service keeps ticking on its own timer,
|
|
22
|
-
* unaffected.
|
|
29
|
+
* unaffected. `start()`/`stop()`/`restart()` are themselves bounded by
|
|
30
|
+
* {@link ServiceSupervisorOptions.lifecycleDeadlineMs} so a wedged bind or close
|
|
31
|
+
* cannot stall daemon startup or shutdown.
|
|
23
32
|
*/
|
|
24
33
|
import type { DaemonServiceId } from '../daemon-services.js';
|
|
25
34
|
import type { DaemonContext, DaemonService, ServiceHealth } from './service.js';
|
|
@@ -30,6 +39,18 @@ export interface ServiceSupervisorOptions {
|
|
|
30
39
|
backoffBaseMs?: number;
|
|
31
40
|
/** Backoff ceiling. Default 5 minutes. */
|
|
32
41
|
backoffMaxMs?: number;
|
|
42
|
+
/**
|
|
43
|
+
* Hard cap on a service's `start()`/`stop()`/`restart()` call (PHNX-3608). A
|
|
44
|
+
* wedged bind/close would otherwise stall `startAll()` (which awaits each
|
|
45
|
+
* `startOne` in turn) or `stopAll()` at shutdown. A start/restart that
|
|
46
|
+
* breaches it is treated as a failure (parked + backoff); a stop that breaches
|
|
47
|
+
* it is logged and the service is left marked stopped. Default 30s.
|
|
48
|
+
*/
|
|
49
|
+
lifecycleDeadlineMs?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface RegisterServiceOptions {
|
|
52
|
+
/** Register the lifecycle owner but leave it stopped until a live enable. */
|
|
53
|
+
enabled?: boolean;
|
|
33
54
|
}
|
|
34
55
|
export declare class ServiceSupervisor {
|
|
35
56
|
private readonly registry;
|
|
@@ -37,7 +58,16 @@ export declare class ServiceSupervisor {
|
|
|
37
58
|
private readonly parkAfterFailures;
|
|
38
59
|
private readonly backoffBaseMs;
|
|
39
60
|
private readonly backoffMaxMs;
|
|
61
|
+
private readonly lifecycleDeadlineMs;
|
|
40
62
|
constructor(opts?: ServiceSupervisorOptions);
|
|
63
|
+
/**
|
|
64
|
+
* Race `op` against a deadline. On breach the returned promise rejects with a
|
|
65
|
+
* labelled error while the real `op` is left to settle in the background —
|
|
66
|
+
* used to bound the lifecycle calls (`start`/`stop`/`restart`) so a wedged one
|
|
67
|
+
* cannot stall startup or shutdown. `Promise.race` cannot cancel `op`; the
|
|
68
|
+
* caller decides what a breach means for that service.
|
|
69
|
+
*/
|
|
70
|
+
private withDeadline;
|
|
41
71
|
/**
|
|
42
72
|
* Register a service. Must be called before `startAll()`.
|
|
43
73
|
*
|
|
@@ -46,14 +76,20 @@ export declare class ServiceSupervisor {
|
|
|
46
76
|
* supervisor uses `isPeriodicService()` to decide whether to schedule a timer
|
|
47
77
|
* for each registered entry.
|
|
48
78
|
*/
|
|
49
|
-
register(service: DaemonService): void;
|
|
79
|
+
register(service: DaemonService, options?: RegisterServiceOptions): void;
|
|
80
|
+
/**
|
|
81
|
+
* Flip a tick's in-flight guard off and release every `awaitIdle()` waiter in
|
|
82
|
+
* the same synchronous step, so no waiter's continuation can run while
|
|
83
|
+
* `inFlight` is still true (PHNX-3608).
|
|
84
|
+
*/
|
|
85
|
+
private clearInFlight;
|
|
50
86
|
/** Start every registered service and begin ticking each on its own timer. */
|
|
51
87
|
startAll(ctx: DaemonContext): Promise<void>;
|
|
52
88
|
/** Stop every registered service and clear all timers. */
|
|
53
89
|
stopAll(): Promise<void>;
|
|
54
90
|
/** Force one service to restart right now, outside its normal backoff schedule. Drives `agents daemon services restart <id>` (RUSH-3193 P4). */
|
|
55
91
|
restartOne(id: DaemonServiceId): Promise<void>;
|
|
56
|
-
/** Whether `id`
|
|
92
|
+
/** Whether `id` has a lifecycle owner on this supervisor, running or stopped. */
|
|
57
93
|
isRegistered(id: DaemonServiceId): boolean;
|
|
58
94
|
/** Every currently registered service id. */
|
|
59
95
|
registeredIds(): DaemonServiceId[];
|
|
@@ -66,9 +102,9 @@ export declare class ServiceSupervisor {
|
|
|
66
102
|
/**
|
|
67
103
|
* Start one registered service live — drives `agents daemon services enable
|
|
68
104
|
* <id>` (RUSH-3193 P4). Only affects a service already registered on this
|
|
69
|
-
* supervisor
|
|
70
|
-
*
|
|
71
|
-
*
|
|
105
|
+
* supervisor. Callers that need live enable from a boot-disabled state must
|
|
106
|
+
* register the service with `{ enabled: false }`, which gives the supervisor
|
|
107
|
+
* ownership without running its startup side effects.
|
|
72
108
|
*/
|
|
73
109
|
start(id: DaemonServiceId): Promise<void>;
|
|
74
110
|
/** Stop one registered service live — drives `agents daemon services disable <id>` (RUSH-3193 P4). */
|
|
@@ -81,12 +117,16 @@ export declare class ServiceSupervisor {
|
|
|
81
117
|
private stopOne;
|
|
82
118
|
private scheduleTimer;
|
|
83
119
|
/**
|
|
84
|
-
* Run one tick under a hard deadline. The
|
|
85
|
-
* `
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
120
|
+
* Run one tick under a hard deadline (PHNX-3608). The tick receives an
|
|
121
|
+
* `AbortSignal` that is aborted when the deadline elapses, so a cooperating
|
|
122
|
+
* tick can bound its own I/O and unwind. The deadline itself is still enforced
|
|
123
|
+
* with `Promise.race` — JS cannot forcibly cancel an arbitrary await — but on
|
|
124
|
+
* a breach the service is ABANDONED, not parked-and-held: the in-flight guard
|
|
125
|
+
* is released immediately and the backoff restart is scheduled right away
|
|
126
|
+
* (via `park()`), so a tick that never settles can no longer wedge backoff,
|
|
127
|
+
* `daemon services restart`, or SIGHUP reload. The runaway promise drains in
|
|
128
|
+
* the background; `finishTick` is version-guarded on `activeTick`, so its late
|
|
129
|
+
* settlement can never disturb the fresh tick a restart has since started.
|
|
90
130
|
*/
|
|
91
131
|
private runTick;
|
|
92
132
|
private park;
|
|
@@ -12,30 +12,62 @@
|
|
|
12
12
|
* - A tick that hangs on an unbounded await (SSH, keychain) used to latch its
|
|
13
13
|
* local in-flight guard `true` forever, silently freezing that one service
|
|
14
14
|
* for the daemon's life (observed ~51h). Here, every tick races a
|
|
15
|
-
* `deadlineMs` timeout; when the deadline wins,
|
|
16
|
-
*
|
|
17
|
-
*
|
|
15
|
+
* `deadlineMs` timeout; when the deadline wins, the service is ABANDONED
|
|
16
|
+
* (PHNX-3608): its `AbortSignal` is aborted so a cooperating tick can unwind
|
|
17
|
+
* its own I/O, the in-flight guard is released immediately, and the backoff
|
|
18
|
+
* restart is scheduled right away — it is NOT gated on the runaway promise
|
|
19
|
+
* settling. An earlier revision kept the guard held until the real promise
|
|
20
|
+
* settled, which meant a tick that never settles parked the service forever
|
|
21
|
+
* and blocked backoff restart, `daemon services restart`, and SIGHUP reload —
|
|
22
|
+
* the exact 12h-usage-dark class this exists to prevent. The orphaned promise
|
|
23
|
+
* is drained separately in the background; `finishTick` is version-guarded so
|
|
24
|
+
* its late settlement can never disturb the fresh tick the restart started.
|
|
18
25
|
*
|
|
19
26
|
* Repeated failures (thrown or timed-out) open a circuit breaker: the service
|
|
20
27
|
* is parked (its timer stopped) and retried on exponential backoff via its own
|
|
21
28
|
* `restart()`, while every sibling service keeps ticking on its own timer,
|
|
22
|
-
* unaffected.
|
|
29
|
+
* unaffected. `start()`/`stop()`/`restart()` are themselves bounded by
|
|
30
|
+
* {@link ServiceSupervisorOptions.lifecycleDeadlineMs} so a wedged bind or close
|
|
31
|
+
* cannot stall daemon startup or shutdown.
|
|
23
32
|
*/
|
|
24
33
|
import { recordSubsystemOk, recordSubsystemError, recordSubsystemState } from '../daemon-health.js';
|
|
25
34
|
import { isPeriodicService } from './service.js';
|
|
26
35
|
const DEFAULT_PARK_AFTER_FAILURES = 3;
|
|
27
36
|
const DEFAULT_BACKOFF_BASE_MS = 5_000;
|
|
28
37
|
const DEFAULT_BACKOFF_MAX_MS = 5 * 60_000;
|
|
38
|
+
const DEFAULT_LIFECYCLE_DEADLINE_MS = 30_000;
|
|
29
39
|
export class ServiceSupervisor {
|
|
30
40
|
registry = new Map();
|
|
31
41
|
ctx = null;
|
|
32
42
|
parkAfterFailures;
|
|
33
43
|
backoffBaseMs;
|
|
34
44
|
backoffMaxMs;
|
|
45
|
+
lifecycleDeadlineMs;
|
|
35
46
|
constructor(opts = {}) {
|
|
36
47
|
this.parkAfterFailures = opts.parkAfterFailures ?? DEFAULT_PARK_AFTER_FAILURES;
|
|
37
48
|
this.backoffBaseMs = opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
38
49
|
this.backoffMaxMs = opts.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
|
|
50
|
+
this.lifecycleDeadlineMs = opts.lifecycleDeadlineMs ?? DEFAULT_LIFECYCLE_DEADLINE_MS;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Race `op` against a deadline. On breach the returned promise rejects with a
|
|
54
|
+
* labelled error while the real `op` is left to settle in the background —
|
|
55
|
+
* used to bound the lifecycle calls (`start`/`stop`/`restart`) so a wedged one
|
|
56
|
+
* cannot stall startup or shutdown. `Promise.race` cannot cancel `op`; the
|
|
57
|
+
* caller decides what a breach means for that service.
|
|
58
|
+
*/
|
|
59
|
+
async withDeadline(op, ms, label) {
|
|
60
|
+
let timer;
|
|
61
|
+
const deadline = new Promise((_, reject) => {
|
|
62
|
+
timer = setTimeout(() => reject(new Error(`${label} exceeded deadline of ${ms}ms`)), ms);
|
|
63
|
+
});
|
|
64
|
+
try {
|
|
65
|
+
return await Promise.race([op(), deadline]);
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
if (timer)
|
|
69
|
+
clearTimeout(timer);
|
|
70
|
+
}
|
|
39
71
|
}
|
|
40
72
|
/**
|
|
41
73
|
* Register a service. Must be called before `startAll()`.
|
|
@@ -45,24 +77,40 @@ export class ServiceSupervisor {
|
|
|
45
77
|
* supervisor uses `isPeriodicService()` to decide whether to schedule a timer
|
|
46
78
|
* for each registered entry.
|
|
47
79
|
*/
|
|
48
|
-
register(service) {
|
|
80
|
+
register(service, options = {}) {
|
|
49
81
|
if (this.registry.has(service.id))
|
|
50
82
|
throw new Error(`service '${service.id}' is already registered`);
|
|
51
83
|
this.registry.set(service.id, {
|
|
52
84
|
service,
|
|
53
|
-
state: 'idle',
|
|
85
|
+
state: options.enabled === false ? 'stopped' : 'idle',
|
|
54
86
|
lastRunMs: 0,
|
|
55
87
|
consecutiveFailures: 0,
|
|
56
88
|
restartAttempts: 0,
|
|
57
89
|
inFlight: false,
|
|
58
90
|
everStarted: false,
|
|
91
|
+
idleWaiters: [],
|
|
59
92
|
});
|
|
60
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* Flip a tick's in-flight guard off and release every `awaitIdle()` waiter in
|
|
96
|
+
* the same synchronous step, so no waiter's continuation can run while
|
|
97
|
+
* `inFlight` is still true (PHNX-3608).
|
|
98
|
+
*/
|
|
99
|
+
clearInFlight(entry) {
|
|
100
|
+
entry.inFlight = false;
|
|
101
|
+
const waiters = entry.idleWaiters;
|
|
102
|
+
entry.idleWaiters = [];
|
|
103
|
+
for (const resolve of waiters)
|
|
104
|
+
resolve();
|
|
105
|
+
}
|
|
61
106
|
/** Start every registered service and begin ticking each on its own timer. */
|
|
62
107
|
async startAll(ctx) {
|
|
63
108
|
this.ctx = ctx;
|
|
64
|
-
for (const id of this.registry
|
|
109
|
+
for (const [id, entry] of this.registry) {
|
|
110
|
+
if (entry.state === 'stopped')
|
|
111
|
+
continue;
|
|
65
112
|
await this.startOne(id);
|
|
113
|
+
}
|
|
66
114
|
}
|
|
67
115
|
/** Stop every registered service and clear all timers. */
|
|
68
116
|
async stopAll() {
|
|
@@ -98,7 +146,7 @@ export class ServiceSupervisor {
|
|
|
98
146
|
recordSubsystemState(id, 'parked');
|
|
99
147
|
await this.attemptRestart(id);
|
|
100
148
|
}
|
|
101
|
-
/** Whether `id`
|
|
149
|
+
/** Whether `id` has a lifecycle owner on this supervisor, running or stopped. */
|
|
102
150
|
isRegistered(id) {
|
|
103
151
|
return this.registry.has(id);
|
|
104
152
|
}
|
|
@@ -115,17 +163,23 @@ export class ServiceSupervisor {
|
|
|
115
163
|
const entry = this.registry.get(id);
|
|
116
164
|
if (!entry)
|
|
117
165
|
throw new Error(`service '${id}' is not registered`);
|
|
118
|
-
|
|
119
|
-
|
|
166
|
+
// Resolve from clearInFlight (which also flips `inFlight` false) rather than
|
|
167
|
+
// from the tick promise directly: awaiting the tick promise resumed this
|
|
168
|
+
// caller on a shorter microtask chain than the tick's own finally→finishTick,
|
|
169
|
+
// so a queued `.then(stop)` could run before `inFlight` was cleared and
|
|
170
|
+
// `stop()` would throw "cannot stop while a tick is still in flight"
|
|
171
|
+
// (PHNX-3608 — the SIGHUP live-disable regression). Waiting on the guard
|
|
172
|
+
// itself makes the ordering deterministic.
|
|
173
|
+
if (!entry.inFlight)
|
|
120
174
|
return;
|
|
121
|
-
await
|
|
175
|
+
await new Promise((resolve) => { entry.idleWaiters.push(resolve); });
|
|
122
176
|
}
|
|
123
177
|
/**
|
|
124
178
|
* Start one registered service live — drives `agents daemon services enable
|
|
125
179
|
* <id>` (RUSH-3193 P4). Only affects a service already registered on this
|
|
126
|
-
* supervisor
|
|
127
|
-
*
|
|
128
|
-
*
|
|
180
|
+
* supervisor. Callers that need live enable from a boot-disabled state must
|
|
181
|
+
* register the service with `{ enabled: false }`, which gives the supervisor
|
|
182
|
+
* ownership without running its startup side effects.
|
|
129
183
|
*/
|
|
130
184
|
async start(id) {
|
|
131
185
|
const entry = this.registry.get(id);
|
|
@@ -172,9 +226,12 @@ export class ServiceSupervisor {
|
|
|
172
226
|
if (!entry || !ctx)
|
|
173
227
|
return;
|
|
174
228
|
try {
|
|
175
|
-
await entry.service.start(ctx);
|
|
229
|
+
await this.withDeadline(() => entry.service.start(ctx), this.lifecycleDeadlineMs, `service '${id}' start`);
|
|
176
230
|
}
|
|
177
231
|
catch (err) {
|
|
232
|
+
// A thrown OR wedged start() parks the service and retries on backoff,
|
|
233
|
+
// rather than propagating out of startAll() and stalling the boot of
|
|
234
|
+
// every service after it (PHNX-3608).
|
|
178
235
|
this.recordFailure(entry, id, err);
|
|
179
236
|
this.park(id);
|
|
180
237
|
return;
|
|
@@ -226,15 +283,26 @@ export class ServiceSupervisor {
|
|
|
226
283
|
}
|
|
227
284
|
entry.state = 'stopped';
|
|
228
285
|
recordSubsystemState(id, 'stopped');
|
|
286
|
+
// Signal any in-flight tick to unwind — a cooperating tick threads this into
|
|
287
|
+
// its I/O and returns promptly at shutdown instead of blocking on it.
|
|
288
|
+
entry.activeController?.abort();
|
|
229
289
|
// Whole-daemon shutdown must not hang forever on an unresolved tick, but it
|
|
230
290
|
// also must not tear down resources the tick may still be using. Stop its
|
|
231
291
|
// timers and mark it stopped; process exit owns final cleanup in this case.
|
|
232
|
-
if (entry.inFlight && force)
|
|
292
|
+
if (entry.inFlight && force) {
|
|
293
|
+
// Release any awaitIdle waiter — the service is stopping, so no one should
|
|
294
|
+
// keep waiting on a tick that will now be abandoned at process exit.
|
|
295
|
+
const waiters = entry.idleWaiters;
|
|
296
|
+
entry.idleWaiters = [];
|
|
297
|
+
for (const resolve of waiters)
|
|
298
|
+
resolve();
|
|
233
299
|
return;
|
|
300
|
+
}
|
|
234
301
|
if (!entry.everStarted)
|
|
235
302
|
return; // start() never succeeded — nothing to stop.
|
|
236
303
|
try {
|
|
237
|
-
|
|
304
|
+
// A wedged stop() must not stall stopAll() at shutdown (PHNX-3608).
|
|
305
|
+
await this.withDeadline(() => entry.service.stop(), this.lifecycleDeadlineMs, `service '${id}' stop`);
|
|
238
306
|
}
|
|
239
307
|
catch (err) {
|
|
240
308
|
this.ctx?.log('WARN', `service '${id}' stop failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -247,12 +315,16 @@ export class ServiceSupervisor {
|
|
|
247
315
|
entry.timer = setInterval(() => { void this.runTick(id); }, entry.service.intervalMs);
|
|
248
316
|
}
|
|
249
317
|
/**
|
|
250
|
-
* Run one tick under a hard deadline. The
|
|
251
|
-
* `
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
318
|
+
* Run one tick under a hard deadline (PHNX-3608). The tick receives an
|
|
319
|
+
* `AbortSignal` that is aborted when the deadline elapses, so a cooperating
|
|
320
|
+
* tick can bound its own I/O and unwind. The deadline itself is still enforced
|
|
321
|
+
* with `Promise.race` — JS cannot forcibly cancel an arbitrary await — but on
|
|
322
|
+
* a breach the service is ABANDONED, not parked-and-held: the in-flight guard
|
|
323
|
+
* is released immediately and the backoff restart is scheduled right away
|
|
324
|
+
* (via `park()`), so a tick that never settles can no longer wedge backoff,
|
|
325
|
+
* `daemon services restart`, or SIGHUP reload. The runaway promise drains in
|
|
326
|
+
* the background; `finishTick` is version-guarded on `activeTick`, so its late
|
|
327
|
+
* settlement can never disturb the fresh tick a restart has since started.
|
|
256
328
|
*/
|
|
257
329
|
async runTick(id) {
|
|
258
330
|
const entry = this.registry.get(id);
|
|
@@ -265,9 +337,11 @@ export class ServiceSupervisor {
|
|
|
265
337
|
return;
|
|
266
338
|
const periodicService = entry.service;
|
|
267
339
|
entry.inFlight = true;
|
|
340
|
+
const controller = new AbortController();
|
|
341
|
+
entry.activeController = controller;
|
|
268
342
|
let deadlineTimer;
|
|
269
343
|
let timedOut = false;
|
|
270
|
-
const tickPromise = periodicService.tick(ctx);
|
|
344
|
+
const tickPromise = periodicService.tick(ctx, controller.signal);
|
|
271
345
|
entry.activeTick = tickPromise;
|
|
272
346
|
try {
|
|
273
347
|
const deadline = new Promise((_, reject) => {
|
|
@@ -291,9 +365,22 @@ export class ServiceSupervisor {
|
|
|
291
365
|
if (deadlineTimer)
|
|
292
366
|
clearTimeout(deadlineTimer);
|
|
293
367
|
if (timedOut) {
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
|
|
368
|
+
// Abandon the runaway tick: abort its signal so a cooperating tick can
|
|
369
|
+
// unwind, then release the in-flight guard NOW so the backoff restart
|
|
370
|
+
// park() scheduled is not blocked on a promise that may never settle.
|
|
371
|
+
// The real promise is drained separately and its result discarded — the
|
|
372
|
+
// version-guarded finishTick keeps that late settlement from touching a
|
|
373
|
+
// restart's fresh tick.
|
|
374
|
+
controller.abort();
|
|
375
|
+
entry.activeTick = undefined;
|
|
376
|
+
entry.activeController = undefined;
|
|
377
|
+
this.clearInFlight(entry);
|
|
378
|
+
// Re-read the entry so its `state` is the full union, not the `'running'`
|
|
379
|
+
// narrowing from this function's entry-guard — park() set it to 'parked'.
|
|
380
|
+
const current = this.registry.get(id);
|
|
381
|
+
if (current && current.state === 'parked' && !current.restartTimer)
|
|
382
|
+
this.scheduleRestart(id);
|
|
383
|
+
void tickPromise.then(() => undefined, () => undefined);
|
|
297
384
|
}
|
|
298
385
|
else {
|
|
299
386
|
this.finishTick(id, tickPromise);
|
|
@@ -328,7 +415,9 @@ export class ServiceSupervisor {
|
|
|
328
415
|
if (!entry || !ctx || entry.state === 'stopped')
|
|
329
416
|
return;
|
|
330
417
|
try {
|
|
331
|
-
|
|
418
|
+
// A wedged restart() must not silently stall the circuit breaker — bound
|
|
419
|
+
// it and treat a breach as another failed restart, retried on backoff.
|
|
420
|
+
await this.withDeadline(() => entry.service.restart(), this.lifecycleDeadlineMs, `service '${id}' restart`);
|
|
332
421
|
entry.everStarted = true; // restart() is stop()+start() on the service — a successful one means it is running again and owes a stop() at shutdown.
|
|
333
422
|
entry.state = 'running';
|
|
334
423
|
recordSubsystemState(id, 'running');
|
|
@@ -364,7 +453,8 @@ export class ServiceSupervisor {
|
|
|
364
453
|
if (!entry || entry.activeTick !== tickPromise)
|
|
365
454
|
return;
|
|
366
455
|
entry.activeTick = undefined;
|
|
367
|
-
entry.
|
|
456
|
+
entry.activeController = undefined;
|
|
457
|
+
this.clearInFlight(entry);
|
|
368
458
|
if (entry.state === 'parked' && !entry.restartTimer)
|
|
369
459
|
this.scheduleRestart(id);
|
|
370
460
|
}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Fleet usage-snapshot sync as a `PeriodicService` (PHNX-3392 usage-sync).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* safe to register everywhere. Single-executor per destination: each daemon only
|
|
9
|
-
* writes the DESTINATION's own cache, and the merge is newest-wins + idempotent.
|
|
4
|
+
* A headed box publishes its local identity-keyed Claude usage rows into its
|
|
5
|
+
* owned file in the fleet-synced user repo. The tick then runs one serialized,
|
|
6
|
+
* timeout-bounded git exchange; a worker reads the delivered peer snapshots
|
|
7
|
+
* and merges newest-wins. No tick opens a device-to-device SSH mesh.
|
|
10
8
|
*/
|
|
11
9
|
import { BasePeriodicService, type DaemonContext } from './service.js';
|
|
12
10
|
import type { DaemonServiceId } from '../daemon-services.js';
|
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Fleet usage-snapshot sync as a `PeriodicService` (PHNX-3392 usage-sync).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* safe to register everywhere. Single-executor per destination: each daemon only
|
|
9
|
-
* writes the DESTINATION's own cache, and the merge is newest-wins + idempotent.
|
|
4
|
+
* A headed box publishes its local identity-keyed Claude usage rows into its
|
|
5
|
+
* owned file in the fleet-synced user repo. The tick then runs one serialized,
|
|
6
|
+
* timeout-bounded git exchange; a worker reads the delivered peer snapshots
|
|
7
|
+
* and merges newest-wins. No tick opens a device-to-device SSH mesh.
|
|
10
8
|
*/
|
|
11
9
|
import { BasePeriodicService } from './service.js';
|
|
12
10
|
const USAGE_SYNC_TICK_MS = 15 * 60_000;
|
|
@@ -24,19 +22,25 @@ export class UsageSyncService extends BasePeriodicService {
|
|
|
24
22
|
// Nothing to release — the supervisor's timer teardown is the only cleanup.
|
|
25
23
|
}
|
|
26
24
|
async onTick(ctx) {
|
|
27
|
-
const {
|
|
28
|
-
const
|
|
29
|
-
if (
|
|
30
|
-
ctx.log('INFO', `usage-sync:
|
|
25
|
+
const { consumeUsageSnapshotsFromSharedStore, publishUsageSnapshotToSharedStore } = await import('../accounting/usage-sync.js');
|
|
26
|
+
const published = publishUsageSnapshotToSharedStore();
|
|
27
|
+
if (published.changed)
|
|
28
|
+
ctx.log('INFO', `usage-sync: published usage snapshot to ${published.path}`);
|
|
29
|
+
if (published.error)
|
|
30
|
+
ctx.log('WARN', `usage-sync: publish: ${published.error}`);
|
|
31
|
+
const { syncFleetSharedStateRepo } = await import('../fleet-shared-repo-sync.js');
|
|
32
|
+
const transport = await syncFleetSharedStateRepo();
|
|
33
|
+
if (transport.skipped)
|
|
34
|
+
ctx.log('WARN', `usage-sync: ${transport.skipped}`);
|
|
35
|
+
if (transport.error)
|
|
36
|
+
ctx.log('WARN', `usage-sync: shared-store transport: ${transport.error}`);
|
|
37
|
+
if (!transport.success)
|
|
38
|
+
return;
|
|
39
|
+
const consumed = consumeUsageSnapshotsFromSharedStore();
|
|
40
|
+
if (consumed.merged > 0) {
|
|
41
|
+
ctx.log('INFO', `usage-sync: merged ${consumed.merged} row(s) from ${consumed.sources.join(', ')}`);
|
|
31
42
|
}
|
|
32
|
-
for (const err of
|
|
43
|
+
for (const err of consumed.errors)
|
|
33
44
|
ctx.log('WARN', `usage-sync: ${err.device}: ${err.message}`);
|
|
34
|
-
}
|
|
35
|
-
const pullResult = pullUsageFromPrimary();
|
|
36
|
-
if (pullResult.pulledFrom) {
|
|
37
|
-
ctx.log('INFO', `usage-sync: pulled usage from ${pullResult.pulledFrom}; merged ${pullResult.merged} row(s)`);
|
|
38
|
-
}
|
|
39
|
-
if (pullResult.error)
|
|
40
|
-
ctx.log('WARN', `usage-sync: pull: ${pullResult.error}`);
|
|
41
45
|
}
|
|
42
46
|
}
|