@phnx-labs/agents-cli 1.22.69 → 1.22.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +34 -9
  3. package/dist/bootstrap.js +4 -4
  4. package/dist/commands/accounts.js +23 -7
  5. package/dist/commands/exec.js +2 -2
  6. package/dist/commands/import.js +2 -2
  7. package/dist/commands/models.js +2 -2
  8. package/dist/commands/permissions.js +2 -2
  9. package/dist/commands/repo.js +2 -2
  10. package/dist/commands/rules.js +1 -1
  11. package/dist/commands/send.js +9 -3
  12. package/dist/commands/sessions-export.d.ts +5 -1
  13. package/dist/commands/sessions-export.js +100 -24
  14. package/dist/commands/sessions-import.d.ts +2 -1
  15. package/dist/commands/sessions-import.js +85 -21
  16. package/dist/commands/traces.js +1 -1
  17. package/dist/lib/account-capabilities.js +0 -2
  18. package/dist/lib/account-registry.d.ts +3 -3
  19. package/dist/lib/account-registry.js +25 -7
  20. package/dist/lib/accounting/usage-sync.d.ts +1 -1
  21. package/dist/lib/accounting/usage-sync.js +3 -3
  22. package/dist/lib/acp/client.d.ts +1 -1
  23. package/dist/lib/acp/client.js +12 -1
  24. package/dist/lib/acp/harnesses.js +1 -1
  25. package/dist/lib/add-dir.js +0 -2
  26. package/dist/lib/agent-cli-commands.js +0 -2
  27. package/dist/lib/agent-spec/agents.d.ts +1 -1
  28. package/dist/lib/agent-spec/agents.js +2 -83
  29. package/dist/lib/browser/ipc.d.ts +34 -0
  30. package/dist/lib/browser/ipc.js +149 -20
  31. package/dist/lib/browser/remote-control.d.ts +6 -3
  32. package/dist/lib/browser/remote-control.js +6 -3
  33. package/dist/lib/browser/service.d.ts +4 -1
  34. package/dist/lib/browser/service.js +7 -1
  35. package/dist/lib/browser/types.d.ts +3 -1
  36. package/dist/lib/channels/registry.d.ts +6 -0
  37. package/dist/lib/channels/send.d.ts +11 -2
  38. package/dist/lib/channels/send.js +11 -2
  39. package/dist/lib/cloud/rush.d.ts +10 -2
  40. package/dist/lib/cloud/rush.js +15 -9
  41. package/dist/lib/daemon/auth-sync-service.js +1 -1
  42. package/dist/lib/daemon/browser-task-reap-service.js +1 -1
  43. package/dist/lib/daemon/daemon.js +41 -11
  44. package/dist/lib/daemon/heartbeat-service.js +3 -3
  45. package/dist/lib/daemon/keychain-reap-service.js +1 -1
  46. package/dist/lib/daemon/runner.d.ts +18 -1
  47. package/dist/lib/daemon/runner.js +237 -80
  48. package/dist/lib/daemon/self-heal-service.js +13 -3
  49. package/dist/lib/daemon/self-update-service.d.ts +174 -0
  50. package/dist/lib/daemon/self-update-service.js +353 -0
  51. package/dist/lib/daemon/state-dir-check-service.js +3 -3
  52. package/dist/lib/daemon/usage-sync-service.js +1 -1
  53. package/dist/lib/daemon/watchdog-service.js +4 -4
  54. package/dist/lib/daemon-services.d.ts +1 -1
  55. package/dist/lib/daemon-services.js +5 -0
  56. package/dist/lib/device-config.d.ts +12 -1
  57. package/dist/lib/device-config.js +63 -13
  58. package/dist/lib/exec-bounded.d.ts +52 -0
  59. package/dist/lib/exec-bounded.js +113 -0
  60. package/dist/lib/exec.d.ts +2 -2
  61. package/dist/lib/exec.js +3 -34
  62. package/dist/lib/feed/events.d.ts +22 -14
  63. package/dist/lib/feed/events.js +84 -44
  64. package/dist/lib/feed-broadcast.d.ts +12 -29
  65. package/dist/lib/feed-broadcast.js +28 -27
  66. package/dist/lib/fleet-shared-state.d.ts +12 -5
  67. package/dist/lib/fleet-shared-state.js +50 -20
  68. package/dist/lib/fs-atomic.d.ts +11 -0
  69. package/dist/lib/fs-atomic.js +60 -0
  70. package/dist/lib/hooks/install.js +0 -87
  71. package/dist/lib/hosts/reconcile.d.ts +11 -4
  72. package/dist/lib/hosts/reconcile.js +31 -5
  73. package/dist/lib/installations/strategies.js +1 -1
  74. package/dist/lib/mcp-registry.js +0 -13
  75. package/dist/lib/mcp.js +2 -2
  76. package/dist/lib/model-tiers.js +1 -1
  77. package/dist/lib/models.js +0 -63
  78. package/dist/lib/notify.d.ts +11 -0
  79. package/dist/lib/notify.js +17 -4
  80. package/dist/lib/owner-message.d.ts +46 -3
  81. package/dist/lib/owner-message.js +26 -6
  82. package/dist/lib/permissions-registry.d.ts +0 -2
  83. package/dist/lib/permissions-registry.js +3 -50
  84. package/dist/lib/permissions.d.ts +3 -17
  85. package/dist/lib/permissions.js +4 -73
  86. package/dist/lib/project-resources.d.ts +12 -0
  87. package/dist/lib/project-resources.js +129 -0
  88. package/dist/lib/routine-process-cleanup.d.ts +2 -2
  89. package/dist/lib/routine-process-cleanup.js +45 -34
  90. package/dist/lib/rush-session.d.ts +19 -0
  91. package/dist/lib/rush-session.js +24 -0
  92. package/dist/lib/secrets/drivers/rush.js +2 -1
  93. package/dist/lib/secrets/reaper.d.ts +2 -2
  94. package/dist/lib/secrets/reaper.js +13 -10
  95. package/dist/lib/secrets/reserved-sync.d.ts +1 -1
  96. package/dist/lib/secrets/reserved-sync.js +4 -4
  97. package/dist/lib/self-update.d.ts +21 -8
  98. package/dist/lib/self-update.js +54 -31
  99. package/dist/lib/session/cloud.js +2 -1
  100. package/dist/lib/session/sync/backend.d.ts +61 -0
  101. package/dist/lib/session/sync/backend.js +89 -0
  102. package/dist/lib/session/sync/managed-config.d.ts +29 -0
  103. package/dist/lib/session/sync/managed-config.js +23 -0
  104. package/dist/lib/session/sync/managed-key.d.ts +45 -0
  105. package/dist/lib/session/sync/managed-key.js +128 -0
  106. package/dist/lib/session/sync/net-client.d.ts +65 -0
  107. package/dist/lib/session/sync/net-client.js +117 -0
  108. package/dist/lib/session/sync/provision.d.ts +19 -0
  109. package/dist/lib/session/sync/provision.js +38 -0
  110. package/dist/lib/session/sync/r2.d.ts +5 -2
  111. package/dist/lib/session/sync/r2.js +5 -2
  112. package/dist/lib/session/sync/worker-template.d.ts +6 -0
  113. package/dist/lib/session/sync/worker-template.js +847 -0
  114. package/dist/lib/sink-format.d.ts +34 -0
  115. package/dist/lib/sink-format.js +17 -0
  116. package/dist/lib/staleness/detectors/permissions.js +0 -20
  117. package/dist/lib/staleness/writers/commands.js +1 -1
  118. package/dist/lib/staleness/writers/hooks.js +2 -2
  119. package/dist/lib/subagents-registry.js +2 -12
  120. package/dist/lib/subagents.d.ts +0 -10
  121. package/dist/lib/subagents.js +0 -37
  122. package/dist/lib/tmux/orphan-reap.js +6 -4
  123. package/dist/lib/tmux/session.js +4 -1
  124. package/dist/lib/traces/classify.d.ts +8 -1
  125. package/dist/lib/traces/insights.d.ts +13 -1
  126. package/dist/lib/traces/insights.js +78 -3
  127. package/dist/lib/traces/sync.js +8 -3
  128. package/dist/lib/traces/worker-template.js +9 -5
  129. package/dist/lib/types.d.ts +1 -1
  130. package/package.json +1 -1
@@ -10,6 +10,7 @@ import * as os from 'os';
10
10
  import * as crypto from 'crypto';
11
11
  import * as yaml from 'yaml';
12
12
  import { resolveDispatchRepos, normalizeProviderStatus, MAX_IMAGES_PER_DISPATCH } from './types.js';
13
+ import { isRushSessionExpired } from '../rush-session.js';
13
14
  import { parseSSE } from './stream.js';
14
15
  import { listInstalledVersions, getVersionHomePath } from '../installations/versions.js';
15
16
  import { getAccountInfo } from '../agents.js';
@@ -19,8 +20,9 @@ const USER_YAML = path.join(os.homedir(), '.rush', 'user.yaml');
19
20
  /**
20
21
  * Returns true when ~/.rush/user.yaml exists, carries an access_token, and
21
22
  * the token has not passed its expires_at timestamp (Unix seconds). A missing
22
- * expires_at is treated as non-expired so tokens written without an expiry
23
- * still work. Pass yamlPath to override the default path in tests.
23
+ * expires_at, or `expires_at: 0` (a non-expiring Phoenix `pid_` bearer), is
24
+ * treated as non-expired (see isRushSessionExpired, PHNX-3645). Pass yamlPath
25
+ * to override the default path in tests.
24
26
  */
25
27
  export function isRushSessionValid(yamlPath = USER_YAML) {
26
28
  try {
@@ -30,8 +32,7 @@ export function isRushSessionValid(yamlPath = USER_YAML) {
30
32
  const data = yaml.parse(raw);
31
33
  if (!data?.session?.access_token)
32
34
  return false;
33
- const expiresAt = data.session.expires_at;
34
- if (typeof expiresAt === 'number' && expiresAt <= Date.now() / 1000)
35
+ if (isRushSessionExpired(data.session.expires_at))
35
36
  return false;
36
37
  return true;
37
38
  }
@@ -39,19 +40,24 @@ export function isRushSessionValid(yamlPath = USER_YAML) {
39
40
  return false;
40
41
  }
41
42
  }
42
- /** Read the Rush session access token from ~/.rush/user.yaml. */
43
- function readToken() {
44
- if (!fs.existsSync(USER_YAML)) {
43
+ /**
44
+ * Read the Rush session access token from ~/.rush/user.yaml. Exported (with an
45
+ * overridable yamlPath, like isRushSessionValid) so the freshness behavior —
46
+ * including the `expires_at: 0` non-expiring case (PHNX-3645) — is directly
47
+ * testable; the class methods call it with the default path.
48
+ */
49
+ export function readToken(yamlPath = USER_YAML) {
50
+ if (!fs.existsSync(yamlPath)) {
45
51
  throw new Error('Not logged in to Rush. Run `rush login` first.');
46
52
  }
47
- const raw = fs.readFileSync(USER_YAML, 'utf-8');
53
+ const raw = fs.readFileSync(yamlPath, 'utf-8');
48
54
  const data = yaml.parse(raw);
49
55
  const token = data?.session?.access_token;
50
56
  if (!token) {
51
57
  throw new Error('No session token in ~/.rush/user.yaml. Run `rush login` first.');
52
58
  }
53
59
  const expiresAt = data.session?.expires_at;
54
- if (typeof expiresAt === 'number' && expiresAt <= Date.now() / 1000) {
60
+ if (isRushSessionExpired(expiresAt)) {
55
61
  const expiredAt = new Date(expiresAt * 1000).toISOString();
56
62
  throw new Error(`Rush session expired at ${expiredAt}. Run \`rush login\` to refresh.`);
57
63
  }
@@ -23,7 +23,7 @@ export class AuthSyncService extends BasePeriodicService {
23
23
  }
24
24
  async onTick(ctx) {
25
25
  const { publishReservedAuthVerdict, syncReservedAuthBundle } = await import('../secrets/reserved-sync.js');
26
- const published = publishReservedAuthVerdict();
26
+ const published = await publishReservedAuthVerdict();
27
27
  if (published.error)
28
28
  ctx.log('WARN', `auth-sync: verdict: ${published.error}`);
29
29
  const { syncFleetSharedStateRepo } = await import('../fleet-shared-repo-sync.js');
@@ -15,7 +15,7 @@ export class BrowserTaskReapService extends BasePeriodicService {
15
15
  async onStart(_ctx) { }
16
16
  async onStop() { }
17
17
  async onTick(ctx) {
18
- const result = await this.browserService.reapAbandoned({ idleMs: resolveBrowserTaskIdleMs() });
18
+ const result = await this.browserService.reapAbandoned({ idleMs: await resolveBrowserTaskIdleMs() });
19
19
  if (result.closed.length > 0) {
20
20
  ctx.log('INFO', `Browser-task reaper: closed ${result.closed.length} task(s)`);
21
21
  for (const closed of result.closed) {
@@ -40,6 +40,7 @@ import { BrowserIPCService } from './browser-ipc-service.js';
40
40
  import { WatchdogService } from './watchdog-service.js';
41
41
  import { DeviceProbeService } from './device-probe-service.js';
42
42
  import { SelfHealService } from './self-heal-service.js';
43
+ import { SelfUpdateService } from './self-update-service.js';
43
44
  import { KeychainReapService } from './keychain-reap-service.js';
44
45
  import { AuthSyncService } from './auth-sync-service.js';
45
46
  import { UsageSyncService } from './usage-sync-service.js';
@@ -49,7 +50,7 @@ import { WebhookReceiverService } from './webhook-receiver-service.js';
49
50
  import { HeartbeatService } from './heartbeat-service.js';
50
51
  import { TmuxReapService } from './tmux-reap-service.js';
51
52
  import { BrowserTaskReapService } from './browser-task-reap-service.js';
52
- import { emit, emitRoutineEnd } from '../feed/events.js';
53
+ import { emit, emitAsync, emitRoutineEnd } from '../feed/events.js';
53
54
  import { readDaemonServicesConfig, drainDaemonServiceRestartQueue } from '../daemon-services.js';
54
55
  import { sleepSync } from '../fs-atomic.js';
55
56
  /**
@@ -749,11 +750,16 @@ export function log(level, message) {
749
750
  : lvl === 'START' || /starting|started/i.test(message) ? 'daemon.start'
750
751
  : lvl === 'STOP' || /stopping|stopped|shutting down/i.test(message) ? 'daemon.stop'
751
752
  : 'daemon.info';
752
- emit(event, {
753
+ // Fire-and-forget the event mirror: `log()` is a synchronous primitive on
754
+ // every daemon tick's `ctx.log`, and the mirror's event-log lock would
755
+ // otherwise block the shared event loop for up to 30s under contention
756
+ // (PHNX-3695). The daemon-log append above is the primary, synchronous sink;
757
+ // the mirror is best-effort, so a fire-and-forget async write is correct.
758
+ void emitAsync(event, {
753
759
  module: 'daemon',
754
760
  detail: redactSecrets(message).slice(0, 500),
755
761
  status: lvl,
756
- });
762
+ }).catch(() => { });
757
763
  }
758
764
  catch { /* never crash the daemon on event-log failure */ }
759
765
  }
@@ -1073,6 +1079,10 @@ export async function runDaemon() {
1073
1079
  supervisor.register(new SelfHealService());
1074
1080
  else
1075
1081
  log('INFO', 'Self-heal service disabled');
1082
+ if (isEnabled('self-update'))
1083
+ supervisor.register(new SelfUpdateService());
1084
+ else
1085
+ log('INFO', 'Self-update service disabled');
1076
1086
  if (isEnabled('keychain-reap'))
1077
1087
  supervisor.register(new KeychainReapService());
1078
1088
  else
@@ -2249,7 +2259,15 @@ function stopDaemonLocked() {
2249
2259
  // any service-manager teardown or direct signal and repairs only under lock.
2250
2260
  const pid = resolveLiveDaemonPid(true);
2251
2261
  const browserSock = process.platform === 'win32' ? null : getBrowserIpcSocketPath();
2252
- const browserSockOwner = pid !== null && browserSock ? readPathIdentity(browserSock) : null;
2262
+ // Capture the socket's inode INDEPENDENTLY of whether a live pid resolved
2263
+ // (PHNX-3618). A daemon that died between the CLI liveness precheck and this
2264
+ // locked read leaves resolveLiveDaemonPid() returning null while its ungraceful
2265
+ // exit left the binding on disk — capturing only when `pid !== null` meant that
2266
+ // genuinely stale socket could never be reclaimed and was reported "ownership
2267
+ // could not be verified". The inode is the successor guard: a fresh daemon that
2268
+ // rebinds during the stop gets a new inode, so a later identity match still
2269
+ // proves this exact binding is the one we captured, never a successor's.
2270
+ const browserSockOwner = browserSock ? readPathIdentity(browserSock) : null;
2253
2271
  if (platform === 'darwin') {
2254
2272
  const plistPath = getLaunchdPlistPath();
2255
2273
  if (fs.existsSync(plistPath)) {
@@ -2380,8 +2398,19 @@ function stopDaemonLocked() {
2380
2398
  // ungracefully (killTree) and left a stale binding — the owner is provably
2381
2399
  // dead, so reclaim it and report.
2382
2400
  if (process.platform !== 'win32') {
2401
+ // A binding is provably stale — and reclaimable — when the inode on disk is
2402
+ // still the exact one captured under this lock AND no daemon (the signalled
2403
+ // target OR any successor for this state dir) survives to own it. Keying the
2404
+ // "a daemon still owns it" test on the whole survivor set rather than only the
2405
+ // captured target is what lets a daemon that died BEFORE this stop (pid ===
2406
+ // null, PHNX-3618) still have its stale socket reclaimed, while a live
2407
+ // successor — which either rebound the inode or shows up as a survivor — is
2408
+ // never touched (PHNX-3607's never-delete-a-successor invariant).
2409
+ const aDaemonSurvives = survivors.length > 0 || unverifiedSurvivors.length > 0;
2383
2410
  if (browserSock && fs.existsSync(browserSock)) {
2384
- if (browserSockOwner && pathIdentityMatches(browserSock, browserSockOwner) && !targetStillOwnsResources) {
2411
+ const ownsCapturedBinding = browserSockOwner !== null
2412
+ && pathIdentityMatches(browserSock, browserSockOwner);
2413
+ if (ownsCapturedBinding && !aDaemonSurvives) {
2385
2414
  try {
2386
2415
  fs.unlinkSync(browserSock);
2387
2416
  }
@@ -2393,13 +2422,14 @@ function stopDaemonLocked() {
2393
2422
  else
2394
2423
  released.push('browser IPC socket (reclaimed)');
2395
2424
  }
2425
+ else if (aDaemonSurvives) {
2426
+ surviving.push('browser IPC socket still owned by a surviving daemon');
2427
+ }
2396
2428
  else {
2397
- // The path was absent when the target was captured, there was no proven
2398
- // daemon target, or a successor replaced the inode. In every case this
2399
- // invocation does not own the current binding and must leave it alone.
2400
- surviving.push(targetStillOwnsResources
2401
- ? 'browser IPC socket still owned by a surviving daemon'
2402
- : 'browser IPC socket ownership could not be verified');
2429
+ // The path was absent when this transaction captured the target (a
2430
+ // successor bound it afterward) or a successor replaced the inode. Either
2431
+ // way this invocation does not own the current binding and must leave it.
2432
+ surviving.push('browser IPC socket ownership could not be verified');
2403
2433
  }
2404
2434
  }
2405
2435
  else {
@@ -1,5 +1,5 @@
1
1
  /** Daemon heartbeat and routine-process reconciliation under supervision. */
2
- import { monitorRunningJobs } from './runner.js';
2
+ import { reapExitedRunningJobs } from './runner.js';
3
3
  import { reapTerminalRoutineProcesses } from '../routine-process-cleanup.js';
4
4
  import { BasePeriodicService } from './service.js';
5
5
  const HEARTBEAT_TICK_MS = 60_000;
@@ -17,8 +17,8 @@ export class HeartbeatService extends BasePeriodicService {
17
17
  async onStop() { }
18
18
  async onTick(ctx) {
19
19
  this.publishHeartbeat();
20
- monitorRunningJobs();
21
- const reaped = reapTerminalRoutineProcesses();
20
+ await reapExitedRunningJobs();
21
+ const reaped = await reapTerminalRoutineProcesses();
22
22
  if (reaped.length > 0) {
23
23
  ctx.log('WARN', `Reaped ${reaped.length} terminal routine process group(s): ${reaped.join(', ')}`);
24
24
  }
@@ -22,7 +22,7 @@ export class KeychainReapService extends BasePeriodicService {
22
22
  }
23
23
  async onTick(ctx) {
24
24
  const { reapOrphanedKeychainProcesses } = await import('../secrets/reaper.js');
25
- const result = reapOrphanedKeychainProcesses();
25
+ const result = await reapOrphanedKeychainProcesses();
26
26
  if (result.reaped > 0) {
27
27
  ctx.log('WARN', `Reaped ${result.reaped} keychain orphan/stuck process(es)`);
28
28
  for (const d of result.details)
@@ -251,6 +251,23 @@ export declare function listLiveRoutineChildren(): number[];
251
251
  * remote `.exit` is reconciled (`finalizeHostRun`).
252
252
  */
253
253
  export declare function isRunGenuinelyInFlight(meta: RunMeta): boolean;
254
- /** Scan all runs marked "running" and finalize any whose process has exited. */
254
+ /**
255
+ * Scan all runs marked "running" and finalize any whose process has exited.
256
+ *
257
+ * SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
258
+ * builders and CLI best-effort orphan reaps), which run in short-lived CLI
259
+ * processes where a bounded synchronous `ps`/scan cannot starve a served event
260
+ * loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
261
+ * instead — a sync scan there would freeze the shared event loop (PHNX-3695).
262
+ */
255
263
  export declare function monitorRunningJobs(): void;
264
+ /**
265
+ * Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
266
+ * heartbeat tick (PHNX-3695). Identical reconciliation via
267
+ * {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
268
+ * the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
269
+ * the every-tick scan never freezes the shared event loop — the freeze that
270
+ * left the browser IPC `version` probe unanswered.
271
+ */
272
+ export declare function reapExitedRunningJobs(): Promise<void>;
256
273
  export {};
@@ -15,6 +15,7 @@
15
15
  */
16
16
  import { spawn, execFileSync } from 'child_process';
17
17
  import * as fs from 'fs';
18
+ import * as fsp from 'fs/promises';
18
19
  import * as path from 'path';
19
20
  import * as os from 'os';
20
21
  import { getCliLaunch, getAgentsBinDir } from '../cli-entry.js';
@@ -23,14 +24,15 @@ import { getRunsDir, getUserAgentsDir, readMeta, getDaemonDir } from '../state.j
23
24
  import { shortCodexHome } from '../codex-home.js';
24
25
  import { prepareJobHome, buildSpawnEnv, getJobHomePath, assertSandboxForwardsHostGhAuth, linkVersionAuth } from '../sandbox.js';
25
26
  import { resolveModel, buildReasoningFlags } from '../models.js';
26
- import { createTimer, redactPrompt, emitRoutineEnd } from '../feed/events.js';
27
+ import { createTimer, redactPrompt, emitRoutineEnd, emitRoutineEndAsync } from '../feed/events.js';
27
28
  import { resolveHarnessAdapter } from '../harness/index.js';
28
29
  import { applyAddDirs } from '../add-dir.js';
29
30
  import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, AGENT_COMMANDS, } from '../exec.js';
30
31
  import { resolveActor } from '../actor.js';
31
32
  import { loadTask as loadHostTask } from '../hosts/tasks.js';
32
- import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
33
- import { backgroundSpawnOptions, killTree } from '../platform/process.js';
33
+ import { reconcileTask as reconcileHostTask, reconcileTaskAsync as reconcileHostTaskAsync } from '../hosts/reconcile.js';
34
+ import { backgroundSpawnOptions, isAlive, killTree } from '../platform/process.js';
35
+ import { execFileBounded } from '../exec-bounded.js';
34
36
  import lockfile from 'proper-lockfile';
35
37
  import { ensureLockTarget } from '../fs-atomic.js';
36
38
  import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
@@ -922,8 +924,12 @@ export async function resolveRoutineLaunch(config, cwd = process.cwd(), deps = {
922
924
  if (config.account && !explicitCredential) {
923
925
  // A native routine account is named by its durable name; the version matcher
924
926
  // keys on the identity (email/accountKey), so translate before resolving,
925
- // and refuse a login that belongs to a different harness.
926
- const unified = findUnifiedAccount(config.account, meta);
927
+ // and refuse a login that belongs to a different harness. Scope the lookup to
928
+ // the routine's own harness: `identityLabel` defaults to the login's email, so
929
+ // `account: <email>` matches every harness that identity is signed into, and
930
+ // un-scoped this resolved whichever row the store ordered first — then rejected
931
+ // it on the very next line.
932
+ const unified = findUnifiedAccount(config.account, meta, undefined, agent);
927
933
  if (unified?.kind === 'native' && unified.agent !== agent) {
928
934
  throw new Error(`Routine '${config.name}' account '${config.account}' is a ${unified.agent} login and cannot authenticate ${agent}.`);
929
935
  }
@@ -2224,6 +2230,40 @@ const MAX_WALL_CLOCK_MS = 24 * 60 * 60 * 1000;
2224
2230
  * compares against the process's actual start time via `ps`. Returns true
2225
2231
  * when the PID is alive AND plausibly ours.
2226
2232
  */
2233
+ /** Bound the identity `ps` — a hung `ps` must never freeze its caller for long (matches routine-process-cleanup's probe). */
2234
+ const PS_IDENTITY_TIMEOUT_MS = 5_000;
2235
+ /**
2236
+ * Pure: does a `ps -o etime=` value place the process's birth within 30s of when
2237
+ * we recorded spawning it? An empty/unknown reading conservatively reads as ours
2238
+ * — never reap a run on a doubtful identity probe.
2239
+ */
2240
+ function etimeIndicatesOurs(etime, spawnedAt) {
2241
+ if (!etime)
2242
+ return true;
2243
+ const parts = etime.replace(/-/g, ':').split(':').reverse();
2244
+ let uptimeSec = 0;
2245
+ if (parts[0])
2246
+ uptimeSec += parseInt(parts[0], 10);
2247
+ if (parts[1])
2248
+ uptimeSec += parseInt(parts[1], 10) * 60;
2249
+ if (parts[2])
2250
+ uptimeSec += parseInt(parts[2], 10) * 3600;
2251
+ if (parts[3])
2252
+ uptimeSec += parseInt(parts[3], 10) * 86400;
2253
+ const processStartMs = Date.now() - uptimeSec * 1000;
2254
+ return Math.abs(processStartMs - spawnedAt) < 30_000;
2255
+ }
2256
+ /**
2257
+ * Synchronous identity check for callers OTHER than the periodic heartbeat tick:
2258
+ * the `agents routines list/status` builders and stop-time enumeration (both in
2259
+ * short-lived CLI processes), plus the scheduler's fire-time slot-claim
2260
+ * (`activeRoutineRun`) and status projection (`isRunGenuinelyInFlight`). Those
2261
+ * last two DO run in-process on the daemon's event loop, but are event-driven
2262
+ * (they fire when a job is dispatched, not on a fixed cadence) and the `ps` here
2263
+ * is now bounded to {@link PS_IDENTITY_TIMEOUT_MS} — they are not the
2264
+ * unconditional every-tick scan PHNX-3695 targets. The heartbeat tick's own scan
2265
+ * uses the non-blocking {@link isPidOursAsync} instead.
2266
+ */
2227
2267
  function isPidOurs(pid, spawnedAt) {
2228
2268
  try {
2229
2269
  process.kill(pid, 0);
@@ -2236,42 +2276,75 @@ function isPidOurs(pid, spawnedAt) {
2236
2276
  if (process.platform === 'win32')
2237
2277
  return true;
2238
2278
  try {
2239
- const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
2240
- if (!etime)
2241
- return true;
2242
- const parts = etime.replace(/-/g, ':').split(':').reverse();
2243
- let uptimeSec = 0;
2244
- if (parts[0])
2245
- uptimeSec += parseInt(parts[0], 10);
2246
- if (parts[1])
2247
- uptimeSec += parseInt(parts[1], 10) * 60;
2248
- if (parts[2])
2249
- uptimeSec += parseInt(parts[2], 10) * 3600;
2250
- if (parts[3])
2251
- uptimeSec += parseInt(parts[3], 10) * 86400;
2252
- const processStartMs = Date.now() - uptimeSec * 1000;
2253
- return Math.abs(processStartMs - spawnedAt) < 30_000;
2279
+ const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: PS_IDENTITY_TIMEOUT_MS }).trim();
2280
+ return etimeIndicatesOurs(etime, spawnedAt);
2254
2281
  }
2255
2282
  catch {
2256
2283
  return true;
2257
2284
  }
2258
2285
  }
2286
+ /**
2287
+ * Async, deadline-bounded twin of {@link isPidOurs} for the daemon's heartbeat
2288
+ * tick (`reapExitedRunningJobs`) — the `ps` probe runs on the shared event loop,
2289
+ * so it MUST NOT block it (PHNX-3695). Same semantics: dead pid → not ours;
2290
+ * no recorded `spawnedAt` or Windows → assume ours; a failed/empty probe reads
2291
+ * as ours (conservative — never reap on a doubtful probe).
2292
+ */
2293
+ async function isPidOursAsync(pid, spawnedAt) {
2294
+ if (!isAlive(pid))
2295
+ return false;
2296
+ if (spawnedAt === undefined)
2297
+ return true;
2298
+ if (process.platform === 'win32')
2299
+ return true;
2300
+ const res = await execFileBounded('ps', ['-p', String(pid), '-o', 'etime='], { timeoutMs: PS_IDENTITY_TIMEOUT_MS });
2301
+ if (res.code !== 0)
2302
+ return true; // probe failed → keep the run, same as the sync catch
2303
+ return etimeIndicatesOurs(res.stdout.trim(), spawnedAt);
2304
+ }
2259
2305
  /**
2260
2306
  * Finalize one `host:`-placed run by healing its host-task sidecar against the
2261
2307
  * remote `.exit` (lib/hosts/reconcile.ts). Mutates + persists the meta only
2262
2308
  * when the sidecar reached a terminal state.
2263
2309
  */
2310
+ /** Apply a healed host-task's terminal state to the local run record. Shared by the sync and async host reconcilers. */
2311
+ function applyHealedHostRun(meta, healed,
2312
+ // The routine-end emit acquires the event-log file lock. On the daemon
2313
+ // heartbeat tick (finalizeHostRunAsync) that MUST be the async, non-blocking
2314
+ // variant, or the synchronous `withFileLock` (lockSync + Atomics.wait, up to
2315
+ // 30s under contention) freezes the whole event loop — the exact wedge PHNX-3695
2316
+ // targets, on a path guard-no-sync-io does not scan (PHNX-3727). The sync CLI
2317
+ // path (finalizeHostRun) keeps the default synchronous emit.
2318
+ emit = emitRoutineEnd) {
2319
+ if (healed.status !== 'completed' && healed.status !== 'failed')
2320
+ return;
2321
+ finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
2322
+ writeRunMeta(meta);
2323
+ emit(meta);
2324
+ }
2264
2325
  function finalizeHostRun(meta) {
2265
2326
  try {
2266
2327
  const task = loadHostTask(meta.hostTaskId);
2267
2328
  if (!task)
2268
2329
  return;
2269
- const healed = reconcileHostTask(task);
2270
- if (healed.status !== 'completed' && healed.status !== 'failed')
2330
+ applyHealedHostRun(meta, reconcileHostTask(task));
2331
+ }
2332
+ catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
2333
+ }
2334
+ /**
2335
+ * Async twin of {@link finalizeHostRun} for the daemon heartbeat tick
2336
+ * (PHNX-3695). A `host:`-placed run's remote `.exit` is read over ssh; on the
2337
+ * tick that read MUST be async (`reconcileHostTaskAsync` → `sshExecAsync`) or a
2338
+ * single 6s ssh timeout freezes the whole event loop — the exact bug class this
2339
+ * effort targets, worse than the local `ps` probe.
2340
+ */
2341
+ async function finalizeHostRunAsync(meta) {
2342
+ try {
2343
+ const task = loadHostTask(meta.hostTaskId);
2344
+ if (!task)
2271
2345
  return;
2272
- finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
2273
- writeRunMeta(meta);
2274
- emitRoutineEnd(meta);
2346
+ // Tick path: emit through the async, non-blocking file lock (PHNX-3727).
2347
+ applyHealedHostRun(meta, await reconcileHostTaskAsync(task), (m) => { void emitRoutineEndAsync(m); });
2275
2348
  }
2276
2349
  catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
2277
2350
  }
@@ -2345,7 +2418,91 @@ export function isRunGenuinelyInFlight(meta) {
2345
2418
  return false;
2346
2419
  return isPidOurs(meta.pid, meta.spawnedAt);
2347
2420
  }
2348
- /** Scan all runs marked "running" and finalize any whose process has exited. */
2421
+ /**
2422
+ * Reconcile ONE `running` record once its process identity is known (`ours` =
2423
+ * the recorded pid is still genuinely the child we spawned). Shared verbatim by
2424
+ * the synchronous {@link monitorRunningJobs} (off-loop callers) and the async
2425
+ * {@link reapExitedRunningJobs} (daemon heartbeat tick) so there is one copy of
2426
+ * the finalize/timeout/report logic — only the surrounding directory/`ps` IO
2427
+ * differs by sync-vs-async flavor. Caller has already confirmed `meta.status`
2428
+ * is `running`. The finalize/report/archive helpers stay synchronous: they run
2429
+ * ONLY when a run actually transitions (times out or its process exited), a
2430
+ * conditional/rare event, not the every-tick scan the loop-starvation fix
2431
+ * (PHNX-3695) targets. The one exception is the `routine.end` EVENT emit, whose
2432
+ * file lock CAN block for up to 30s under contention even on that rare path — so
2433
+ * the emitter is injected: the off-loop sync caller passes `emitRoutineEnd`
2434
+ * (synchronous, so a short-lived CLI process flushes it before exit), the daemon
2435
+ * tick passes a fire-and-forget `emitRoutineEndAsync` (the long-lived daemon
2436
+ * flushes it, and the shared event loop is never blocked on the lock).
2437
+ */
2438
+ function reconcileRunningRecord(meta, jobRunsPath, runDirName, ours, emitEnd) {
2439
+ // Host-placed runs (meta.hostTaskId) are handled by the caller, sync or async
2440
+ // — their remote `.exit` read is ssh I/O that must stay off the daemon tick's
2441
+ // event loop (finalizeHostRunAsync). Records reaching here are local-pid runs.
2442
+ const runDirPath = path.join(jobRunsPath, runDirName);
2443
+ const stdoutPath = path.join(runDirPath, 'stdout.log');
2444
+ // Command-mode records carry no agent; there is no stream-json report to
2445
+ // parse or extract. Reap them on pid liveness alone.
2446
+ const isCommandRun = Boolean(meta.command) || !meta.agent;
2447
+ // Age-out runs FIRST, before the null-pid guard below, so a wedged record
2448
+ // still marked `running` past its deadline is always finalized — a
2449
+ // provisional launcher claim whose daemon crashed before spawning a child
2450
+ // (pid null), or an old record whose recorded pid was reused, would
2451
+ // otherwise linger as `running` forever (RUSH-2640). Only kill a process
2452
+ // group that is genuinely still ours; terminating a reused pid would kill
2453
+ // an unrelated process, and a null pid has nothing to kill.
2454
+ const wallClockMs = Date.now() - Date.parse(meta.startedAt);
2455
+ const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
2456
+ if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
2457
+ if (meta.pid && ours)
2458
+ terminateRoutineTree(meta.pid);
2459
+ finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
2460
+ writeRunMeta(meta);
2461
+ emitEnd(meta);
2462
+ if (!isCommandRun) {
2463
+ extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2464
+ archiveRoutineTranscripts(meta, runDirPath);
2465
+ }
2466
+ return;
2467
+ }
2468
+ if (!meta.pid)
2469
+ return;
2470
+ if (!ours) {
2471
+ if (isCommandRun) {
2472
+ // Command routines normally record their own terminal status via
2473
+ // child.on('exit') (so this record would already be non-'running' and
2474
+ // skipped above). Reaching here means the daemon restarted mid-run and
2475
+ // missed the exit event — recover the true code from the exit-code file
2476
+ // the child wrote; its absence means the child was killed/crashed.
2477
+ const ec = readCommandExitCode(runDirPath);
2478
+ finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
2479
+ }
2480
+ else {
2481
+ const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
2482
+ if (inferred) {
2483
+ finalizeRunMeta(meta, inferred.status, inferred.exitCode);
2484
+ }
2485
+ else {
2486
+ finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
2487
+ }
2488
+ }
2489
+ writeRunMeta(meta);
2490
+ emitEnd(meta);
2491
+ if (!isCommandRun) {
2492
+ extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2493
+ archiveRoutineTranscripts(meta, runDirPath);
2494
+ }
2495
+ }
2496
+ }
2497
+ /**
2498
+ * Scan all runs marked "running" and finalize any whose process has exited.
2499
+ *
2500
+ * SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
2501
+ * builders and CLI best-effort orphan reaps), which run in short-lived CLI
2502
+ * processes where a bounded synchronous `ps`/scan cannot starve a served event
2503
+ * loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
2504
+ * instead — a sync scan there would freeze the shared event loop (PHNX-3695).
2505
+ */
2349
2506
  export function monitorRunningJobs() {
2350
2507
  const runsDir = getRunsDir();
2351
2508
  if (!fs.existsSync(runsDir))
@@ -2374,67 +2531,67 @@ export function monitorRunningJobs() {
2374
2531
  const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
2375
2532
  if (meta.status !== 'running')
2376
2533
  continue;
2377
- // `host:`-placed run — no local pid to watch. Reconcile against the
2378
- // remote `.exit` (completion is confirmed, never guessed: an
2379
- // unreachable host leaves the run `running` for the next sweep).
2534
+ // `host:`-placed run — no local pid; reconcile against the remote `.exit`.
2380
2535
  if (meta.hostTaskId) {
2381
2536
  finalizeHostRun(meta);
2382
2537
  continue;
2383
2538
  }
2384
- const runDirPath = path.join(jobRunsPath, runDirEntry.name);
2385
- const stdoutPath = path.join(runDirPath, 'stdout.log');
2386
- // Command-mode records carry no agent; there is no stream-json report to
2387
- // parse or extract. Reap them on pid liveness alone.
2388
- const isCommandRun = Boolean(meta.command) || !meta.agent;
2389
- // Age-out runs FIRST, before the null-pid guard below, so a wedged record
2390
- // still marked `running` past its deadline is always finalized — a
2391
- // provisional launcher claim whose daemon crashed before spawning a child
2392
- // (pid null), or an old record whose recorded pid was reused, would
2393
- // otherwise linger as `running` forever (RUSH-2640). Only kill a process
2394
- // group that is genuinely still ours; terminating a reused pid would kill
2395
- // an unrelated process, and a null pid has nothing to kill.
2396
- const wallClockMs = Date.now() - Date.parse(meta.startedAt);
2397
- const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
2398
- if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
2399
- if (meta.pid && isPidOurs(meta.pid, meta.spawnedAt))
2400
- terminateRoutineTree(meta.pid);
2401
- finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
2402
- writeRunMeta(meta);
2403
- emitRoutineEnd(meta);
2404
- if (!isCommandRun) {
2405
- extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2406
- archiveRoutineTranscripts(meta, runDirPath);
2407
- }
2539
+ const ours = meta.pid ? isPidOurs(meta.pid, meta.spawnedAt) : false;
2540
+ // Sync emit: an off-loop CLI process must flush the event before it exits.
2541
+ reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, emitRoutineEnd);
2542
+ }
2543
+ catch { /* corrupt or unreadable meta.json */ }
2544
+ }
2545
+ }
2546
+ }
2547
+ /**
2548
+ * Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
2549
+ * heartbeat tick (PHNX-3695). Identical reconciliation via
2550
+ * {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
2551
+ * the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
2552
+ * the every-tick scan never freezes the shared event loop — the freeze that
2553
+ * left the browser IPC `version` probe unanswered.
2554
+ */
2555
+ export async function reapExitedRunningJobs() {
2556
+ const runsDir = getRunsDir();
2557
+ let jobDirs;
2558
+ try {
2559
+ jobDirs = await fsp.readdir(runsDir, { withFileTypes: true });
2560
+ }
2561
+ catch {
2562
+ return; // runs dir absent — nothing to reap
2563
+ }
2564
+ for (const jobDir of jobDirs) {
2565
+ if (!jobDir.isDirectory())
2566
+ continue;
2567
+ const jobRunsPath = path.join(runsDir, jobDir.name);
2568
+ let runDirs;
2569
+ try {
2570
+ runDirs = await fsp.readdir(jobRunsPath, { withFileTypes: true });
2571
+ }
2572
+ catch {
2573
+ // A retention/cleanup pass may remove a job directory after the root
2574
+ // snapshot. The next sweep observes the remaining state.
2575
+ continue;
2576
+ }
2577
+ for (const runDirEntry of runDirs) {
2578
+ if (!runDirEntry.isDirectory())
2579
+ continue;
2580
+ try {
2581
+ const raw = await fsp.readFile(path.join(jobRunsPath, runDirEntry.name, 'meta.json'), 'utf-8');
2582
+ const meta = JSON.parse(raw);
2583
+ if (meta.status !== 'running')
2408
2584
  continue;
2409
- }
2410
- if (!meta.pid)
2585
+ // `host:`-placed run — read its remote `.exit` over ssh ASYNC so a slow
2586
+ // host never freezes the tick's event loop (PHNX-3695).
2587
+ if (meta.hostTaskId) {
2588
+ await finalizeHostRunAsync(meta);
2411
2589
  continue;
2412
- if (!isPidOurs(meta.pid, meta.spawnedAt)) {
2413
- if (isCommandRun) {
2414
- // Command routines normally record their own terminal status via
2415
- // child.on('exit') (so this record would already be non-'running' and
2416
- // skipped above). Reaching here means the daemon restarted mid-run and
2417
- // missed the exit event — recover the true code from the exit-code file
2418
- // the child wrote; its absence means the child was killed/crashed.
2419
- const ec = readCommandExitCode(runDirPath);
2420
- finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
2421
- }
2422
- else {
2423
- const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
2424
- if (inferred) {
2425
- finalizeRunMeta(meta, inferred.status, inferred.exitCode);
2426
- }
2427
- else {
2428
- finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
2429
- }
2430
- }
2431
- writeRunMeta(meta);
2432
- emitRoutineEnd(meta);
2433
- if (!isCommandRun) {
2434
- extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2435
- archiveRoutineTranscripts(meta, runDirPath);
2436
- }
2437
2590
  }
2591
+ const ours = meta.pid ? await isPidOursAsync(meta.pid, meta.spawnedAt) : false;
2592
+ // Fire-and-forget async emit: the event-log lock must never block the
2593
+ // daemon tick's shared event loop; the long-lived daemon still flushes it.
2594
+ reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, (m) => { void emitRoutineEndAsync(m); });
2438
2595
  }
2439
2596
  catch { /* corrupt or unreadable meta.json */ }
2440
2597
  }