@phnx-labs/agents-cli 1.22.70 → 1.22.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +31 -1
  3. package/dist/bootstrap.js +4 -4
  4. package/dist/commands/repo.js +2 -2
  5. package/dist/commands/sessions-export.d.ts +5 -1
  6. package/dist/commands/sessions-export.js +100 -24
  7. package/dist/commands/sessions-import.d.ts +2 -1
  8. package/dist/commands/sessions-import.js +85 -21
  9. package/dist/lib/accounting/usage-sync.d.ts +1 -1
  10. package/dist/lib/accounting/usage-sync.js +3 -3
  11. package/dist/lib/browser/ipc.d.ts +34 -0
  12. package/dist/lib/browser/ipc.js +140 -19
  13. package/dist/lib/browser/types.d.ts +3 -1
  14. package/dist/lib/daemon/auth-sync-service.js +1 -1
  15. package/dist/lib/daemon/browser-task-reap-service.js +1 -1
  16. package/dist/lib/daemon/daemon.js +13 -3
  17. package/dist/lib/daemon/heartbeat-service.js +3 -3
  18. package/dist/lib/daemon/keychain-reap-service.js +1 -1
  19. package/dist/lib/daemon/runner.d.ts +18 -1
  20. package/dist/lib/daemon/runner.js +231 -78
  21. package/dist/lib/daemon/self-heal-service.js +13 -3
  22. package/dist/lib/daemon/self-update-service.d.ts +174 -0
  23. package/dist/lib/daemon/self-update-service.js +353 -0
  24. package/dist/lib/daemon/state-dir-check-service.js +3 -3
  25. package/dist/lib/daemon/usage-sync-service.js +1 -1
  26. package/dist/lib/daemon/watchdog-service.js +4 -4
  27. package/dist/lib/daemon-services.d.ts +1 -1
  28. package/dist/lib/daemon-services.js +5 -0
  29. package/dist/lib/device-config.d.ts +12 -1
  30. package/dist/lib/device-config.js +63 -13
  31. package/dist/lib/exec-bounded.d.ts +52 -0
  32. package/dist/lib/exec-bounded.js +113 -0
  33. package/dist/lib/feed/events.d.ts +22 -14
  34. package/dist/lib/feed/events.js +84 -44
  35. package/dist/lib/fleet-shared-state.d.ts +12 -5
  36. package/dist/lib/fleet-shared-state.js +50 -20
  37. package/dist/lib/fs-atomic.d.ts +11 -0
  38. package/dist/lib/fs-atomic.js +60 -0
  39. package/dist/lib/hosts/reconcile.d.ts +11 -4
  40. package/dist/lib/hosts/reconcile.js +31 -5
  41. package/dist/lib/project-resources.d.ts +12 -0
  42. package/dist/lib/project-resources.js +138 -0
  43. package/dist/lib/routine-process-cleanup.d.ts +2 -2
  44. package/dist/lib/routine-process-cleanup.js +45 -34
  45. package/dist/lib/secrets/reaper.d.ts +2 -2
  46. package/dist/lib/secrets/reaper.js +13 -10
  47. package/dist/lib/secrets/reserved-sync.d.ts +1 -1
  48. package/dist/lib/secrets/reserved-sync.js +4 -4
  49. package/dist/lib/self-update.d.ts +21 -8
  50. package/dist/lib/self-update.js +54 -31
  51. package/dist/lib/session/sync/backend.d.ts +61 -0
  52. package/dist/lib/session/sync/backend.js +89 -0
  53. package/dist/lib/session/sync/managed-config.d.ts +29 -0
  54. package/dist/lib/session/sync/managed-config.js +23 -0
  55. package/dist/lib/session/sync/managed-key.d.ts +45 -0
  56. package/dist/lib/session/sync/managed-key.js +128 -0
  57. package/dist/lib/session/sync/net-client.d.ts +65 -0
  58. package/dist/lib/session/sync/net-client.js +117 -0
  59. package/dist/lib/session/sync/provision.d.ts +19 -0
  60. package/dist/lib/session/sync/provision.js +38 -0
  61. package/dist/lib/session/sync/r2.d.ts +5 -2
  62. package/dist/lib/session/sync/r2.js +5 -2
  63. package/dist/lib/session/sync/worker-template.d.ts +6 -0
  64. package/dist/lib/session/sync/worker-template.js +847 -0
  65. package/dist/lib/tmux/orphan-reap.js +6 -4
  66. package/dist/lib/tmux/session.js +4 -1
  67. package/dist/lib/traces/classify.d.ts +8 -1
  68. package/dist/lib/traces/insights.d.ts +13 -1
  69. package/dist/lib/traces/insights.js +78 -3
  70. package/dist/lib/traces/sync.js +8 -3
  71. package/dist/lib/traces/worker-template.js +9 -5
  72. package/package.json +1 -1
@@ -15,6 +15,7 @@
15
15
  */
16
16
  import { spawn, execFileSync } from 'child_process';
17
17
  import * as fs from 'fs';
18
+ import * as fsp from 'fs/promises';
18
19
  import * as path from 'path';
19
20
  import * as os from 'os';
20
21
  import { getCliLaunch, getAgentsBinDir } from '../cli-entry.js';
@@ -23,14 +24,15 @@ import { getRunsDir, getUserAgentsDir, readMeta, getDaemonDir } from '../state.j
23
24
  import { shortCodexHome } from '../codex-home.js';
24
25
  import { prepareJobHome, buildSpawnEnv, getJobHomePath, assertSandboxForwardsHostGhAuth, linkVersionAuth } from '../sandbox.js';
25
26
  import { resolveModel, buildReasoningFlags } from '../models.js';
26
- import { createTimer, redactPrompt, emitRoutineEnd } from '../feed/events.js';
27
+ import { createTimer, redactPrompt, emitRoutineEnd, emitRoutineEndAsync } from '../feed/events.js';
27
28
  import { resolveHarnessAdapter } from '../harness/index.js';
28
29
  import { applyAddDirs } from '../add-dir.js';
29
30
  import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, AGENT_COMMANDS, } from '../exec.js';
30
31
  import { resolveActor } from '../actor.js';
31
32
  import { loadTask as loadHostTask } from '../hosts/tasks.js';
32
- import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
33
- import { backgroundSpawnOptions, killTree } from '../platform/process.js';
33
+ import { reconcileTask as reconcileHostTask, reconcileTaskAsync as reconcileHostTaskAsync } from '../hosts/reconcile.js';
34
+ import { backgroundSpawnOptions, isAlive, killTree } from '../platform/process.js';
35
+ import { execFileBounded } from '../exec-bounded.js';
34
36
  import lockfile from 'proper-lockfile';
35
37
  import { ensureLockTarget } from '../fs-atomic.js';
36
38
  import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
@@ -2228,6 +2230,40 @@ const MAX_WALL_CLOCK_MS = 24 * 60 * 60 * 1000;
2228
2230
  * compares against the process's actual start time via `ps`. Returns true
2229
2231
  * when the PID is alive AND plausibly ours.
2230
2232
  */
2233
+ /** Bound the identity `ps` — a hung `ps` must never freeze its caller for long (matches routine-process-cleanup's probe). */
2234
+ const PS_IDENTITY_TIMEOUT_MS = 5_000;
2235
+ /**
2236
+ * Pure: does a `ps -o etime=` value place the process's birth within 30s of when
2237
+ * we recorded spawning it? An empty/unknown reading conservatively reads as ours
2238
+ * — never reap a run on a doubtful identity probe.
2239
+ */
2240
+ function etimeIndicatesOurs(etime, spawnedAt) {
2241
+ if (!etime)
2242
+ return true;
2243
+ const parts = etime.replace(/-/g, ':').split(':').reverse();
2244
+ let uptimeSec = 0;
2245
+ if (parts[0])
2246
+ uptimeSec += parseInt(parts[0], 10);
2247
+ if (parts[1])
2248
+ uptimeSec += parseInt(parts[1], 10) * 60;
2249
+ if (parts[2])
2250
+ uptimeSec += parseInt(parts[2], 10) * 3600;
2251
+ if (parts[3])
2252
+ uptimeSec += parseInt(parts[3], 10) * 86400;
2253
+ const processStartMs = Date.now() - uptimeSec * 1000;
2254
+ return Math.abs(processStartMs - spawnedAt) < 30_000;
2255
+ }
2256
+ /**
2257
+ * Synchronous identity check for callers OTHER than the periodic heartbeat tick:
2258
+ * the `agents routines list/status` builders and stop-time enumeration (both in
2259
+ * short-lived CLI processes), plus the scheduler's fire-time slot-claim
2260
+ * (`activeRoutineRun`) and status projection (`isRunGenuinelyInFlight`). Those
2261
+ * last two DO run in-process on the daemon's event loop, but are event-driven
2262
+ * (they fire when a job is dispatched, not on a fixed cadence) and the `ps` here
2263
+ * is now bounded to {@link PS_IDENTITY_TIMEOUT_MS} — they are not the
2264
+ * unconditional every-tick scan PHNX-3695 targets. The heartbeat tick's own scan
2265
+ * uses the non-blocking {@link isPidOursAsync} instead.
2266
+ */
2231
2267
  function isPidOurs(pid, spawnedAt) {
2232
2268
  try {
2233
2269
  process.kill(pid, 0);
@@ -2240,42 +2276,75 @@ function isPidOurs(pid, spawnedAt) {
2240
2276
  if (process.platform === 'win32')
2241
2277
  return true;
2242
2278
  try {
2243
- const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
2244
- if (!etime)
2245
- return true;
2246
- const parts = etime.replace(/-/g, ':').split(':').reverse();
2247
- let uptimeSec = 0;
2248
- if (parts[0])
2249
- uptimeSec += parseInt(parts[0], 10);
2250
- if (parts[1])
2251
- uptimeSec += parseInt(parts[1], 10) * 60;
2252
- if (parts[2])
2253
- uptimeSec += parseInt(parts[2], 10) * 3600;
2254
- if (parts[3])
2255
- uptimeSec += parseInt(parts[3], 10) * 86400;
2256
- const processStartMs = Date.now() - uptimeSec * 1000;
2257
- return Math.abs(processStartMs - spawnedAt) < 30_000;
2279
+ const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: PS_IDENTITY_TIMEOUT_MS }).trim();
2280
+ return etimeIndicatesOurs(etime, spawnedAt);
2258
2281
  }
2259
2282
  catch {
2260
2283
  return true;
2261
2284
  }
2262
2285
  }
2286
+ /**
2287
+ * Async, deadline-bounded twin of {@link isPidOurs} for the daemon's heartbeat
2288
+ * tick (`reapExitedRunningJobs`) — the `ps` probe runs on the shared event loop,
2289
+ * so it MUST NOT block it (PHNX-3695). Same semantics: dead pid → not ours;
2290
+ * no recorded `spawnedAt` or Windows → assume ours; a failed/empty probe reads
2291
+ * as ours (conservative — never reap on a doubtful probe).
2292
+ */
2293
+ async function isPidOursAsync(pid, spawnedAt) {
2294
+ if (!isAlive(pid))
2295
+ return false;
2296
+ if (spawnedAt === undefined)
2297
+ return true;
2298
+ if (process.platform === 'win32')
2299
+ return true;
2300
+ const res = await execFileBounded('ps', ['-p', String(pid), '-o', 'etime='], { timeoutMs: PS_IDENTITY_TIMEOUT_MS });
2301
+ if (res.code !== 0)
2302
+ return true; // probe failed → keep the run, same as the sync catch
2303
+ return etimeIndicatesOurs(res.stdout.trim(), spawnedAt);
2304
+ }
2263
2305
  /**
2264
2306
  * Finalize one `host:`-placed run by healing its host-task sidecar against the
2265
2307
  * remote `.exit` (lib/hosts/reconcile.ts). Mutates + persists the meta only
2266
2308
  * when the sidecar reached a terminal state.
2267
2309
  */
2310
+ /** Apply a healed host-task's terminal state to the local run record. Shared by the sync and async host reconcilers. */
2311
+ function applyHealedHostRun(meta, healed,
2312
+ // The routine-end emit acquires the event-log file lock. On the daemon
2313
+ // heartbeat tick (finalizeHostRunAsync) that MUST be the async, non-blocking
2314
+ // variant, or the synchronous `withFileLock` (lockSync + Atomics.wait, up to
2315
+ // 30s under contention) freezes the whole event loop — the exact wedge PHNX-3695
2316
+ // targets, on a path guard-no-sync-io does not scan (PHNX-3727). The sync CLI
2317
+ // path (finalizeHostRun) keeps the default synchronous emit.
2318
+ emit = emitRoutineEnd) {
2319
+ if (healed.status !== 'completed' && healed.status !== 'failed')
2320
+ return;
2321
+ finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
2322
+ writeRunMeta(meta);
2323
+ emit(meta);
2324
+ }
2268
2325
  function finalizeHostRun(meta) {
2269
2326
  try {
2270
2327
  const task = loadHostTask(meta.hostTaskId);
2271
2328
  if (!task)
2272
2329
  return;
2273
- const healed = reconcileHostTask(task);
2274
- if (healed.status !== 'completed' && healed.status !== 'failed')
2330
+ applyHealedHostRun(meta, reconcileHostTask(task));
2331
+ }
2332
+ catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
2333
+ }
2334
+ /**
2335
+ * Async twin of {@link finalizeHostRun} for the daemon heartbeat tick
2336
+ * (PHNX-3695). A `host:`-placed run's remote `.exit` is read over ssh; on the
2337
+ * tick that read MUST be async (`reconcileHostTaskAsync` → `sshExecAsync`) or a
2338
+ * single 6s ssh timeout freezes the whole event loop — the exact bug class this
2339
+ * effort targets, worse than the local `ps` probe.
2340
+ */
2341
+ async function finalizeHostRunAsync(meta) {
2342
+ try {
2343
+ const task = loadHostTask(meta.hostTaskId);
2344
+ if (!task)
2275
2345
  return;
2276
- finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
2277
- writeRunMeta(meta);
2278
- emitRoutineEnd(meta);
2346
+ // Tick path: emit through the async, non-blocking file lock (PHNX-3727).
2347
+ applyHealedHostRun(meta, await reconcileHostTaskAsync(task), (m) => { void emitRoutineEndAsync(m); });
2279
2348
  }
2280
2349
  catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
2281
2350
  }
@@ -2349,7 +2418,91 @@ export function isRunGenuinelyInFlight(meta) {
2349
2418
  return false;
2350
2419
  return isPidOurs(meta.pid, meta.spawnedAt);
2351
2420
  }
2352
- /** Scan all runs marked "running" and finalize any whose process has exited. */
2421
+ /**
2422
+ * Reconcile ONE `running` record once its process identity is known (`ours` =
2423
+ * the recorded pid is still genuinely the child we spawned). Shared verbatim by
2424
+ * the synchronous {@link monitorRunningJobs} (off-loop callers) and the async
2425
+ * {@link reapExitedRunningJobs} (daemon heartbeat tick) so there is one copy of
2426
+ * the finalize/timeout/report logic — only the surrounding directory/`ps` IO
2427
+ * differs by sync-vs-async flavor. Caller has already confirmed `meta.status`
2428
+ * is `running`. The finalize/report/archive helpers stay synchronous: they run
2429
+ * ONLY when a run actually transitions (times out or its process exited), a
2430
+ * conditional/rare event, not the every-tick scan the loop-starvation fix
2431
+ * (PHNX-3695) targets. The one exception is the `routine.end` EVENT emit, whose
2432
+ * file lock CAN block for up to 30s under contention even on that rare path — so
2433
+ * the emitter is injected: the off-loop sync caller passes `emitRoutineEnd`
2434
+ * (synchronous, so a short-lived CLI process flushes it before exit), the daemon
2435
+ * tick passes a fire-and-forget `emitRoutineEndAsync` (the long-lived daemon
2436
+ * flushes it, and the shared event loop is never blocked on the lock).
2437
+ */
2438
+ function reconcileRunningRecord(meta, jobRunsPath, runDirName, ours, emitEnd) {
2439
+ // Host-placed runs (meta.hostTaskId) are handled by the caller, sync or async
2440
+ // — their remote `.exit` read is ssh I/O that must stay off the daemon tick's
2441
+ // event loop (finalizeHostRunAsync). Records reaching here are local-pid runs.
2442
+ const runDirPath = path.join(jobRunsPath, runDirName);
2443
+ const stdoutPath = path.join(runDirPath, 'stdout.log');
2444
+ // Command-mode records carry no agent; there is no stream-json report to
2445
+ // parse or extract. Reap them on pid liveness alone.
2446
+ const isCommandRun = Boolean(meta.command) || !meta.agent;
2447
+ // Age-out runs FIRST, before the null-pid guard below, so a wedged record
2448
+ // still marked `running` past its deadline is always finalized — a
2449
+ // provisional launcher claim whose daemon crashed before spawning a child
2450
+ // (pid null), or an old record whose recorded pid was reused, would
2451
+ // otherwise linger as `running` forever (RUSH-2640). Only kill a process
2452
+ // group that is genuinely still ours; terminating a reused pid would kill
2453
+ // an unrelated process, and a null pid has nothing to kill.
2454
+ const wallClockMs = Date.now() - Date.parse(meta.startedAt);
2455
+ const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
2456
+ if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
2457
+ if (meta.pid && ours)
2458
+ terminateRoutineTree(meta.pid);
2459
+ finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
2460
+ writeRunMeta(meta);
2461
+ emitEnd(meta);
2462
+ if (!isCommandRun) {
2463
+ extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2464
+ archiveRoutineTranscripts(meta, runDirPath);
2465
+ }
2466
+ return;
2467
+ }
2468
+ if (!meta.pid)
2469
+ return;
2470
+ if (!ours) {
2471
+ if (isCommandRun) {
2472
+ // Command routines normally record their own terminal status via
2473
+ // child.on('exit') (so this record would already be non-'running' and
2474
+ // skipped above). Reaching here means the daemon restarted mid-run and
2475
+ // missed the exit event — recover the true code from the exit-code file
2476
+ // the child wrote; its absence means the child was killed/crashed.
2477
+ const ec = readCommandExitCode(runDirPath);
2478
+ finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
2479
+ }
2480
+ else {
2481
+ const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
2482
+ if (inferred) {
2483
+ finalizeRunMeta(meta, inferred.status, inferred.exitCode);
2484
+ }
2485
+ else {
2486
+ finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
2487
+ }
2488
+ }
2489
+ writeRunMeta(meta);
2490
+ emitEnd(meta);
2491
+ if (!isCommandRun) {
2492
+ extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2493
+ archiveRoutineTranscripts(meta, runDirPath);
2494
+ }
2495
+ }
2496
+ }
2497
+ /**
2498
+ * Scan all runs marked "running" and finalize any whose process has exited.
2499
+ *
2500
+ * SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
2501
+ * builders and CLI best-effort orphan reaps), which run in short-lived CLI
2502
+ * processes where a bounded synchronous `ps`/scan cannot starve a served event
2503
+ * loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
2504
+ * instead — a sync scan there would freeze the shared event loop (PHNX-3695).
2505
+ */
2353
2506
  export function monitorRunningJobs() {
2354
2507
  const runsDir = getRunsDir();
2355
2508
  if (!fs.existsSync(runsDir))
@@ -2378,67 +2531,67 @@ export function monitorRunningJobs() {
2378
2531
  const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
2379
2532
  if (meta.status !== 'running')
2380
2533
  continue;
2381
- // `host:`-placed run — no local pid to watch. Reconcile against the
2382
- // remote `.exit` (completion is confirmed, never guessed: an
2383
- // unreachable host leaves the run `running` for the next sweep).
2534
+ // `host:`-placed run — no local pid; reconcile against the remote `.exit`.
2384
2535
  if (meta.hostTaskId) {
2385
2536
  finalizeHostRun(meta);
2386
2537
  continue;
2387
2538
  }
2388
- const runDirPath = path.join(jobRunsPath, runDirEntry.name);
2389
- const stdoutPath = path.join(runDirPath, 'stdout.log');
2390
- // Command-mode records carry no agent; there is no stream-json report to
2391
- // parse or extract. Reap them on pid liveness alone.
2392
- const isCommandRun = Boolean(meta.command) || !meta.agent;
2393
- // Age-out runs FIRST, before the null-pid guard below, so a wedged record
2394
- // still marked `running` past its deadline is always finalized — a
2395
- // provisional launcher claim whose daemon crashed before spawning a child
2396
- // (pid null), or an old record whose recorded pid was reused, would
2397
- // otherwise linger as `running` forever (RUSH-2640). Only kill a process
2398
- // group that is genuinely still ours; terminating a reused pid would kill
2399
- // an unrelated process, and a null pid has nothing to kill.
2400
- const wallClockMs = Date.now() - Date.parse(meta.startedAt);
2401
- const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
2402
- if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
2403
- if (meta.pid && isPidOurs(meta.pid, meta.spawnedAt))
2404
- terminateRoutineTree(meta.pid);
2405
- finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
2406
- writeRunMeta(meta);
2407
- emitRoutineEnd(meta);
2408
- if (!isCommandRun) {
2409
- extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2410
- archiveRoutineTranscripts(meta, runDirPath);
2411
- }
2539
+ const ours = meta.pid ? isPidOurs(meta.pid, meta.spawnedAt) : false;
2540
+ // Sync emit: an off-loop CLI process must flush the event before it exits.
2541
+ reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, emitRoutineEnd);
2542
+ }
2543
+ catch { /* corrupt or unreadable meta.json */ }
2544
+ }
2545
+ }
2546
+ }
2547
+ /**
2548
+ * Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
2549
+ * heartbeat tick (PHNX-3695). Identical reconciliation via
2550
+ * {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
2551
+ * the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
2552
+ * the every-tick scan never freezes the shared event loop — the freeze that
2553
+ * left the browser IPC `version` probe unanswered.
2554
+ */
2555
+ export async function reapExitedRunningJobs() {
2556
+ const runsDir = getRunsDir();
2557
+ let jobDirs;
2558
+ try {
2559
+ jobDirs = await fsp.readdir(runsDir, { withFileTypes: true });
2560
+ }
2561
+ catch {
2562
+ return; // runs dir absent — nothing to reap
2563
+ }
2564
+ for (const jobDir of jobDirs) {
2565
+ if (!jobDir.isDirectory())
2566
+ continue;
2567
+ const jobRunsPath = path.join(runsDir, jobDir.name);
2568
+ let runDirs;
2569
+ try {
2570
+ runDirs = await fsp.readdir(jobRunsPath, { withFileTypes: true });
2571
+ }
2572
+ catch {
2573
+ // A retention/cleanup pass may remove a job directory after the root
2574
+ // snapshot. The next sweep observes the remaining state.
2575
+ continue;
2576
+ }
2577
+ for (const runDirEntry of runDirs) {
2578
+ if (!runDirEntry.isDirectory())
2579
+ continue;
2580
+ try {
2581
+ const raw = await fsp.readFile(path.join(jobRunsPath, runDirEntry.name, 'meta.json'), 'utf-8');
2582
+ const meta = JSON.parse(raw);
2583
+ if (meta.status !== 'running')
2412
2584
  continue;
2413
- }
2414
- if (!meta.pid)
2585
+ // `host:`-placed run — read its remote `.exit` over ssh ASYNC so a slow
2586
+ // host never freezes the tick's event loop (PHNX-3695).
2587
+ if (meta.hostTaskId) {
2588
+ await finalizeHostRunAsync(meta);
2415
2589
  continue;
2416
- if (!isPidOurs(meta.pid, meta.spawnedAt)) {
2417
- if (isCommandRun) {
2418
- // Command routines normally record their own terminal status via
2419
- // child.on('exit') (so this record would already be non-'running' and
2420
- // skipped above). Reaching here means the daemon restarted mid-run and
2421
- // missed the exit event — recover the true code from the exit-code file
2422
- // the child wrote; its absence means the child was killed/crashed.
2423
- const ec = readCommandExitCode(runDirPath);
2424
- finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
2425
- }
2426
- else {
2427
- const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
2428
- if (inferred) {
2429
- finalizeRunMeta(meta, inferred.status, inferred.exitCode);
2430
- }
2431
- else {
2432
- finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
2433
- }
2434
- }
2435
- writeRunMeta(meta);
2436
- emitRoutineEnd(meta);
2437
- if (!isCommandRun) {
2438
- extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
2439
- archiveRoutineTranscripts(meta, runDirPath);
2440
- }
2441
2590
  }
2591
+ const ours = meta.pid ? await isPidOursAsync(meta.pid, meta.spawnedAt) : false;
2592
+ // Fire-and-forget async emit: the event-log lock must never block the
2593
+ // daemon tick's shared event loop; the long-lived daemon still flushes it.
2594
+ reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, (m) => { void emitRoutineEndAsync(m); });
2442
2595
  }
2443
2596
  catch { /* corrupt or unreadable meta.json */ }
2444
2597
  }
@@ -16,7 +16,17 @@
16
16
  */
17
17
  import { BasePeriodicService } from './service.js';
18
18
  import { getDaemonDir } from '../state.js';
19
- import * as fs from 'fs';
19
+ import * as fsp from 'fs/promises';
20
+ /** Async `existsSync` — never a synchronous stat on the daemon tick loop (PHNX-3695). */
21
+ async function pathExists(p) {
22
+ try {
23
+ await fsp.access(p);
24
+ return true;
25
+ }
26
+ catch {
27
+ return false;
28
+ }
29
+ }
20
30
  /** Matches the historical inline interval (daemon.ts SELF_HEAL_TICK_MS). Runs ~every 6h. */
21
31
  const SELF_HEAL_TICK_MS = 6 * 60 * 60_000;
22
32
  /** Hard cap per tick — a full resource repair sweep across every version home, short enough a hang never freezes the service for long relative to its 6h cadence. */
@@ -35,10 +45,10 @@ export class SelfHealService extends BasePeriodicService {
35
45
  // Nothing to release — the supervisor's timer teardown is the only cleanup needed.
36
46
  }
37
47
  async onTick(ctx) {
38
- if (!fs.existsSync(getDaemonDir()))
48
+ if (!(await pathExists(getDaemonDir())))
39
49
  return;
40
50
  const { runSelfHeal, selfHealChangedAnything, selfHealNeedsAttention, summarizeSelfHeal } = await import('../self-heal/registry.js');
41
- if (!fs.existsSync(getDaemonDir()))
51
+ if (!(await pathExists(getDaemonDir())))
42
52
  return;
43
53
  const report = await runSelfHeal({ mode: 'safe' });
44
54
  if (selfHealChangedAnything(report) || selfHealNeedsAttention(report)) {
@@ -0,0 +1,174 @@
1
+ /**
2
+ * Daemon self-update service (PHNX-3695, "Fix 2").
3
+ *
4
+ * The daemon (`agents __daemon-run`) is a long-running background process that
5
+ * historically opted OUT of the CLI's interactive auto-update: `bootstrap.ts`
6
+ * force-sets `AGENTS_CLI_DISABLE_AUTO_UPDATE=1` for `__daemon-run`, so a
7
+ * running daemon never picked up new agents-cli code until a human ran
8
+ * `agents daemon restart`. R5 in the root CLAUDE.md ("An installed CLI and its
9
+ * installed helpers auto-update from the public channel") is binding for the
10
+ * interactive CLI path (`self-update.ts` / `agents upgrade`) but was silently
11
+ * NOT held for the one process that runs unattended for days. This service
12
+ * closes that gap for the daemon specifically, reusing the exact same
13
+ * verified-install primitives `agents upgrade` already uses — it does not
14
+ * fork or reimplement them.
15
+ *
16
+ * Model: verify-then-exit, not swap-in-place. A daemon cannot safely hot-swap
17
+ * its own loaded JS mid-process (in-flight ticks, open sockets, a live
18
+ * ServiceSupervisor). Instead this tick installs + BYTE-VERIFIES the new
19
+ * package on disk, and only once that succeeds does it `process.exit(0)` —
20
+ * the OS supervisor (launchd `KeepAlive` / systemd `Restart=always`, see
21
+ * `daemon/AGENTS.md`'s crash-recovery model) relaunches the daemon, which
22
+ * then boots the new code. Clients do not need to be told anything: browser
23
+ * IPC clients re-probe the socket via `waitForBrowserService`
24
+ * (`browser/ipc.ts`), and the scheduler's atomic `(routine, scheduledFor)`
25
+ * claim (see `docs/specifications.md` §Scheduling & execution singularity)
26
+ * means a routine mid-fire at the moment of exit is deduped safely across the
27
+ * restart rather than double-fired.
28
+ *
29
+ * Fail-closed is the whole point: every step below that can fail — the
30
+ * registry check, the install, the post-install verify — leaves the OLD
31
+ * daemon running untouched and logs a WARN/ERROR for the next tick to retry.
32
+ * The daemon must never exit into code it has not proven is the real,
33
+ * verified, requested version; an unverified exit would let the OS supervisor
34
+ * relaunch-loop on a broken install.
35
+ */
36
+ import { BasePeriodicService, type DaemonContext } from './service.js';
37
+ import type { DaemonServiceId } from '../daemon-services.js';
38
+ /**
39
+ * Hard cap per tick: a real download + npm/bun install + verify can
40
+ * legitimately take minutes on a slow link. 15 minutes matches the task's
41
+ * stated budget and is short relative to the ~75min cadence. Exported so the
42
+ * on-demand `request-self-update` IPC handler (`browser/ipc.ts`) can bound
43
+ * its own `AbortController` on the SAME budget the periodic tick runs under —
44
+ * one deadline, not two independently-tuned numbers that could drift apart.
45
+ */
46
+ export declare const SELF_UPDATE_DEADLINE_MS: number;
47
+ export interface SelfUpdateOutcome {
48
+ updated: boolean;
49
+ reason?: string;
50
+ }
51
+ interface NpmLatestMetadata {
52
+ version: string;
53
+ integrity: string;
54
+ tarball: string;
55
+ }
56
+ /**
57
+ * Dependency seam so `self-update-service.test.ts` can drive a REAL install
58
+ * against a fixture npm prefix/tarball without touching the real npm
59
+ * registry or the real running install. Production code always uses
60
+ * {@link defaultSelfUpdateDeps} (the default parameter below) — no test-only
61
+ * branch exists in the exported logic itself.
62
+ */
63
+ export interface SelfUpdateDeps {
64
+ currentVersion(): string;
65
+ isDevBuild(): boolean;
66
+ detectShadow(): boolean;
67
+ packageRoot(): string;
68
+ fetchLatestMetadata(signal: AbortSignal): Promise<NpmLatestMetadata>;
69
+ installAndVerify(metadata: NpmLatestMetadata, packageRoot: string, signal: AbortSignal): Promise<void>;
70
+ syncSystemRepo(): Promise<void>;
71
+ syncLocal(): Promise<void>;
72
+ }
73
+ export declare function fetchLatestNpmMetadata(signal: AbortSignal): Promise<NpmLatestMetadata>;
74
+ /**
75
+ * Install + byte-verify `metadata` into `packageRoot`'s install, exactly the
76
+ * sequence `bootstrap.ts`'s `installResolvedPackage` runs for `agents
77
+ * upgrade` (download+integrity-verify -> sweep stale staging -> package-
78
+ * manager install -> verify installed version -> refresh alias shims).
79
+ * `bootstrap.ts` cannot be imported here — it runs side-effecting top-level
80
+ * code (argv parsing, command registration) on import, which the daemon must
81
+ * never trigger — so this is the same primitives from `self-update.ts`
82
+ * composed directly, not a fork of the upgrade logic.
83
+ *
84
+ * `signal` is threaded into EVERY step's own `signal` option
85
+ * (`downloadVerifiedTarball` / `installPackageIntoPrefix` /
86
+ * `installPackageWithBun`, all in `self-update.ts`) rather than raced against
87
+ * from the outside: those primitives kill the underlying fetch/child process
88
+ * on abort and their promise rejects only once that real cancellation has
89
+ * happened. A wrapper that merely stopped AWAITING an unkillable operation
90
+ * (the prior approach here) left an orphaned `npm install -g`/`bun add -g`
91
+ * writing into the shared global prefix — which the very next tick's fresh
92
+ * install (started as soon as the supervisor's backoff fires, seconds later)
93
+ * would then race into the same directory. Real cancellation is what makes
94
+ * `attemptSelfUpdateAndExit`'s `inFlightAttempt` dedupe (below) an actual
95
+ * guarantee instead of a guard whose lifetime is shorter than the operation
96
+ * it's guarding (found in review, PHNX-3695).
97
+ */
98
+ export declare function installAndVerifyDefault(metadata: NpmLatestMetadata, packageRoot: string, signal: AbortSignal): Promise<void>;
99
+ export declare function defaultSelfUpdateDeps(): SelfUpdateDeps;
100
+ /**
101
+ * Core self-update decision + action, shared by the periodic tick
102
+ * ({@link SelfUpdateService.onTick}) and the on-demand IPC path
103
+ * (`request-self-update`, `browser/ipc.ts`) — one implementation, so a
104
+ * version-skew client asking "update now" runs exactly the same fail-closed
105
+ * logic as the scheduled sweep. Returns rather than throws so callers decide
106
+ * their own exit timing (the periodic service exits immediately; the IPC
107
+ * handler must respond to the client on the socket BEFORE exiting, or the
108
+ * client hangs on a socket that is closing mid-write). Concurrent callers
109
+ * share one in-flight attempt rather than racing separate installs.
110
+ */
111
+ export declare function attemptSelfUpdateAndExit(ctx: DaemonContext, signal: AbortSignal, deps?: SelfUpdateDeps): Promise<SelfUpdateOutcome>;
112
+ /**
113
+ * Schedule the process exit for a verified self-update, exactly once, no
114
+ * matter how many callers observe `outcome.updated` on the shared
115
+ * `inFlightAttempt` promise. The periodic tick and an on-demand
116
+ * `request-self-update` IPC call (`browser/ipc.ts`) can both be awaiting that
117
+ * SAME promise — if the tick's continuation ran an immediate `process.exit(0)`
118
+ * while the IPC handler's continuation had not yet reached `socket.write`,
119
+ * the tick's exit could win the race and the client would see a closed socket
120
+ * before any response (found in review, PHNX-3695). Routing every caller
121
+ * through this one guarded, always-delayed scheduling point means the delay
122
+ * protects EVERY caller's in-flight response, not just the IPC handler's own.
123
+ */
124
+ export declare function scheduleSelfUpdateExit(): void;
125
+ /**
126
+ * Fire the on-demand self-update in the BACKGROUND and return its (bounded)
127
+ * promise WITHOUT the caller having to await it. This is what keeps the
128
+ * `request-self-update` IPC handler (`browser/ipc.ts`) from parking a
129
+ * version-skewed `agents browser` verb behind the full
130
+ * check→download→install→verify: that handler routes through
131
+ * `reconcileDaemonVersion` on every version-skewed call, so awaiting the whole
132
+ * install there reintroduces exactly the client-stall PHNX-3605 was written to
133
+ * prevent (tens of seconds, worst case ~15 min). The handler instead responds
134
+ * "triggered" immediately and lets this run in the background — the daemon does
135
+ * install→verify→exit(0) on its own, the OS supervisor relaunches it, and the
136
+ * browser reconnects.
137
+ *
138
+ * The work still shares the module-level {@link attemptSelfUpdateAndExit}
139
+ * `inFlightAttempt` guard, so a concurrent trigger (or the periodic tick) can't
140
+ * race a second install into the same prefix. It is bounded by
141
+ * {@link SELF_UPDATE_DEADLINE_MS} via an `AbortController` nobody awaits (the
142
+ * timer is `unref`'d so it never keeps the daemon alive on its own and never
143
+ * dangles in a test). Fail-closed is preserved end to end:
144
+ * `runSelfUpdateAttempt` already turns an install/verify failure into a
145
+ * not-updated outcome that leaves the running daemon untouched. The caller
146
+ * schedules the one decoupled {@link scheduleSelfUpdateExit} off the returned
147
+ * promise once `updated` is true, so the exit still fires after the IPC
148
+ * response has flushed.
149
+ */
150
+ export declare function triggerSelfUpdateInBackground(ctx: DaemonContext, deps?: SelfUpdateDeps): Promise<SelfUpdateOutcome>;
151
+ /**
152
+ * The subset of self-update decline checks that are INSTANT and network-free —
153
+ * a dev build, or a shadowed install. The on-demand IPC handler
154
+ * (`request-self-update`) runs these SYNCHRONOUSLY so a version-skewed browser
155
+ * client gets the PHNX-3605 "nothing changed, not evicting" advisory
156
+ * immediately, instead of a "triggered" it would never act on. The remaining
157
+ * checks (registry probe, already-current, install/verify) stay inside the
158
+ * backgrounded {@link attemptSelfUpdateAndExit} so the handler never blocks on
159
+ * the network or the install. This is the single source of the two instant
160
+ * decline reasons — {@link runSelfUpdateAttempt} calls it too, so the on-demand
161
+ * decline text can never drift from the periodic tick's. Returns the decline
162
+ * reason, or `null` to proceed to the (backgrounded) install path.
163
+ */
164
+ export declare function selfUpdateSyncDeclineReason(deps?: SelfUpdateDeps): string | null;
165
+ export declare class SelfUpdateService extends BasePeriodicService {
166
+ readonly id: DaemonServiceId;
167
+ readonly intervalMs: number;
168
+ readonly deadlineMs: number;
169
+ readonly startupDelayMs: number;
170
+ protected onStart(_ctx: DaemonContext): Promise<void>;
171
+ protected onStop(): Promise<void>;
172
+ protected onTick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
173
+ }
174
+ export {};