@phnx-labs/agents-cli 1.22.70 → 1.22.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +31 -1
- package/dist/bootstrap.js +4 -4
- package/dist/commands/repo.js +2 -2
- package/dist/commands/sessions-export.d.ts +5 -1
- package/dist/commands/sessions-export.js +100 -24
- package/dist/commands/sessions-import.d.ts +2 -1
- package/dist/commands/sessions-import.js +85 -21
- package/dist/lib/accounting/usage-sync.d.ts +1 -1
- package/dist/lib/accounting/usage-sync.js +3 -3
- package/dist/lib/browser/ipc.d.ts +34 -0
- package/dist/lib/browser/ipc.js +140 -19
- package/dist/lib/browser/types.d.ts +3 -1
- package/dist/lib/daemon/auth-sync-service.js +1 -1
- package/dist/lib/daemon/browser-task-reap-service.js +1 -1
- package/dist/lib/daemon/daemon.js +13 -3
- package/dist/lib/daemon/heartbeat-service.js +3 -3
- package/dist/lib/daemon/keychain-reap-service.js +1 -1
- package/dist/lib/daemon/runner.d.ts +18 -1
- package/dist/lib/daemon/runner.js +231 -78
- package/dist/lib/daemon/self-heal-service.js +13 -3
- package/dist/lib/daemon/self-update-service.d.ts +174 -0
- package/dist/lib/daemon/self-update-service.js +353 -0
- package/dist/lib/daemon/state-dir-check-service.js +3 -3
- package/dist/lib/daemon/usage-sync-service.js +1 -1
- package/dist/lib/daemon/watchdog-service.js +4 -4
- package/dist/lib/daemon-services.d.ts +1 -1
- package/dist/lib/daemon-services.js +5 -0
- package/dist/lib/device-config.d.ts +12 -1
- package/dist/lib/device-config.js +63 -13
- package/dist/lib/exec-bounded.d.ts +52 -0
- package/dist/lib/exec-bounded.js +113 -0
- package/dist/lib/feed/events.d.ts +22 -14
- package/dist/lib/feed/events.js +84 -44
- package/dist/lib/fleet-shared-state.d.ts +12 -5
- package/dist/lib/fleet-shared-state.js +50 -20
- package/dist/lib/fs-atomic.d.ts +11 -0
- package/dist/lib/fs-atomic.js +60 -0
- package/dist/lib/hosts/reconcile.d.ts +11 -4
- package/dist/lib/hosts/reconcile.js +31 -5
- package/dist/lib/project-resources.d.ts +12 -0
- package/dist/lib/project-resources.js +129 -0
- package/dist/lib/routine-process-cleanup.d.ts +2 -2
- package/dist/lib/routine-process-cleanup.js +45 -34
- package/dist/lib/secrets/reaper.d.ts +2 -2
- package/dist/lib/secrets/reaper.js +13 -10
- package/dist/lib/secrets/reserved-sync.d.ts +1 -1
- package/dist/lib/secrets/reserved-sync.js +4 -4
- package/dist/lib/self-update.d.ts +21 -8
- package/dist/lib/self-update.js +54 -31
- package/dist/lib/session/sync/backend.d.ts +61 -0
- package/dist/lib/session/sync/backend.js +89 -0
- package/dist/lib/session/sync/managed-config.d.ts +29 -0
- package/dist/lib/session/sync/managed-config.js +23 -0
- package/dist/lib/session/sync/managed-key.d.ts +45 -0
- package/dist/lib/session/sync/managed-key.js +128 -0
- package/dist/lib/session/sync/net-client.d.ts +65 -0
- package/dist/lib/session/sync/net-client.js +117 -0
- package/dist/lib/session/sync/provision.d.ts +19 -0
- package/dist/lib/session/sync/provision.js +38 -0
- package/dist/lib/session/sync/r2.d.ts +5 -2
- package/dist/lib/session/sync/r2.js +5 -2
- package/dist/lib/session/sync/worker-template.d.ts +6 -0
- package/dist/lib/session/sync/worker-template.js +847 -0
- package/dist/lib/tmux/orphan-reap.js +6 -4
- package/dist/lib/tmux/session.js +4 -1
- package/dist/lib/traces/classify.d.ts +8 -1
- package/dist/lib/traces/insights.d.ts +13 -1
- package/dist/lib/traces/insights.js +78 -3
- package/dist/lib/traces/sync.js +8 -3
- package/dist/lib/traces/worker-template.js +9 -5
- package/package.json +1 -1
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { spawn, execFileSync } from 'child_process';
|
|
17
17
|
import * as fs from 'fs';
|
|
18
|
+
import * as fsp from 'fs/promises';
|
|
18
19
|
import * as path from 'path';
|
|
19
20
|
import * as os from 'os';
|
|
20
21
|
import { getCliLaunch, getAgentsBinDir } from '../cli-entry.js';
|
|
@@ -23,14 +24,15 @@ import { getRunsDir, getUserAgentsDir, readMeta, getDaemonDir } from '../state.j
|
|
|
23
24
|
import { shortCodexHome } from '../codex-home.js';
|
|
24
25
|
import { prepareJobHome, buildSpawnEnv, getJobHomePath, assertSandboxForwardsHostGhAuth, linkVersionAuth } from '../sandbox.js';
|
|
25
26
|
import { resolveModel, buildReasoningFlags } from '../models.js';
|
|
26
|
-
import { createTimer, redactPrompt, emitRoutineEnd } from '../feed/events.js';
|
|
27
|
+
import { createTimer, redactPrompt, emitRoutineEnd, emitRoutineEndAsync } from '../feed/events.js';
|
|
27
28
|
import { resolveHarnessAdapter } from '../harness/index.js';
|
|
28
29
|
import { applyAddDirs } from '../add-dir.js';
|
|
29
30
|
import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, AGENT_COMMANDS, } from '../exec.js';
|
|
30
31
|
import { resolveActor } from '../actor.js';
|
|
31
32
|
import { loadTask as loadHostTask } from '../hosts/tasks.js';
|
|
32
|
-
import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
|
|
33
|
-
import { backgroundSpawnOptions, killTree } from '../platform/process.js';
|
|
33
|
+
import { reconcileTask as reconcileHostTask, reconcileTaskAsync as reconcileHostTaskAsync } from '../hosts/reconcile.js';
|
|
34
|
+
import { backgroundSpawnOptions, isAlive, killTree } from '../platform/process.js';
|
|
35
|
+
import { execFileBounded } from '../exec-bounded.js';
|
|
34
36
|
import lockfile from 'proper-lockfile';
|
|
35
37
|
import { ensureLockTarget } from '../fs-atomic.js';
|
|
36
38
|
import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
|
|
@@ -2228,6 +2230,40 @@ const MAX_WALL_CLOCK_MS = 24 * 60 * 60 * 1000;
|
|
|
2228
2230
|
* compares against the process's actual start time via `ps`. Returns true
|
|
2229
2231
|
* when the PID is alive AND plausibly ours.
|
|
2230
2232
|
*/
|
|
2233
|
+
/** Bound the identity `ps` — a hung `ps` must never freeze its caller for long (matches routine-process-cleanup's probe). */
|
|
2234
|
+
const PS_IDENTITY_TIMEOUT_MS = 5_000;
|
|
2235
|
+
/**
|
|
2236
|
+
* Pure: does a `ps -o etime=` value place the process's birth within 30s of when
|
|
2237
|
+
* we recorded spawning it? An empty/unknown reading conservatively reads as ours
|
|
2238
|
+
* — never reap a run on a doubtful identity probe.
|
|
2239
|
+
*/
|
|
2240
|
+
function etimeIndicatesOurs(etime, spawnedAt) {
|
|
2241
|
+
if (!etime)
|
|
2242
|
+
return true;
|
|
2243
|
+
const parts = etime.replace(/-/g, ':').split(':').reverse();
|
|
2244
|
+
let uptimeSec = 0;
|
|
2245
|
+
if (parts[0])
|
|
2246
|
+
uptimeSec += parseInt(parts[0], 10);
|
|
2247
|
+
if (parts[1])
|
|
2248
|
+
uptimeSec += parseInt(parts[1], 10) * 60;
|
|
2249
|
+
if (parts[2])
|
|
2250
|
+
uptimeSec += parseInt(parts[2], 10) * 3600;
|
|
2251
|
+
if (parts[3])
|
|
2252
|
+
uptimeSec += parseInt(parts[3], 10) * 86400;
|
|
2253
|
+
const processStartMs = Date.now() - uptimeSec * 1000;
|
|
2254
|
+
return Math.abs(processStartMs - spawnedAt) < 30_000;
|
|
2255
|
+
}
|
|
2256
|
+
/**
|
|
2257
|
+
* Synchronous identity check for callers OTHER than the periodic heartbeat tick:
|
|
2258
|
+
* the `agents routines list/status` builders and stop-time enumeration (both in
|
|
2259
|
+
* short-lived CLI processes), plus the scheduler's fire-time slot-claim
|
|
2260
|
+
* (`activeRoutineRun`) and status projection (`isRunGenuinelyInFlight`). Those
|
|
2261
|
+
* last two DO run in-process on the daemon's event loop, but are event-driven
|
|
2262
|
+
* (they fire when a job is dispatched, not on a fixed cadence) and the `ps` here
|
|
2263
|
+
* is now bounded to {@link PS_IDENTITY_TIMEOUT_MS} — they are not the
|
|
2264
|
+
* unconditional every-tick scan PHNX-3695 targets. The heartbeat tick's own scan
|
|
2265
|
+
* uses the non-blocking {@link isPidOursAsync} instead.
|
|
2266
|
+
*/
|
|
2231
2267
|
function isPidOurs(pid, spawnedAt) {
|
|
2232
2268
|
try {
|
|
2233
2269
|
process.kill(pid, 0);
|
|
@@ -2240,42 +2276,75 @@ function isPidOurs(pid, spawnedAt) {
|
|
|
2240
2276
|
if (process.platform === 'win32')
|
|
2241
2277
|
return true;
|
|
2242
2278
|
try {
|
|
2243
|
-
const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
2244
|
-
|
|
2245
|
-
return true;
|
|
2246
|
-
const parts = etime.replace(/-/g, ':').split(':').reverse();
|
|
2247
|
-
let uptimeSec = 0;
|
|
2248
|
-
if (parts[0])
|
|
2249
|
-
uptimeSec += parseInt(parts[0], 10);
|
|
2250
|
-
if (parts[1])
|
|
2251
|
-
uptimeSec += parseInt(parts[1], 10) * 60;
|
|
2252
|
-
if (parts[2])
|
|
2253
|
-
uptimeSec += parseInt(parts[2], 10) * 3600;
|
|
2254
|
-
if (parts[3])
|
|
2255
|
-
uptimeSec += parseInt(parts[3], 10) * 86400;
|
|
2256
|
-
const processStartMs = Date.now() - uptimeSec * 1000;
|
|
2257
|
-
return Math.abs(processStartMs - spawnedAt) < 30_000;
|
|
2279
|
+
const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: PS_IDENTITY_TIMEOUT_MS }).trim();
|
|
2280
|
+
return etimeIndicatesOurs(etime, spawnedAt);
|
|
2258
2281
|
}
|
|
2259
2282
|
catch {
|
|
2260
2283
|
return true;
|
|
2261
2284
|
}
|
|
2262
2285
|
}
|
|
2286
|
+
/**
|
|
2287
|
+
* Async, deadline-bounded twin of {@link isPidOurs} for the daemon's heartbeat
|
|
2288
|
+
* tick (`reapExitedRunningJobs`) — the `ps` probe runs on the shared event loop,
|
|
2289
|
+
* so it MUST NOT block it (PHNX-3695). Same semantics: dead pid → not ours;
|
|
2290
|
+
* no recorded `spawnedAt` or Windows → assume ours; a failed/empty probe reads
|
|
2291
|
+
* as ours (conservative — never reap on a doubtful probe).
|
|
2292
|
+
*/
|
|
2293
|
+
async function isPidOursAsync(pid, spawnedAt) {
|
|
2294
|
+
if (!isAlive(pid))
|
|
2295
|
+
return false;
|
|
2296
|
+
if (spawnedAt === undefined)
|
|
2297
|
+
return true;
|
|
2298
|
+
if (process.platform === 'win32')
|
|
2299
|
+
return true;
|
|
2300
|
+
const res = await execFileBounded('ps', ['-p', String(pid), '-o', 'etime='], { timeoutMs: PS_IDENTITY_TIMEOUT_MS });
|
|
2301
|
+
if (res.code !== 0)
|
|
2302
|
+
return true; // probe failed → keep the run, same as the sync catch
|
|
2303
|
+
return etimeIndicatesOurs(res.stdout.trim(), spawnedAt);
|
|
2304
|
+
}
|
|
2263
2305
|
/**
|
|
2264
2306
|
* Finalize one `host:`-placed run by healing its host-task sidecar against the
|
|
2265
2307
|
* remote `.exit` (lib/hosts/reconcile.ts). Mutates + persists the meta only
|
|
2266
2308
|
* when the sidecar reached a terminal state.
|
|
2267
2309
|
*/
|
|
2310
|
+
/** Apply a healed host-task's terminal state to the local run record. Shared by the sync and async host reconcilers. */
|
|
2311
|
+
function applyHealedHostRun(meta, healed,
|
|
2312
|
+
// The routine-end emit acquires the event-log file lock. On the daemon
|
|
2313
|
+
// heartbeat tick (finalizeHostRunAsync) that MUST be the async, non-blocking
|
|
2314
|
+
// variant, or the synchronous `withFileLock` (lockSync + Atomics.wait, up to
|
|
2315
|
+
// 30s under contention) freezes the whole event loop — the exact wedge PHNX-3695
|
|
2316
|
+
// targets, on a path guard-no-sync-io does not scan (PHNX-3727). The sync CLI
|
|
2317
|
+
// path (finalizeHostRun) keeps the default synchronous emit.
|
|
2318
|
+
emit = emitRoutineEnd) {
|
|
2319
|
+
if (healed.status !== 'completed' && healed.status !== 'failed')
|
|
2320
|
+
return;
|
|
2321
|
+
finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
|
|
2322
|
+
writeRunMeta(meta);
|
|
2323
|
+
emit(meta);
|
|
2324
|
+
}
|
|
2268
2325
|
function finalizeHostRun(meta) {
|
|
2269
2326
|
try {
|
|
2270
2327
|
const task = loadHostTask(meta.hostTaskId);
|
|
2271
2328
|
if (!task)
|
|
2272
2329
|
return;
|
|
2273
|
-
|
|
2274
|
-
|
|
2330
|
+
applyHealedHostRun(meta, reconcileHostTask(task));
|
|
2331
|
+
}
|
|
2332
|
+
catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
|
|
2333
|
+
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Async twin of {@link finalizeHostRun} for the daemon heartbeat tick
|
|
2336
|
+
* (PHNX-3695). A `host:`-placed run's remote `.exit` is read over ssh; on the
|
|
2337
|
+
* tick that read MUST be async (`reconcileHostTaskAsync` → `sshExecAsync`) or a
|
|
2338
|
+
* single 6s ssh timeout freezes the whole event loop — the exact bug class this
|
|
2339
|
+
* effort targets, worse than the local `ps` probe.
|
|
2340
|
+
*/
|
|
2341
|
+
async function finalizeHostRunAsync(meta) {
|
|
2342
|
+
try {
|
|
2343
|
+
const task = loadHostTask(meta.hostTaskId);
|
|
2344
|
+
if (!task)
|
|
2275
2345
|
return;
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
emitRoutineEnd(meta);
|
|
2346
|
+
// Tick path: emit through the async, non-blocking file lock (PHNX-3727).
|
|
2347
|
+
applyHealedHostRun(meta, await reconcileHostTaskAsync(task), (m) => { void emitRoutineEndAsync(m); });
|
|
2279
2348
|
}
|
|
2280
2349
|
catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
|
|
2281
2350
|
}
|
|
@@ -2349,7 +2418,91 @@ export function isRunGenuinelyInFlight(meta) {
|
|
|
2349
2418
|
return false;
|
|
2350
2419
|
return isPidOurs(meta.pid, meta.spawnedAt);
|
|
2351
2420
|
}
|
|
2352
|
-
/**
|
|
2421
|
+
/**
|
|
2422
|
+
* Reconcile ONE `running` record once its process identity is known (`ours` =
|
|
2423
|
+
* the recorded pid is still genuinely the child we spawned). Shared verbatim by
|
|
2424
|
+
* the synchronous {@link monitorRunningJobs} (off-loop callers) and the async
|
|
2425
|
+
* {@link reapExitedRunningJobs} (daemon heartbeat tick) so there is one copy of
|
|
2426
|
+
* the finalize/timeout/report logic — only the surrounding directory/`ps` IO
|
|
2427
|
+
* differs by sync-vs-async flavor. Caller has already confirmed `meta.status`
|
|
2428
|
+
* is `running`. The finalize/report/archive helpers stay synchronous: they run
|
|
2429
|
+
* ONLY when a run actually transitions (times out or its process exited), a
|
|
2430
|
+
* conditional/rare event, not the every-tick scan the loop-starvation fix
|
|
2431
|
+
* (PHNX-3695) targets. The one exception is the `routine.end` EVENT emit, whose
|
|
2432
|
+
* file lock CAN block for up to 30s under contention even on that rare path — so
|
|
2433
|
+
* the emitter is injected: the off-loop sync caller passes `emitRoutineEnd`
|
|
2434
|
+
* (synchronous, so a short-lived CLI process flushes it before exit), the daemon
|
|
2435
|
+
* tick passes a fire-and-forget `emitRoutineEndAsync` (the long-lived daemon
|
|
2436
|
+
* flushes it, and the shared event loop is never blocked on the lock).
|
|
2437
|
+
*/
|
|
2438
|
+
function reconcileRunningRecord(meta, jobRunsPath, runDirName, ours, emitEnd) {
|
|
2439
|
+
// Host-placed runs (meta.hostTaskId) are handled by the caller, sync or async
|
|
2440
|
+
// — their remote `.exit` read is ssh I/O that must stay off the daemon tick's
|
|
2441
|
+
// event loop (finalizeHostRunAsync). Records reaching here are local-pid runs.
|
|
2442
|
+
const runDirPath = path.join(jobRunsPath, runDirName);
|
|
2443
|
+
const stdoutPath = path.join(runDirPath, 'stdout.log');
|
|
2444
|
+
// Command-mode records carry no agent; there is no stream-json report to
|
|
2445
|
+
// parse or extract. Reap them on pid liveness alone.
|
|
2446
|
+
const isCommandRun = Boolean(meta.command) || !meta.agent;
|
|
2447
|
+
// Age-out runs FIRST, before the null-pid guard below, so a wedged record
|
|
2448
|
+
// still marked `running` past its deadline is always finalized — a
|
|
2449
|
+
// provisional launcher claim whose daemon crashed before spawning a child
|
|
2450
|
+
// (pid null), or an old record whose recorded pid was reused, would
|
|
2451
|
+
// otherwise linger as `running` forever (RUSH-2640). Only kill a process
|
|
2452
|
+
// group that is genuinely still ours; terminating a reused pid would kill
|
|
2453
|
+
// an unrelated process, and a null pid has nothing to kill.
|
|
2454
|
+
const wallClockMs = Date.now() - Date.parse(meta.startedAt);
|
|
2455
|
+
const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
|
|
2456
|
+
if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
|
|
2457
|
+
if (meta.pid && ours)
|
|
2458
|
+
terminateRoutineTree(meta.pid);
|
|
2459
|
+
finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
|
|
2460
|
+
writeRunMeta(meta);
|
|
2461
|
+
emitEnd(meta);
|
|
2462
|
+
if (!isCommandRun) {
|
|
2463
|
+
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2464
|
+
archiveRoutineTranscripts(meta, runDirPath);
|
|
2465
|
+
}
|
|
2466
|
+
return;
|
|
2467
|
+
}
|
|
2468
|
+
if (!meta.pid)
|
|
2469
|
+
return;
|
|
2470
|
+
if (!ours) {
|
|
2471
|
+
if (isCommandRun) {
|
|
2472
|
+
// Command routines normally record their own terminal status via
|
|
2473
|
+
// child.on('exit') (so this record would already be non-'running' and
|
|
2474
|
+
// skipped above). Reaching here means the daemon restarted mid-run and
|
|
2475
|
+
// missed the exit event — recover the true code from the exit-code file
|
|
2476
|
+
// the child wrote; its absence means the child was killed/crashed.
|
|
2477
|
+
const ec = readCommandExitCode(runDirPath);
|
|
2478
|
+
finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
|
|
2479
|
+
}
|
|
2480
|
+
else {
|
|
2481
|
+
const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
|
|
2482
|
+
if (inferred) {
|
|
2483
|
+
finalizeRunMeta(meta, inferred.status, inferred.exitCode);
|
|
2484
|
+
}
|
|
2485
|
+
else {
|
|
2486
|
+
finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
writeRunMeta(meta);
|
|
2490
|
+
emitEnd(meta);
|
|
2491
|
+
if (!isCommandRun) {
|
|
2492
|
+
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2493
|
+
archiveRoutineTranscripts(meta, runDirPath);
|
|
2494
|
+
}
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
/**
|
|
2498
|
+
* Scan all runs marked "running" and finalize any whose process has exited.
|
|
2499
|
+
*
|
|
2500
|
+
* SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
|
|
2501
|
+
* builders and CLI best-effort orphan reaps), which run in short-lived CLI
|
|
2502
|
+
* processes where a bounded synchronous `ps`/scan cannot starve a served event
|
|
2503
|
+
* loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
|
|
2504
|
+
* instead — a sync scan there would freeze the shared event loop (PHNX-3695).
|
|
2505
|
+
*/
|
|
2353
2506
|
export function monitorRunningJobs() {
|
|
2354
2507
|
const runsDir = getRunsDir();
|
|
2355
2508
|
if (!fs.existsSync(runsDir))
|
|
@@ -2378,67 +2531,67 @@ export function monitorRunningJobs() {
|
|
|
2378
2531
|
const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
|
|
2379
2532
|
if (meta.status !== 'running')
|
|
2380
2533
|
continue;
|
|
2381
|
-
// `host:`-placed run — no local pid
|
|
2382
|
-
// remote `.exit` (completion is confirmed, never guessed: an
|
|
2383
|
-
// unreachable host leaves the run `running` for the next sweep).
|
|
2534
|
+
// `host:`-placed run — no local pid; reconcile against the remote `.exit`.
|
|
2384
2535
|
if (meta.hostTaskId) {
|
|
2385
2536
|
finalizeHostRun(meta);
|
|
2386
2537
|
continue;
|
|
2387
2538
|
}
|
|
2388
|
-
const
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2539
|
+
const ours = meta.pid ? isPidOurs(meta.pid, meta.spawnedAt) : false;
|
|
2540
|
+
// Sync emit: an off-loop CLI process must flush the event before it exits.
|
|
2541
|
+
reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, emitRoutineEnd);
|
|
2542
|
+
}
|
|
2543
|
+
catch { /* corrupt or unreadable meta.json */ }
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
/**
|
|
2548
|
+
* Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
|
|
2549
|
+
* heartbeat tick (PHNX-3695). Identical reconciliation via
|
|
2550
|
+
* {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
|
|
2551
|
+
* the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
|
|
2552
|
+
* the every-tick scan never freezes the shared event loop — the freeze that
|
|
2553
|
+
* left the browser IPC `version` probe unanswered.
|
|
2554
|
+
*/
|
|
2555
|
+
export async function reapExitedRunningJobs() {
|
|
2556
|
+
const runsDir = getRunsDir();
|
|
2557
|
+
let jobDirs;
|
|
2558
|
+
try {
|
|
2559
|
+
jobDirs = await fsp.readdir(runsDir, { withFileTypes: true });
|
|
2560
|
+
}
|
|
2561
|
+
catch {
|
|
2562
|
+
return; // runs dir absent — nothing to reap
|
|
2563
|
+
}
|
|
2564
|
+
for (const jobDir of jobDirs) {
|
|
2565
|
+
if (!jobDir.isDirectory())
|
|
2566
|
+
continue;
|
|
2567
|
+
const jobRunsPath = path.join(runsDir, jobDir.name);
|
|
2568
|
+
let runDirs;
|
|
2569
|
+
try {
|
|
2570
|
+
runDirs = await fsp.readdir(jobRunsPath, { withFileTypes: true });
|
|
2571
|
+
}
|
|
2572
|
+
catch {
|
|
2573
|
+
// A retention/cleanup pass may remove a job directory after the root
|
|
2574
|
+
// snapshot. The next sweep observes the remaining state.
|
|
2575
|
+
continue;
|
|
2576
|
+
}
|
|
2577
|
+
for (const runDirEntry of runDirs) {
|
|
2578
|
+
if (!runDirEntry.isDirectory())
|
|
2579
|
+
continue;
|
|
2580
|
+
try {
|
|
2581
|
+
const raw = await fsp.readFile(path.join(jobRunsPath, runDirEntry.name, 'meta.json'), 'utf-8');
|
|
2582
|
+
const meta = JSON.parse(raw);
|
|
2583
|
+
if (meta.status !== 'running')
|
|
2412
2584
|
continue;
|
|
2413
|
-
|
|
2414
|
-
|
|
2585
|
+
// `host:`-placed run — read its remote `.exit` over ssh ASYNC so a slow
|
|
2586
|
+
// host never freezes the tick's event loop (PHNX-3695).
|
|
2587
|
+
if (meta.hostTaskId) {
|
|
2588
|
+
await finalizeHostRunAsync(meta);
|
|
2415
2589
|
continue;
|
|
2416
|
-
if (!isPidOurs(meta.pid, meta.spawnedAt)) {
|
|
2417
|
-
if (isCommandRun) {
|
|
2418
|
-
// Command routines normally record their own terminal status via
|
|
2419
|
-
// child.on('exit') (so this record would already be non-'running' and
|
|
2420
|
-
// skipped above). Reaching here means the daemon restarted mid-run and
|
|
2421
|
-
// missed the exit event — recover the true code from the exit-code file
|
|
2422
|
-
// the child wrote; its absence means the child was killed/crashed.
|
|
2423
|
-
const ec = readCommandExitCode(runDirPath);
|
|
2424
|
-
finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
|
|
2425
|
-
}
|
|
2426
|
-
else {
|
|
2427
|
-
const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
|
|
2428
|
-
if (inferred) {
|
|
2429
|
-
finalizeRunMeta(meta, inferred.status, inferred.exitCode);
|
|
2430
|
-
}
|
|
2431
|
-
else {
|
|
2432
|
-
finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
|
|
2433
|
-
}
|
|
2434
|
-
}
|
|
2435
|
-
writeRunMeta(meta);
|
|
2436
|
-
emitRoutineEnd(meta);
|
|
2437
|
-
if (!isCommandRun) {
|
|
2438
|
-
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2439
|
-
archiveRoutineTranscripts(meta, runDirPath);
|
|
2440
|
-
}
|
|
2441
2590
|
}
|
|
2591
|
+
const ours = meta.pid ? await isPidOursAsync(meta.pid, meta.spawnedAt) : false;
|
|
2592
|
+
// Fire-and-forget async emit: the event-log lock must never block the
|
|
2593
|
+
// daemon tick's shared event loop; the long-lived daemon still flushes it.
|
|
2594
|
+
reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, (m) => { void emitRoutineEndAsync(m); });
|
|
2442
2595
|
}
|
|
2443
2596
|
catch { /* corrupt or unreadable meta.json */ }
|
|
2444
2597
|
}
|
|
@@ -16,7 +16,17 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { BasePeriodicService } from './service.js';
|
|
18
18
|
import { getDaemonDir } from '../state.js';
|
|
19
|
-
import * as
|
|
19
|
+
import * as fsp from 'fs/promises';
|
|
20
|
+
/** Async `existsSync` — never a synchronous stat on the daemon tick loop (PHNX-3695). */
|
|
21
|
+
async function pathExists(p) {
|
|
22
|
+
try {
|
|
23
|
+
await fsp.access(p);
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
20
30
|
/** Matches the historical inline interval (daemon.ts SELF_HEAL_TICK_MS). Runs ~every 6h. */
|
|
21
31
|
const SELF_HEAL_TICK_MS = 6 * 60 * 60_000;
|
|
22
32
|
/** Hard cap per tick — a full resource repair sweep across every version home, short enough a hang never freezes the service for long relative to its 6h cadence. */
|
|
@@ -35,10 +45,10 @@ export class SelfHealService extends BasePeriodicService {
|
|
|
35
45
|
// Nothing to release — the supervisor's timer teardown is the only cleanup needed.
|
|
36
46
|
}
|
|
37
47
|
async onTick(ctx) {
|
|
38
|
-
if (!
|
|
48
|
+
if (!(await pathExists(getDaemonDir())))
|
|
39
49
|
return;
|
|
40
50
|
const { runSelfHeal, selfHealChangedAnything, selfHealNeedsAttention, summarizeSelfHeal } = await import('../self-heal/registry.js');
|
|
41
|
-
if (!
|
|
51
|
+
if (!(await pathExists(getDaemonDir())))
|
|
42
52
|
return;
|
|
43
53
|
const report = await runSelfHeal({ mode: 'safe' });
|
|
44
54
|
if (selfHealChangedAnything(report) || selfHealNeedsAttention(report)) {
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Daemon self-update service (PHNX-3695, "Fix 2").
|
|
3
|
+
*
|
|
4
|
+
* The daemon (`agents __daemon-run`) is a long-running background process that
|
|
5
|
+
* historically opted OUT of the CLI's interactive auto-update: `bootstrap.ts`
|
|
6
|
+
* force-sets `AGENTS_CLI_DISABLE_AUTO_UPDATE=1` for `__daemon-run`, so a
|
|
7
|
+
* running daemon never picked up new agents-cli code until a human ran
|
|
8
|
+
* `agents daemon restart`. R5 in the root CLAUDE.md ("An installed CLI and its
|
|
9
|
+
* installed helpers auto-update from the public channel") is binding for the
|
|
10
|
+
* interactive CLI path (`self-update.ts` / `agents upgrade`) but was silently
|
|
11
|
+
* NOT held for the one process that runs unattended for days. This service
|
|
12
|
+
* closes that gap for the daemon specifically, reusing the exact same
|
|
13
|
+
* verified-install primitives `agents upgrade` already uses — it does not
|
|
14
|
+
* fork or reimplement them.
|
|
15
|
+
*
|
|
16
|
+
* Model: verify-then-exit, not swap-in-place. A daemon cannot safely hot-swap
|
|
17
|
+
* its own loaded JS mid-process (in-flight ticks, open sockets, a live
|
|
18
|
+
* ServiceSupervisor). Instead this tick installs + BYTE-VERIFIES the new
|
|
19
|
+
* package on disk, and only once that succeeds does it `process.exit(0)` —
|
|
20
|
+
* the OS supervisor (launchd `KeepAlive` / systemd `Restart=always`, see
|
|
21
|
+
* `daemon/AGENTS.md`'s crash-recovery model) relaunches the daemon, which
|
|
22
|
+
* then boots the new code. Clients do not need to be told anything: browser
|
|
23
|
+
* IPC clients re-probe the socket via `waitForBrowserService`
|
|
24
|
+
* (`browser/ipc.ts`), and the scheduler's atomic `(routine, scheduledFor)`
|
|
25
|
+
* claim (see `docs/specifications.md` §Scheduling & execution singularity)
|
|
26
|
+
* means a routine mid-fire at the moment of exit is deduped safely across the
|
|
27
|
+
* restart rather than double-fired.
|
|
28
|
+
*
|
|
29
|
+
* Fail-closed is the whole point: every step below that can fail — the
|
|
30
|
+
* registry check, the install, the post-install verify — leaves the OLD
|
|
31
|
+
* daemon running untouched and logs a WARN/ERROR for the next tick to retry.
|
|
32
|
+
* The daemon must never exit into code it has not proven is the real,
|
|
33
|
+
* verified, requested version; an unverified exit would let the OS supervisor
|
|
34
|
+
* relaunch-loop on a broken install.
|
|
35
|
+
*/
|
|
36
|
+
import { BasePeriodicService, type DaemonContext } from './service.js';
|
|
37
|
+
import type { DaemonServiceId } from '../daemon-services.js';
|
|
38
|
+
/**
|
|
39
|
+
* Hard cap per tick: a real download + npm/bun install + verify can
|
|
40
|
+
* legitimately take minutes on a slow link. 15 minutes matches the task's
|
|
41
|
+
* stated budget and is short relative to the ~75min cadence. Exported so the
|
|
42
|
+
* on-demand `request-self-update` IPC handler (`browser/ipc.ts`) can bound
|
|
43
|
+
* its own `AbortController` on the SAME budget the periodic tick runs under —
|
|
44
|
+
* one deadline, not two independently-tuned numbers that could drift apart.
|
|
45
|
+
*/
|
|
46
|
+
export declare const SELF_UPDATE_DEADLINE_MS: number;
|
|
47
|
+
export interface SelfUpdateOutcome {
|
|
48
|
+
updated: boolean;
|
|
49
|
+
reason?: string;
|
|
50
|
+
}
|
|
51
|
+
interface NpmLatestMetadata {
|
|
52
|
+
version: string;
|
|
53
|
+
integrity: string;
|
|
54
|
+
tarball: string;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Dependency seam so `self-update-service.test.ts` can drive a REAL install
|
|
58
|
+
* against a fixture npm prefix/tarball without touching the real npm
|
|
59
|
+
* registry or the real running install. Production code always uses
|
|
60
|
+
* {@link defaultSelfUpdateDeps} (the default parameter below) — no test-only
|
|
61
|
+
* branch exists in the exported logic itself.
|
|
62
|
+
*/
|
|
63
|
+
export interface SelfUpdateDeps {
|
|
64
|
+
currentVersion(): string;
|
|
65
|
+
isDevBuild(): boolean;
|
|
66
|
+
detectShadow(): boolean;
|
|
67
|
+
packageRoot(): string;
|
|
68
|
+
fetchLatestMetadata(signal: AbortSignal): Promise<NpmLatestMetadata>;
|
|
69
|
+
installAndVerify(metadata: NpmLatestMetadata, packageRoot: string, signal: AbortSignal): Promise<void>;
|
|
70
|
+
syncSystemRepo(): Promise<void>;
|
|
71
|
+
syncLocal(): Promise<void>;
|
|
72
|
+
}
|
|
73
|
+
export declare function fetchLatestNpmMetadata(signal: AbortSignal): Promise<NpmLatestMetadata>;
|
|
74
|
+
/**
|
|
75
|
+
* Install + byte-verify `metadata` into `packageRoot`'s install, exactly the
|
|
76
|
+
* sequence `bootstrap.ts`'s `installResolvedPackage` runs for `agents
|
|
77
|
+
* upgrade` (download+integrity-verify -> sweep stale staging -> package-
|
|
78
|
+
* manager install -> verify installed version -> refresh alias shims).
|
|
79
|
+
* `bootstrap.ts` cannot be imported here — it runs side-effecting top-level
|
|
80
|
+
* code (argv parsing, command registration) on import, which the daemon must
|
|
81
|
+
* never trigger — so this is the same primitives from `self-update.ts`
|
|
82
|
+
* composed directly, not a fork of the upgrade logic.
|
|
83
|
+
*
|
|
84
|
+
* `signal` is threaded into EVERY step's own `signal` option
|
|
85
|
+
* (`downloadVerifiedTarball` / `installPackageIntoPrefix` /
|
|
86
|
+
* `installPackageWithBun`, all in `self-update.ts`) rather than raced against
|
|
87
|
+
* from the outside: those primitives kill the underlying fetch/child process
|
|
88
|
+
* on abort and their promise rejects only once that real cancellation has
|
|
89
|
+
* happened. A wrapper that merely stopped AWAITING an unkillable operation
|
|
90
|
+
* (the prior approach here) left an orphaned `npm install -g`/`bun add -g`
|
|
91
|
+
* writing into the shared global prefix — which the very next tick's fresh
|
|
92
|
+
* install (started as soon as the supervisor's backoff fires, seconds later)
|
|
93
|
+
* would then race into the same directory. Real cancellation is what makes
|
|
94
|
+
* `attemptSelfUpdateAndExit`'s `inFlightAttempt` dedupe (below) an actual
|
|
95
|
+
* guarantee instead of a guard whose lifetime is shorter than the operation
|
|
96
|
+
* it's guarding (found in review, PHNX-3695).
|
|
97
|
+
*/
|
|
98
|
+
export declare function installAndVerifyDefault(metadata: NpmLatestMetadata, packageRoot: string, signal: AbortSignal): Promise<void>;
|
|
99
|
+
export declare function defaultSelfUpdateDeps(): SelfUpdateDeps;
|
|
100
|
+
/**
|
|
101
|
+
* Core self-update decision + action, shared by the periodic tick
|
|
102
|
+
* ({@link SelfUpdateService.onTick}) and the on-demand IPC path
|
|
103
|
+
* (`request-self-update`, `browser/ipc.ts`) — one implementation, so a
|
|
104
|
+
* version-skew client asking "update now" runs exactly the same fail-closed
|
|
105
|
+
* logic as the scheduled sweep. Returns rather than throws so callers decide
|
|
106
|
+
* their own exit timing (the periodic service exits immediately; the IPC
|
|
107
|
+
* handler must respond to the client on the socket BEFORE exiting, or the
|
|
108
|
+
* client hangs on a socket that is closing mid-write). Concurrent callers
|
|
109
|
+
* share one in-flight attempt rather than racing separate installs.
|
|
110
|
+
*/
|
|
111
|
+
export declare function attemptSelfUpdateAndExit(ctx: DaemonContext, signal: AbortSignal, deps?: SelfUpdateDeps): Promise<SelfUpdateOutcome>;
|
|
112
|
+
/**
|
|
113
|
+
* Schedule the process exit for a verified self-update, exactly once, no
|
|
114
|
+
* matter how many callers observe `outcome.updated` on the shared
|
|
115
|
+
* `inFlightAttempt` promise. The periodic tick and an on-demand
|
|
116
|
+
* `request-self-update` IPC call (`browser/ipc.ts`) can both be awaiting that
|
|
117
|
+
* SAME promise — if the tick's continuation ran an immediate `process.exit(0)`
|
|
118
|
+
* while the IPC handler's continuation had not yet reached `socket.write`,
|
|
119
|
+
* the tick's exit could win the race and the client would see a closed socket
|
|
120
|
+
* before any response (found in review, PHNX-3695). Routing every caller
|
|
121
|
+
* through this one guarded, always-delayed scheduling point means the delay
|
|
122
|
+
* protects EVERY caller's in-flight response, not just the IPC handler's own.
|
|
123
|
+
*/
|
|
124
|
+
export declare function scheduleSelfUpdateExit(): void;
|
|
125
|
+
/**
|
|
126
|
+
* Fire the on-demand self-update in the BACKGROUND and return its (bounded)
|
|
127
|
+
* promise WITHOUT the caller having to await it. This is what keeps the
|
|
128
|
+
* `request-self-update` IPC handler (`browser/ipc.ts`) from parking a
|
|
129
|
+
* version-skewed `agents browser` verb behind the full
|
|
130
|
+
* check→download→install→verify: that handler routes through
|
|
131
|
+
* `reconcileDaemonVersion` on every version-skewed call, so awaiting the whole
|
|
132
|
+
* install there reintroduces exactly the client-stall PHNX-3605 was written to
|
|
133
|
+
* prevent (tens of seconds, worst case ~15 min). The handler instead responds
|
|
134
|
+
* "triggered" immediately and lets this run in the background — the daemon does
|
|
135
|
+
* install→verify→exit(0) on its own, the OS supervisor relaunches it, and the
|
|
136
|
+
* browser reconnects.
|
|
137
|
+
*
|
|
138
|
+
* The work still shares the module-level {@link attemptSelfUpdateAndExit}
|
|
139
|
+
* `inFlightAttempt` guard, so a concurrent trigger (or the periodic tick) can't
|
|
140
|
+
* race a second install into the same prefix. It is bounded by
|
|
141
|
+
* {@link SELF_UPDATE_DEADLINE_MS} via an `AbortController` nobody awaits (the
|
|
142
|
+
* timer is `unref`'d so it never keeps the daemon alive on its own and never
|
|
143
|
+
* dangles in a test). Fail-closed is preserved end to end:
|
|
144
|
+
* `runSelfUpdateAttempt` already turns an install/verify failure into a
|
|
145
|
+
* not-updated outcome that leaves the running daemon untouched. The caller
|
|
146
|
+
* schedules the one decoupled {@link scheduleSelfUpdateExit} off the returned
|
|
147
|
+
* promise once `updated` is true, so the exit still fires after the IPC
|
|
148
|
+
* response has flushed.
|
|
149
|
+
*/
|
|
150
|
+
export declare function triggerSelfUpdateInBackground(ctx: DaemonContext, deps?: SelfUpdateDeps): Promise<SelfUpdateOutcome>;
|
|
151
|
+
/**
|
|
152
|
+
* The subset of self-update decline checks that are INSTANT and network-free —
|
|
153
|
+
* a dev build, or a shadowed install. The on-demand IPC handler
|
|
154
|
+
* (`request-self-update`) runs these SYNCHRONOUSLY so a version-skewed browser
|
|
155
|
+
* client gets the PHNX-3605 "nothing changed, not evicting" advisory
|
|
156
|
+
* immediately, instead of a "triggered" it would never act on. The remaining
|
|
157
|
+
* checks (registry probe, already-current, install/verify) stay inside the
|
|
158
|
+
* backgrounded {@link attemptSelfUpdateAndExit} so the handler never blocks on
|
|
159
|
+
* the network or the install. This is the single source of the two instant
|
|
160
|
+
* decline reasons — {@link runSelfUpdateAttempt} calls it too, so the on-demand
|
|
161
|
+
* decline text can never drift from the periodic tick's. Returns the decline
|
|
162
|
+
* reason, or `null` to proceed to the (backgrounded) install path.
|
|
163
|
+
*/
|
|
164
|
+
export declare function selfUpdateSyncDeclineReason(deps?: SelfUpdateDeps): string | null;
|
|
165
|
+
export declare class SelfUpdateService extends BasePeriodicService {
|
|
166
|
+
readonly id: DaemonServiceId;
|
|
167
|
+
readonly intervalMs: number;
|
|
168
|
+
readonly deadlineMs: number;
|
|
169
|
+
readonly startupDelayMs: number;
|
|
170
|
+
protected onStart(_ctx: DaemonContext): Promise<void>;
|
|
171
|
+
protected onStop(): Promise<void>;
|
|
172
|
+
protected onTick(ctx: DaemonContext, signal: AbortSignal): Promise<void>;
|
|
173
|
+
}
|
|
174
|
+
export {};
|