@phnx-labs/agents-cli 1.22.69 → 1.22.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +34 -9
- package/dist/bootstrap.js +4 -4
- package/dist/commands/accounts.js +23 -7
- package/dist/commands/exec.js +2 -2
- package/dist/commands/import.js +2 -2
- package/dist/commands/models.js +2 -2
- package/dist/commands/permissions.js +2 -2
- package/dist/commands/repo.js +2 -2
- package/dist/commands/rules.js +1 -1
- package/dist/commands/send.js +9 -3
- package/dist/commands/sessions-export.d.ts +5 -1
- package/dist/commands/sessions-export.js +100 -24
- package/dist/commands/sessions-import.d.ts +2 -1
- package/dist/commands/sessions-import.js +85 -21
- package/dist/commands/traces.js +1 -1
- package/dist/lib/account-capabilities.js +0 -2
- package/dist/lib/account-registry.d.ts +3 -3
- package/dist/lib/account-registry.js +25 -7
- package/dist/lib/accounting/usage-sync.d.ts +1 -1
- package/dist/lib/accounting/usage-sync.js +3 -3
- package/dist/lib/acp/client.d.ts +1 -1
- package/dist/lib/acp/client.js +12 -1
- package/dist/lib/acp/harnesses.js +1 -1
- package/dist/lib/add-dir.js +0 -2
- package/dist/lib/agent-cli-commands.js +0 -2
- package/dist/lib/agent-spec/agents.d.ts +1 -1
- package/dist/lib/agent-spec/agents.js +2 -83
- package/dist/lib/browser/ipc.d.ts +34 -0
- package/dist/lib/browser/ipc.js +149 -20
- package/dist/lib/browser/remote-control.d.ts +6 -3
- package/dist/lib/browser/remote-control.js +6 -3
- package/dist/lib/browser/service.d.ts +4 -1
- package/dist/lib/browser/service.js +7 -1
- package/dist/lib/browser/types.d.ts +3 -1
- package/dist/lib/channels/registry.d.ts +6 -0
- package/dist/lib/channels/send.d.ts +11 -2
- package/dist/lib/channels/send.js +11 -2
- package/dist/lib/cloud/rush.d.ts +10 -2
- package/dist/lib/cloud/rush.js +15 -9
- package/dist/lib/daemon/auth-sync-service.js +1 -1
- package/dist/lib/daemon/browser-task-reap-service.js +1 -1
- package/dist/lib/daemon/daemon.js +41 -11
- package/dist/lib/daemon/heartbeat-service.js +3 -3
- package/dist/lib/daemon/keychain-reap-service.js +1 -1
- package/dist/lib/daemon/runner.d.ts +18 -1
- package/dist/lib/daemon/runner.js +237 -80
- package/dist/lib/daemon/self-heal-service.js +13 -3
- package/dist/lib/daemon/self-update-service.d.ts +174 -0
- package/dist/lib/daemon/self-update-service.js +353 -0
- package/dist/lib/daemon/state-dir-check-service.js +3 -3
- package/dist/lib/daemon/usage-sync-service.js +1 -1
- package/dist/lib/daemon/watchdog-service.js +4 -4
- package/dist/lib/daemon-services.d.ts +1 -1
- package/dist/lib/daemon-services.js +5 -0
- package/dist/lib/device-config.d.ts +12 -1
- package/dist/lib/device-config.js +63 -13
- package/dist/lib/exec-bounded.d.ts +52 -0
- package/dist/lib/exec-bounded.js +113 -0
- package/dist/lib/exec.d.ts +2 -2
- package/dist/lib/exec.js +3 -34
- package/dist/lib/feed/events.d.ts +22 -14
- package/dist/lib/feed/events.js +84 -44
- package/dist/lib/feed-broadcast.d.ts +12 -29
- package/dist/lib/feed-broadcast.js +28 -27
- package/dist/lib/fleet-shared-state.d.ts +12 -5
- package/dist/lib/fleet-shared-state.js +50 -20
- package/dist/lib/fs-atomic.d.ts +11 -0
- package/dist/lib/fs-atomic.js +60 -0
- package/dist/lib/hooks/install.js +0 -87
- package/dist/lib/hosts/reconcile.d.ts +11 -4
- package/dist/lib/hosts/reconcile.js +31 -5
- package/dist/lib/installations/strategies.js +1 -1
- package/dist/lib/mcp-registry.js +0 -13
- package/dist/lib/mcp.js +2 -2
- package/dist/lib/model-tiers.js +1 -1
- package/dist/lib/models.js +0 -63
- package/dist/lib/notify.d.ts +11 -0
- package/dist/lib/notify.js +17 -4
- package/dist/lib/owner-message.d.ts +46 -3
- package/dist/lib/owner-message.js +26 -6
- package/dist/lib/permissions-registry.d.ts +0 -2
- package/dist/lib/permissions-registry.js +3 -50
- package/dist/lib/permissions.d.ts +3 -17
- package/dist/lib/permissions.js +4 -73
- package/dist/lib/project-resources.d.ts +12 -0
- package/dist/lib/project-resources.js +129 -0
- package/dist/lib/routine-process-cleanup.d.ts +2 -2
- package/dist/lib/routine-process-cleanup.js +45 -34
- package/dist/lib/rush-session.d.ts +19 -0
- package/dist/lib/rush-session.js +24 -0
- package/dist/lib/secrets/drivers/rush.js +2 -1
- package/dist/lib/secrets/reaper.d.ts +2 -2
- package/dist/lib/secrets/reaper.js +13 -10
- package/dist/lib/secrets/reserved-sync.d.ts +1 -1
- package/dist/lib/secrets/reserved-sync.js +4 -4
- package/dist/lib/self-update.d.ts +21 -8
- package/dist/lib/self-update.js +54 -31
- package/dist/lib/session/cloud.js +2 -1
- package/dist/lib/session/sync/backend.d.ts +61 -0
- package/dist/lib/session/sync/backend.js +89 -0
- package/dist/lib/session/sync/managed-config.d.ts +29 -0
- package/dist/lib/session/sync/managed-config.js +23 -0
- package/dist/lib/session/sync/managed-key.d.ts +45 -0
- package/dist/lib/session/sync/managed-key.js +128 -0
- package/dist/lib/session/sync/net-client.d.ts +65 -0
- package/dist/lib/session/sync/net-client.js +117 -0
- package/dist/lib/session/sync/provision.d.ts +19 -0
- package/dist/lib/session/sync/provision.js +38 -0
- package/dist/lib/session/sync/r2.d.ts +5 -2
- package/dist/lib/session/sync/r2.js +5 -2
- package/dist/lib/session/sync/worker-template.d.ts +6 -0
- package/dist/lib/session/sync/worker-template.js +847 -0
- package/dist/lib/sink-format.d.ts +34 -0
- package/dist/lib/sink-format.js +17 -0
- package/dist/lib/staleness/detectors/permissions.js +0 -20
- package/dist/lib/staleness/writers/commands.js +1 -1
- package/dist/lib/staleness/writers/hooks.js +2 -2
- package/dist/lib/subagents-registry.js +2 -12
- package/dist/lib/subagents.d.ts +0 -10
- package/dist/lib/subagents.js +0 -37
- package/dist/lib/tmux/orphan-reap.js +6 -4
- package/dist/lib/tmux/session.js +4 -1
- package/dist/lib/traces/classify.d.ts +8 -1
- package/dist/lib/traces/insights.d.ts +13 -1
- package/dist/lib/traces/insights.js +78 -3
- package/dist/lib/traces/sync.js +8 -3
- package/dist/lib/traces/worker-template.js +9 -5
- package/dist/lib/types.d.ts +1 -1
- package/package.json +1 -1
package/dist/lib/cloud/rush.js
CHANGED
|
@@ -10,6 +10,7 @@ import * as os from 'os';
|
|
|
10
10
|
import * as crypto from 'crypto';
|
|
11
11
|
import * as yaml from 'yaml';
|
|
12
12
|
import { resolveDispatchRepos, normalizeProviderStatus, MAX_IMAGES_PER_DISPATCH } from './types.js';
|
|
13
|
+
import { isRushSessionExpired } from '../rush-session.js';
|
|
13
14
|
import { parseSSE } from './stream.js';
|
|
14
15
|
import { listInstalledVersions, getVersionHomePath } from '../installations/versions.js';
|
|
15
16
|
import { getAccountInfo } from '../agents.js';
|
|
@@ -19,8 +20,9 @@ const USER_YAML = path.join(os.homedir(), '.rush', 'user.yaml');
|
|
|
19
20
|
/**
|
|
20
21
|
* Returns true when ~/.rush/user.yaml exists, carries an access_token, and
|
|
21
22
|
* the token has not passed its expires_at timestamp (Unix seconds). A missing
|
|
22
|
-
* expires_at
|
|
23
|
-
*
|
|
23
|
+
* expires_at, or `expires_at: 0` (a non-expiring Phoenix `pid_` bearer), is
|
|
24
|
+
* treated as non-expired (see isRushSessionExpired, PHNX-3645). Pass yamlPath
|
|
25
|
+
* to override the default path in tests.
|
|
24
26
|
*/
|
|
25
27
|
export function isRushSessionValid(yamlPath = USER_YAML) {
|
|
26
28
|
try {
|
|
@@ -30,8 +32,7 @@ export function isRushSessionValid(yamlPath = USER_YAML) {
|
|
|
30
32
|
const data = yaml.parse(raw);
|
|
31
33
|
if (!data?.session?.access_token)
|
|
32
34
|
return false;
|
|
33
|
-
|
|
34
|
-
if (typeof expiresAt === 'number' && expiresAt <= Date.now() / 1000)
|
|
35
|
+
if (isRushSessionExpired(data.session.expires_at))
|
|
35
36
|
return false;
|
|
36
37
|
return true;
|
|
37
38
|
}
|
|
@@ -39,19 +40,24 @@ export function isRushSessionValid(yamlPath = USER_YAML) {
|
|
|
39
40
|
return false;
|
|
40
41
|
}
|
|
41
42
|
}
|
|
42
|
-
/**
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Read the Rush session access token from ~/.rush/user.yaml. Exported (with an
|
|
45
|
+
* overridable yamlPath, like isRushSessionValid) so the freshness behavior —
|
|
46
|
+
* including the `expires_at: 0` non-expiring case (PHNX-3645) — is directly
|
|
47
|
+
* testable; the class methods call it with the default path.
|
|
48
|
+
*/
|
|
49
|
+
export function readToken(yamlPath = USER_YAML) {
|
|
50
|
+
if (!fs.existsSync(yamlPath)) {
|
|
45
51
|
throw new Error('Not logged in to Rush. Run `rush login` first.');
|
|
46
52
|
}
|
|
47
|
-
const raw = fs.readFileSync(
|
|
53
|
+
const raw = fs.readFileSync(yamlPath, 'utf-8');
|
|
48
54
|
const data = yaml.parse(raw);
|
|
49
55
|
const token = data?.session?.access_token;
|
|
50
56
|
if (!token) {
|
|
51
57
|
throw new Error('No session token in ~/.rush/user.yaml. Run `rush login` first.');
|
|
52
58
|
}
|
|
53
59
|
const expiresAt = data.session?.expires_at;
|
|
54
|
-
if (
|
|
60
|
+
if (isRushSessionExpired(expiresAt)) {
|
|
55
61
|
const expiredAt = new Date(expiresAt * 1000).toISOString();
|
|
56
62
|
throw new Error(`Rush session expired at ${expiredAt}. Run \`rush login\` to refresh.`);
|
|
57
63
|
}
|
|
@@ -23,7 +23,7 @@ export class AuthSyncService extends BasePeriodicService {
|
|
|
23
23
|
}
|
|
24
24
|
async onTick(ctx) {
|
|
25
25
|
const { publishReservedAuthVerdict, syncReservedAuthBundle } = await import('../secrets/reserved-sync.js');
|
|
26
|
-
const published = publishReservedAuthVerdict();
|
|
26
|
+
const published = await publishReservedAuthVerdict();
|
|
27
27
|
if (published.error)
|
|
28
28
|
ctx.log('WARN', `auth-sync: verdict: ${published.error}`);
|
|
29
29
|
const { syncFleetSharedStateRepo } = await import('../fleet-shared-repo-sync.js');
|
|
@@ -15,7 +15,7 @@ export class BrowserTaskReapService extends BasePeriodicService {
|
|
|
15
15
|
async onStart(_ctx) { }
|
|
16
16
|
async onStop() { }
|
|
17
17
|
async onTick(ctx) {
|
|
18
|
-
const result = await this.browserService.reapAbandoned({ idleMs: resolveBrowserTaskIdleMs() });
|
|
18
|
+
const result = await this.browserService.reapAbandoned({ idleMs: await resolveBrowserTaskIdleMs() });
|
|
19
19
|
if (result.closed.length > 0) {
|
|
20
20
|
ctx.log('INFO', `Browser-task reaper: closed ${result.closed.length} task(s)`);
|
|
21
21
|
for (const closed of result.closed) {
|
|
@@ -40,6 +40,7 @@ import { BrowserIPCService } from './browser-ipc-service.js';
|
|
|
40
40
|
import { WatchdogService } from './watchdog-service.js';
|
|
41
41
|
import { DeviceProbeService } from './device-probe-service.js';
|
|
42
42
|
import { SelfHealService } from './self-heal-service.js';
|
|
43
|
+
import { SelfUpdateService } from './self-update-service.js';
|
|
43
44
|
import { KeychainReapService } from './keychain-reap-service.js';
|
|
44
45
|
import { AuthSyncService } from './auth-sync-service.js';
|
|
45
46
|
import { UsageSyncService } from './usage-sync-service.js';
|
|
@@ -49,7 +50,7 @@ import { WebhookReceiverService } from './webhook-receiver-service.js';
|
|
|
49
50
|
import { HeartbeatService } from './heartbeat-service.js';
|
|
50
51
|
import { TmuxReapService } from './tmux-reap-service.js';
|
|
51
52
|
import { BrowserTaskReapService } from './browser-task-reap-service.js';
|
|
52
|
-
import { emit, emitRoutineEnd } from '../feed/events.js';
|
|
53
|
+
import { emit, emitAsync, emitRoutineEnd } from '../feed/events.js';
|
|
53
54
|
import { readDaemonServicesConfig, drainDaemonServiceRestartQueue } from '../daemon-services.js';
|
|
54
55
|
import { sleepSync } from '../fs-atomic.js';
|
|
55
56
|
/**
|
|
@@ -749,11 +750,16 @@ export function log(level, message) {
|
|
|
749
750
|
: lvl === 'START' || /starting|started/i.test(message) ? 'daemon.start'
|
|
750
751
|
: lvl === 'STOP' || /stopping|stopped|shutting down/i.test(message) ? 'daemon.stop'
|
|
751
752
|
: 'daemon.info';
|
|
752
|
-
|
|
753
|
+
// Fire-and-forget the event mirror: `log()` is a synchronous primitive on
|
|
754
|
+
// every daemon tick's `ctx.log`, and the mirror's event-log lock would
|
|
755
|
+
// otherwise block the shared event loop for up to 30s under contention
|
|
756
|
+
// (PHNX-3695). The daemon-log append above is the primary, synchronous sink;
|
|
757
|
+
// the mirror is best-effort, so a fire-and-forget async write is correct.
|
|
758
|
+
void emitAsync(event, {
|
|
753
759
|
module: 'daemon',
|
|
754
760
|
detail: redactSecrets(message).slice(0, 500),
|
|
755
761
|
status: lvl,
|
|
756
|
-
});
|
|
762
|
+
}).catch(() => { });
|
|
757
763
|
}
|
|
758
764
|
catch { /* never crash the daemon on event-log failure */ }
|
|
759
765
|
}
|
|
@@ -1073,6 +1079,10 @@ export async function runDaemon() {
|
|
|
1073
1079
|
supervisor.register(new SelfHealService());
|
|
1074
1080
|
else
|
|
1075
1081
|
log('INFO', 'Self-heal service disabled');
|
|
1082
|
+
if (isEnabled('self-update'))
|
|
1083
|
+
supervisor.register(new SelfUpdateService());
|
|
1084
|
+
else
|
|
1085
|
+
log('INFO', 'Self-update service disabled');
|
|
1076
1086
|
if (isEnabled('keychain-reap'))
|
|
1077
1087
|
supervisor.register(new KeychainReapService());
|
|
1078
1088
|
else
|
|
@@ -2249,7 +2259,15 @@ function stopDaemonLocked() {
|
|
|
2249
2259
|
// any service-manager teardown or direct signal and repairs only under lock.
|
|
2250
2260
|
const pid = resolveLiveDaemonPid(true);
|
|
2251
2261
|
const browserSock = process.platform === 'win32' ? null : getBrowserIpcSocketPath();
|
|
2252
|
-
|
|
2262
|
+
// Capture the socket's inode INDEPENDENTLY of whether a live pid resolved
|
|
2263
|
+
// (PHNX-3618). A daemon that died between the CLI liveness precheck and this
|
|
2264
|
+
// locked read leaves resolveLiveDaemonPid() returning null while its ungraceful
|
|
2265
|
+
// exit left the binding on disk — capturing only when `pid !== null` meant that
|
|
2266
|
+
// genuinely stale socket could never be reclaimed and was reported "ownership
|
|
2267
|
+
// could not be verified". The inode is the successor guard: a fresh daemon that
|
|
2268
|
+
// rebinds during the stop gets a new inode, so a later identity match still
|
|
2269
|
+
// proves this exact binding is the one we captured, never a successor's.
|
|
2270
|
+
const browserSockOwner = browserSock ? readPathIdentity(browserSock) : null;
|
|
2253
2271
|
if (platform === 'darwin') {
|
|
2254
2272
|
const plistPath = getLaunchdPlistPath();
|
|
2255
2273
|
if (fs.existsSync(plistPath)) {
|
|
@@ -2380,8 +2398,19 @@ function stopDaemonLocked() {
|
|
|
2380
2398
|
// ungracefully (killTree) and left a stale binding — the owner is provably
|
|
2381
2399
|
// dead, so reclaim it and report.
|
|
2382
2400
|
if (process.platform !== 'win32') {
|
|
2401
|
+
// A binding is provably stale — and reclaimable — when the inode on disk is
|
|
2402
|
+
// still the exact one captured under this lock AND no daemon (the signalled
|
|
2403
|
+
// target OR any successor for this state dir) survives to own it. Keying the
|
|
2404
|
+
// "a daemon still owns it" test on the whole survivor set rather than only the
|
|
2405
|
+
// captured target is what lets a daemon that died BEFORE this stop (pid ===
|
|
2406
|
+
// null, PHNX-3618) still have its stale socket reclaimed, while a live
|
|
2407
|
+
// successor — which either rebound the inode or shows up as a survivor — is
|
|
2408
|
+
// never touched (PHNX-3607's never-delete-a-successor invariant).
|
|
2409
|
+
const aDaemonSurvives = survivors.length > 0 || unverifiedSurvivors.length > 0;
|
|
2383
2410
|
if (browserSock && fs.existsSync(browserSock)) {
|
|
2384
|
-
|
|
2411
|
+
const ownsCapturedBinding = browserSockOwner !== null
|
|
2412
|
+
&& pathIdentityMatches(browserSock, browserSockOwner);
|
|
2413
|
+
if (ownsCapturedBinding && !aDaemonSurvives) {
|
|
2385
2414
|
try {
|
|
2386
2415
|
fs.unlinkSync(browserSock);
|
|
2387
2416
|
}
|
|
@@ -2393,13 +2422,14 @@ function stopDaemonLocked() {
|
|
|
2393
2422
|
else
|
|
2394
2423
|
released.push('browser IPC socket (reclaimed)');
|
|
2395
2424
|
}
|
|
2425
|
+
else if (aDaemonSurvives) {
|
|
2426
|
+
surviving.push('browser IPC socket still owned by a surviving daemon');
|
|
2427
|
+
}
|
|
2396
2428
|
else {
|
|
2397
|
-
// The path was absent when
|
|
2398
|
-
//
|
|
2399
|
-
// invocation does not own the current binding and must leave it
|
|
2400
|
-
surviving.push(
|
|
2401
|
-
? 'browser IPC socket still owned by a surviving daemon'
|
|
2402
|
-
: 'browser IPC socket ownership could not be verified');
|
|
2429
|
+
// The path was absent when this transaction captured the target (a
|
|
2430
|
+
// successor bound it afterward) or a successor replaced the inode. Either
|
|
2431
|
+
// way this invocation does not own the current binding and must leave it.
|
|
2432
|
+
surviving.push('browser IPC socket ownership could not be verified');
|
|
2403
2433
|
}
|
|
2404
2434
|
}
|
|
2405
2435
|
else {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** Daemon heartbeat and routine-process reconciliation under supervision. */
|
|
2
|
-
import {
|
|
2
|
+
import { reapExitedRunningJobs } from './runner.js';
|
|
3
3
|
import { reapTerminalRoutineProcesses } from '../routine-process-cleanup.js';
|
|
4
4
|
import { BasePeriodicService } from './service.js';
|
|
5
5
|
const HEARTBEAT_TICK_MS = 60_000;
|
|
@@ -17,8 +17,8 @@ export class HeartbeatService extends BasePeriodicService {
|
|
|
17
17
|
async onStop() { }
|
|
18
18
|
async onTick(ctx) {
|
|
19
19
|
this.publishHeartbeat();
|
|
20
|
-
|
|
21
|
-
const reaped = reapTerminalRoutineProcesses();
|
|
20
|
+
await reapExitedRunningJobs();
|
|
21
|
+
const reaped = await reapTerminalRoutineProcesses();
|
|
22
22
|
if (reaped.length > 0) {
|
|
23
23
|
ctx.log('WARN', `Reaped ${reaped.length} terminal routine process group(s): ${reaped.join(', ')}`);
|
|
24
24
|
}
|
|
@@ -22,7 +22,7 @@ export class KeychainReapService extends BasePeriodicService {
|
|
|
22
22
|
}
|
|
23
23
|
async onTick(ctx) {
|
|
24
24
|
const { reapOrphanedKeychainProcesses } = await import('../secrets/reaper.js');
|
|
25
|
-
const result = reapOrphanedKeychainProcesses();
|
|
25
|
+
const result = await reapOrphanedKeychainProcesses();
|
|
26
26
|
if (result.reaped > 0) {
|
|
27
27
|
ctx.log('WARN', `Reaped ${result.reaped} keychain orphan/stuck process(es)`);
|
|
28
28
|
for (const d of result.details)
|
|
@@ -251,6 +251,23 @@ export declare function listLiveRoutineChildren(): number[];
|
|
|
251
251
|
* remote `.exit` is reconciled (`finalizeHostRun`).
|
|
252
252
|
*/
|
|
253
253
|
export declare function isRunGenuinelyInFlight(meta: RunMeta): boolean;
|
|
254
|
-
/**
|
|
254
|
+
/**
|
|
255
|
+
* Scan all runs marked "running" and finalize any whose process has exited.
|
|
256
|
+
*
|
|
257
|
+
* SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
|
|
258
|
+
* builders and CLI best-effort orphan reaps), which run in short-lived CLI
|
|
259
|
+
* processes where a bounded synchronous `ps`/scan cannot starve a served event
|
|
260
|
+
* loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
|
|
261
|
+
* instead — a sync scan there would freeze the shared event loop (PHNX-3695).
|
|
262
|
+
*/
|
|
255
263
|
export declare function monitorRunningJobs(): void;
|
|
264
|
+
/**
|
|
265
|
+
* Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
|
|
266
|
+
* heartbeat tick (PHNX-3695). Identical reconciliation via
|
|
267
|
+
* {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
|
|
268
|
+
* the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
|
|
269
|
+
* the every-tick scan never freezes the shared event loop — the freeze that
|
|
270
|
+
* left the browser IPC `version` probe unanswered.
|
|
271
|
+
*/
|
|
272
|
+
export declare function reapExitedRunningJobs(): Promise<void>;
|
|
256
273
|
export {};
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { spawn, execFileSync } from 'child_process';
|
|
17
17
|
import * as fs from 'fs';
|
|
18
|
+
import * as fsp from 'fs/promises';
|
|
18
19
|
import * as path from 'path';
|
|
19
20
|
import * as os from 'os';
|
|
20
21
|
import { getCliLaunch, getAgentsBinDir } from '../cli-entry.js';
|
|
@@ -23,14 +24,15 @@ import { getRunsDir, getUserAgentsDir, readMeta, getDaemonDir } from '../state.j
|
|
|
23
24
|
import { shortCodexHome } from '../codex-home.js';
|
|
24
25
|
import { prepareJobHome, buildSpawnEnv, getJobHomePath, assertSandboxForwardsHostGhAuth, linkVersionAuth } from '../sandbox.js';
|
|
25
26
|
import { resolveModel, buildReasoningFlags } from '../models.js';
|
|
26
|
-
import { createTimer, redactPrompt, emitRoutineEnd } from '../feed/events.js';
|
|
27
|
+
import { createTimer, redactPrompt, emitRoutineEnd, emitRoutineEndAsync } from '../feed/events.js';
|
|
27
28
|
import { resolveHarnessAdapter } from '../harness/index.js';
|
|
28
29
|
import { applyAddDirs } from '../add-dir.js';
|
|
29
30
|
import { normalizeMode, resolveHeadlessMode, buildExecEnv, detectRateLimit, detectAuthFailure, isAuthFailureFromLog, authFailureReason, AGENT_COMMANDS, } from '../exec.js';
|
|
30
31
|
import { resolveActor } from '../actor.js';
|
|
31
32
|
import { loadTask as loadHostTask } from '../hosts/tasks.js';
|
|
32
|
-
import { reconcileTask as reconcileHostTask } from '../hosts/reconcile.js';
|
|
33
|
-
import { backgroundSpawnOptions, killTree } from '../platform/process.js';
|
|
33
|
+
import { reconcileTask as reconcileHostTask, reconcileTaskAsync as reconcileHostTaskAsync } from '../hosts/reconcile.js';
|
|
34
|
+
import { backgroundSpawnOptions, isAlive, killTree } from '../platform/process.js';
|
|
35
|
+
import { execFileBounded } from '../exec-bounded.js';
|
|
34
36
|
import lockfile from 'proper-lockfile';
|
|
35
37
|
import { ensureLockTarget } from '../fs-atomic.js';
|
|
36
38
|
import { logAndContinueOnLockCompromised } from '../lock-compromise.js';
|
|
@@ -922,8 +924,12 @@ export async function resolveRoutineLaunch(config, cwd = process.cwd(), deps = {
|
|
|
922
924
|
if (config.account && !explicitCredential) {
|
|
923
925
|
// A native routine account is named by its durable name; the version matcher
|
|
924
926
|
// keys on the identity (email/accountKey), so translate before resolving,
|
|
925
|
-
// and refuse a login that belongs to a different harness.
|
|
926
|
-
|
|
927
|
+
// and refuse a login that belongs to a different harness. Scope the lookup to
|
|
928
|
+
// the routine's own harness: `identityLabel` defaults to the login's email, so
|
|
929
|
+
// `account: <email>` matches every harness that identity is signed into, and
|
|
930
|
+
// un-scoped this resolved whichever row the store ordered first — then rejected
|
|
931
|
+
// it on the very next line.
|
|
932
|
+
const unified = findUnifiedAccount(config.account, meta, undefined, agent);
|
|
927
933
|
if (unified?.kind === 'native' && unified.agent !== agent) {
|
|
928
934
|
throw new Error(`Routine '${config.name}' account '${config.account}' is a ${unified.agent} login and cannot authenticate ${agent}.`);
|
|
929
935
|
}
|
|
@@ -2224,6 +2230,40 @@ const MAX_WALL_CLOCK_MS = 24 * 60 * 60 * 1000;
|
|
|
2224
2230
|
* compares against the process's actual start time via `ps`. Returns true
|
|
2225
2231
|
* when the PID is alive AND plausibly ours.
|
|
2226
2232
|
*/
|
|
2233
|
+
/** Bound the identity `ps` — a hung `ps` must never freeze its caller for long (matches routine-process-cleanup's probe). */
|
|
2234
|
+
const PS_IDENTITY_TIMEOUT_MS = 5_000;
|
|
2235
|
+
/**
|
|
2236
|
+
* Pure: does a `ps -o etime=` value place the process's birth within 30s of when
|
|
2237
|
+
* we recorded spawning it? An empty/unknown reading conservatively reads as ours
|
|
2238
|
+
* — never reap a run on a doubtful identity probe.
|
|
2239
|
+
*/
|
|
2240
|
+
function etimeIndicatesOurs(etime, spawnedAt) {
|
|
2241
|
+
if (!etime)
|
|
2242
|
+
return true;
|
|
2243
|
+
const parts = etime.replace(/-/g, ':').split(':').reverse();
|
|
2244
|
+
let uptimeSec = 0;
|
|
2245
|
+
if (parts[0])
|
|
2246
|
+
uptimeSec += parseInt(parts[0], 10);
|
|
2247
|
+
if (parts[1])
|
|
2248
|
+
uptimeSec += parseInt(parts[1], 10) * 60;
|
|
2249
|
+
if (parts[2])
|
|
2250
|
+
uptimeSec += parseInt(parts[2], 10) * 3600;
|
|
2251
|
+
if (parts[3])
|
|
2252
|
+
uptimeSec += parseInt(parts[3], 10) * 86400;
|
|
2253
|
+
const processStartMs = Date.now() - uptimeSec * 1000;
|
|
2254
|
+
return Math.abs(processStartMs - spawnedAt) < 30_000;
|
|
2255
|
+
}
|
|
2256
|
+
/**
|
|
2257
|
+
* Synchronous identity check for callers OTHER than the periodic heartbeat tick:
|
|
2258
|
+
* the `agents routines list/status` builders and stop-time enumeration (both in
|
|
2259
|
+
* short-lived CLI processes), plus the scheduler's fire-time slot-claim
|
|
2260
|
+
* (`activeRoutineRun`) and status projection (`isRunGenuinelyInFlight`). Those
|
|
2261
|
+
* last two DO run in-process on the daemon's event loop, but are event-driven
|
|
2262
|
+
* (they fire when a job is dispatched, not on a fixed cadence) and the `ps` here
|
|
2263
|
+
* is now bounded to {@link PS_IDENTITY_TIMEOUT_MS} — they are not the
|
|
2264
|
+
* unconditional every-tick scan PHNX-3695 targets. The heartbeat tick's own scan
|
|
2265
|
+
* uses the non-blocking {@link isPidOursAsync} instead.
|
|
2266
|
+
*/
|
|
2227
2267
|
function isPidOurs(pid, spawnedAt) {
|
|
2228
2268
|
try {
|
|
2229
2269
|
process.kill(pid, 0);
|
|
@@ -2236,42 +2276,75 @@ function isPidOurs(pid, spawnedAt) {
|
|
|
2236
2276
|
if (process.platform === 'win32')
|
|
2237
2277
|
return true;
|
|
2238
2278
|
try {
|
|
2239
|
-
const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
2240
|
-
|
|
2241
|
-
return true;
|
|
2242
|
-
const parts = etime.replace(/-/g, ':').split(':').reverse();
|
|
2243
|
-
let uptimeSec = 0;
|
|
2244
|
-
if (parts[0])
|
|
2245
|
-
uptimeSec += parseInt(parts[0], 10);
|
|
2246
|
-
if (parts[1])
|
|
2247
|
-
uptimeSec += parseInt(parts[1], 10) * 60;
|
|
2248
|
-
if (parts[2])
|
|
2249
|
-
uptimeSec += parseInt(parts[2], 10) * 3600;
|
|
2250
|
-
if (parts[3])
|
|
2251
|
-
uptimeSec += parseInt(parts[3], 10) * 86400;
|
|
2252
|
-
const processStartMs = Date.now() - uptimeSec * 1000;
|
|
2253
|
-
return Math.abs(processStartMs - spawnedAt) < 30_000;
|
|
2279
|
+
const etime = execFileSync('ps', ['-p', String(pid), '-o', 'etime='], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: PS_IDENTITY_TIMEOUT_MS }).trim();
|
|
2280
|
+
return etimeIndicatesOurs(etime, spawnedAt);
|
|
2254
2281
|
}
|
|
2255
2282
|
catch {
|
|
2256
2283
|
return true;
|
|
2257
2284
|
}
|
|
2258
2285
|
}
|
|
2286
|
+
/**
|
|
2287
|
+
* Async, deadline-bounded twin of {@link isPidOurs} for the daemon's heartbeat
|
|
2288
|
+
* tick (`reapExitedRunningJobs`) — the `ps` probe runs on the shared event loop,
|
|
2289
|
+
* so it MUST NOT block it (PHNX-3695). Same semantics: dead pid → not ours;
|
|
2290
|
+
* no recorded `spawnedAt` or Windows → assume ours; a failed/empty probe reads
|
|
2291
|
+
* as ours (conservative — never reap on a doubtful probe).
|
|
2292
|
+
*/
|
|
2293
|
+
async function isPidOursAsync(pid, spawnedAt) {
|
|
2294
|
+
if (!isAlive(pid))
|
|
2295
|
+
return false;
|
|
2296
|
+
if (spawnedAt === undefined)
|
|
2297
|
+
return true;
|
|
2298
|
+
if (process.platform === 'win32')
|
|
2299
|
+
return true;
|
|
2300
|
+
const res = await execFileBounded('ps', ['-p', String(pid), '-o', 'etime='], { timeoutMs: PS_IDENTITY_TIMEOUT_MS });
|
|
2301
|
+
if (res.code !== 0)
|
|
2302
|
+
return true; // probe failed → keep the run, same as the sync catch
|
|
2303
|
+
return etimeIndicatesOurs(res.stdout.trim(), spawnedAt);
|
|
2304
|
+
}
|
|
2259
2305
|
/**
|
|
2260
2306
|
* Finalize one `host:`-placed run by healing its host-task sidecar against the
|
|
2261
2307
|
* remote `.exit` (lib/hosts/reconcile.ts). Mutates + persists the meta only
|
|
2262
2308
|
* when the sidecar reached a terminal state.
|
|
2263
2309
|
*/
|
|
2310
|
+
/** Apply a healed host-task's terminal state to the local run record. Shared by the sync and async host reconcilers. */
|
|
2311
|
+
function applyHealedHostRun(meta, healed,
|
|
2312
|
+
// The routine-end emit acquires the event-log file lock. On the daemon
|
|
2313
|
+
// heartbeat tick (finalizeHostRunAsync) that MUST be the async, non-blocking
|
|
2314
|
+
// variant, or the synchronous `withFileLock` (lockSync + Atomics.wait, up to
|
|
2315
|
+
// 30s under contention) freezes the whole event loop — the exact wedge PHNX-3695
|
|
2316
|
+
// targets, on a path guard-no-sync-io does not scan (PHNX-3727). The sync CLI
|
|
2317
|
+
// path (finalizeHostRun) keeps the default synchronous emit.
|
|
2318
|
+
emit = emitRoutineEnd) {
|
|
2319
|
+
if (healed.status !== 'completed' && healed.status !== 'failed')
|
|
2320
|
+
return;
|
|
2321
|
+
finalizeRunMeta(meta, healed.status, healed.exitCode ?? (healed.status === 'completed' ? 0 : 1), { completedAt: healed.finishedAt ?? undefined });
|
|
2322
|
+
writeRunMeta(meta);
|
|
2323
|
+
emit(meta);
|
|
2324
|
+
}
|
|
2264
2325
|
function finalizeHostRun(meta) {
|
|
2265
2326
|
try {
|
|
2266
2327
|
const task = loadHostTask(meta.hostTaskId);
|
|
2267
2328
|
if (!task)
|
|
2268
2329
|
return;
|
|
2269
|
-
|
|
2270
|
-
|
|
2330
|
+
applyHealedHostRun(meta, reconcileHostTask(task));
|
|
2331
|
+
}
|
|
2332
|
+
catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
|
|
2333
|
+
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Async twin of {@link finalizeHostRun} for the daemon heartbeat tick
|
|
2336
|
+
* (PHNX-3695). A `host:`-placed run's remote `.exit` is read over ssh; on the
|
|
2337
|
+
* tick that read MUST be async (`reconcileHostTaskAsync` → `sshExecAsync`) or a
|
|
2338
|
+
* single 6s ssh timeout freezes the whole event loop — the exact bug class this
|
|
2339
|
+
* effort targets, worse than the local `ps` probe.
|
|
2340
|
+
*/
|
|
2341
|
+
async function finalizeHostRunAsync(meta) {
|
|
2342
|
+
try {
|
|
2343
|
+
const task = loadHostTask(meta.hostTaskId);
|
|
2344
|
+
if (!task)
|
|
2271
2345
|
return;
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
emitRoutineEnd(meta);
|
|
2346
|
+
// Tick path: emit through the async, non-blocking file lock (PHNX-3727).
|
|
2347
|
+
applyHealedHostRun(meta, await reconcileHostTaskAsync(task), (m) => { void emitRoutineEndAsync(m); });
|
|
2275
2348
|
}
|
|
2276
2349
|
catch { /* unreachable host or unreadable sidecar — retry next sweep */ }
|
|
2277
2350
|
}
|
|
@@ -2345,7 +2418,91 @@ export function isRunGenuinelyInFlight(meta) {
|
|
|
2345
2418
|
return false;
|
|
2346
2419
|
return isPidOurs(meta.pid, meta.spawnedAt);
|
|
2347
2420
|
}
|
|
2348
|
-
/**
|
|
2421
|
+
/**
|
|
2422
|
+
* Reconcile ONE `running` record once its process identity is known (`ours` =
|
|
2423
|
+
* the recorded pid is still genuinely the child we spawned). Shared verbatim by
|
|
2424
|
+
* the synchronous {@link monitorRunningJobs} (off-loop callers) and the async
|
|
2425
|
+
* {@link reapExitedRunningJobs} (daemon heartbeat tick) so there is one copy of
|
|
2426
|
+
* the finalize/timeout/report logic — only the surrounding directory/`ps` IO
|
|
2427
|
+
* differs by sync-vs-async flavor. Caller has already confirmed `meta.status`
|
|
2428
|
+
* is `running`. The finalize/report/archive helpers stay synchronous: they run
|
|
2429
|
+
* ONLY when a run actually transitions (times out or its process exited), a
|
|
2430
|
+
* conditional/rare event, not the every-tick scan the loop-starvation fix
|
|
2431
|
+
* (PHNX-3695) targets. The one exception is the `routine.end` EVENT emit, whose
|
|
2432
|
+
* file lock CAN block for up to 30s under contention even on that rare path — so
|
|
2433
|
+
* the emitter is injected: the off-loop sync caller passes `emitRoutineEnd`
|
|
2434
|
+
* (synchronous, so a short-lived CLI process flushes it before exit), the daemon
|
|
2435
|
+
* tick passes a fire-and-forget `emitRoutineEndAsync` (the long-lived daemon
|
|
2436
|
+
* flushes it, and the shared event loop is never blocked on the lock).
|
|
2437
|
+
*/
|
|
2438
|
+
function reconcileRunningRecord(meta, jobRunsPath, runDirName, ours, emitEnd) {
|
|
2439
|
+
// Host-placed runs (meta.hostTaskId) are handled by the caller, sync or async
|
|
2440
|
+
// — their remote `.exit` read is ssh I/O that must stay off the daemon tick's
|
|
2441
|
+
// event loop (finalizeHostRunAsync). Records reaching here are local-pid runs.
|
|
2442
|
+
const runDirPath = path.join(jobRunsPath, runDirName);
|
|
2443
|
+
const stdoutPath = path.join(runDirPath, 'stdout.log');
|
|
2444
|
+
// Command-mode records carry no agent; there is no stream-json report to
|
|
2445
|
+
// parse or extract. Reap them on pid liveness alone.
|
|
2446
|
+
const isCommandRun = Boolean(meta.command) || !meta.agent;
|
|
2447
|
+
// Age-out runs FIRST, before the null-pid guard below, so a wedged record
|
|
2448
|
+
// still marked `running` past its deadline is always finalized — a
|
|
2449
|
+
// provisional launcher claim whose daemon crashed before spawning a child
|
|
2450
|
+
// (pid null), or an old record whose recorded pid was reused, would
|
|
2451
|
+
// otherwise linger as `running` forever (RUSH-2640). Only kill a process
|
|
2452
|
+
// group that is genuinely still ours; terminating a reused pid would kill
|
|
2453
|
+
// an unrelated process, and a null pid has nothing to kill.
|
|
2454
|
+
const wallClockMs = Date.now() - Date.parse(meta.startedAt);
|
|
2455
|
+
const timeoutMs = meta.timeoutMs ?? MAX_WALL_CLOCK_MS;
|
|
2456
|
+
if (Number.isFinite(wallClockMs) && wallClockMs > timeoutMs) {
|
|
2457
|
+
if (meta.pid && ours)
|
|
2458
|
+
terminateRoutineTree(meta.pid);
|
|
2459
|
+
finalizeRunMeta(meta, 'timeout', null, { errorMessage: 'exceeded configured timeout' });
|
|
2460
|
+
writeRunMeta(meta);
|
|
2461
|
+
emitEnd(meta);
|
|
2462
|
+
if (!isCommandRun) {
|
|
2463
|
+
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2464
|
+
archiveRoutineTranscripts(meta, runDirPath);
|
|
2465
|
+
}
|
|
2466
|
+
return;
|
|
2467
|
+
}
|
|
2468
|
+
if (!meta.pid)
|
|
2469
|
+
return;
|
|
2470
|
+
if (!ours) {
|
|
2471
|
+
if (isCommandRun) {
|
|
2472
|
+
// Command routines normally record their own terminal status via
|
|
2473
|
+
// child.on('exit') (so this record would already be non-'running' and
|
|
2474
|
+
// skipped above). Reaching here means the daemon restarted mid-run and
|
|
2475
|
+
// missed the exit event — recover the true code from the exit-code file
|
|
2476
|
+
// the child wrote; its absence means the child was killed/crashed.
|
|
2477
|
+
const ec = readCommandExitCode(runDirPath);
|
|
2478
|
+
finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
|
|
2479
|
+
}
|
|
2480
|
+
else {
|
|
2481
|
+
const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
|
|
2482
|
+
if (inferred) {
|
|
2483
|
+
finalizeRunMeta(meta, inferred.status, inferred.exitCode);
|
|
2484
|
+
}
|
|
2485
|
+
else {
|
|
2486
|
+
finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
writeRunMeta(meta);
|
|
2490
|
+
emitEnd(meta);
|
|
2491
|
+
if (!isCommandRun) {
|
|
2492
|
+
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2493
|
+
archiveRoutineTranscripts(meta, runDirPath);
|
|
2494
|
+
}
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
/**
|
|
2498
|
+
* Scan all runs marked "running" and finalize any whose process has exited.
|
|
2499
|
+
*
|
|
2500
|
+
* SYNCHRONOUS — for OFF-loop callers only (the `agents routines list/status`
|
|
2501
|
+
* builders and CLI best-effort orphan reaps), which run in short-lived CLI
|
|
2502
|
+
* processes where a bounded synchronous `ps`/scan cannot starve a served event
|
|
2503
|
+
* loop. The daemon's own heartbeat tick MUST use {@link reapExitedRunningJobs}
|
|
2504
|
+
* instead — a sync scan there would freeze the shared event loop (PHNX-3695).
|
|
2505
|
+
*/
|
|
2349
2506
|
export function monitorRunningJobs() {
|
|
2350
2507
|
const runsDir = getRunsDir();
|
|
2351
2508
|
if (!fs.existsSync(runsDir))
|
|
@@ -2374,67 +2531,67 @@ export function monitorRunningJobs() {
|
|
|
2374
2531
|
const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
|
|
2375
2532
|
if (meta.status !== 'running')
|
|
2376
2533
|
continue;
|
|
2377
|
-
// `host:`-placed run — no local pid
|
|
2378
|
-
// remote `.exit` (completion is confirmed, never guessed: an
|
|
2379
|
-
// unreachable host leaves the run `running` for the next sweep).
|
|
2534
|
+
// `host:`-placed run — no local pid; reconcile against the remote `.exit`.
|
|
2380
2535
|
if (meta.hostTaskId) {
|
|
2381
2536
|
finalizeHostRun(meta);
|
|
2382
2537
|
continue;
|
|
2383
2538
|
}
|
|
2384
|
-
const
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2539
|
+
const ours = meta.pid ? isPidOurs(meta.pid, meta.spawnedAt) : false;
|
|
2540
|
+
// Sync emit: an off-loop CLI process must flush the event before it exits.
|
|
2541
|
+
reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, emitRoutineEnd);
|
|
2542
|
+
}
|
|
2543
|
+
catch { /* corrupt or unreadable meta.json */ }
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
/**
|
|
2548
|
+
* Async, non-blocking twin of {@link monitorRunningJobs} for the daemon's
|
|
2549
|
+
* heartbeat tick (PHNX-3695). Identical reconciliation via
|
|
2550
|
+
* {@link reconcileRunningRecord}, but the directory walk uses `fs/promises` and
|
|
2551
|
+
* the pid-identity probe uses the deadline-bounded {@link isPidOursAsync}, so
|
|
2552
|
+
* the every-tick scan never freezes the shared event loop — the freeze that
|
|
2553
|
+
* left the browser IPC `version` probe unanswered.
|
|
2554
|
+
*/
|
|
2555
|
+
export async function reapExitedRunningJobs() {
|
|
2556
|
+
const runsDir = getRunsDir();
|
|
2557
|
+
let jobDirs;
|
|
2558
|
+
try {
|
|
2559
|
+
jobDirs = await fsp.readdir(runsDir, { withFileTypes: true });
|
|
2560
|
+
}
|
|
2561
|
+
catch {
|
|
2562
|
+
return; // runs dir absent — nothing to reap
|
|
2563
|
+
}
|
|
2564
|
+
for (const jobDir of jobDirs) {
|
|
2565
|
+
if (!jobDir.isDirectory())
|
|
2566
|
+
continue;
|
|
2567
|
+
const jobRunsPath = path.join(runsDir, jobDir.name);
|
|
2568
|
+
let runDirs;
|
|
2569
|
+
try {
|
|
2570
|
+
runDirs = await fsp.readdir(jobRunsPath, { withFileTypes: true });
|
|
2571
|
+
}
|
|
2572
|
+
catch {
|
|
2573
|
+
// A retention/cleanup pass may remove a job directory after the root
|
|
2574
|
+
// snapshot. The next sweep observes the remaining state.
|
|
2575
|
+
continue;
|
|
2576
|
+
}
|
|
2577
|
+
for (const runDirEntry of runDirs) {
|
|
2578
|
+
if (!runDirEntry.isDirectory())
|
|
2579
|
+
continue;
|
|
2580
|
+
try {
|
|
2581
|
+
const raw = await fsp.readFile(path.join(jobRunsPath, runDirEntry.name, 'meta.json'), 'utf-8');
|
|
2582
|
+
const meta = JSON.parse(raw);
|
|
2583
|
+
if (meta.status !== 'running')
|
|
2408
2584
|
continue;
|
|
2409
|
-
|
|
2410
|
-
|
|
2585
|
+
// `host:`-placed run — read its remote `.exit` over ssh ASYNC so a slow
|
|
2586
|
+
// host never freezes the tick's event loop (PHNX-3695).
|
|
2587
|
+
if (meta.hostTaskId) {
|
|
2588
|
+
await finalizeHostRunAsync(meta);
|
|
2411
2589
|
continue;
|
|
2412
|
-
if (!isPidOurs(meta.pid, meta.spawnedAt)) {
|
|
2413
|
-
if (isCommandRun) {
|
|
2414
|
-
// Command routines normally record their own terminal status via
|
|
2415
|
-
// child.on('exit') (so this record would already be non-'running' and
|
|
2416
|
-
// skipped above). Reaching here means the daemon restarted mid-run and
|
|
2417
|
-
// missed the exit event — recover the true code from the exit-code file
|
|
2418
|
-
// the child wrote; its absence means the child was killed/crashed.
|
|
2419
|
-
const ec = readCommandExitCode(runDirPath);
|
|
2420
|
-
finalizeRunMeta(meta, ec === 0 ? 'completed' : 'failed', ec);
|
|
2421
|
-
}
|
|
2422
|
-
else {
|
|
2423
|
-
const inferred = inferFinalStatusFromLog(stdoutPath, meta.agent);
|
|
2424
|
-
if (inferred) {
|
|
2425
|
-
finalizeRunMeta(meta, inferred.status, inferred.exitCode);
|
|
2426
|
-
}
|
|
2427
|
-
else {
|
|
2428
|
-
finalizeRunMeta(meta, 'failed', null, { errorMessage: 'process exited before final status could be inferred' });
|
|
2429
|
-
}
|
|
2430
|
-
}
|
|
2431
|
-
writeRunMeta(meta);
|
|
2432
|
-
emitRoutineEnd(meta);
|
|
2433
|
-
if (!isCommandRun) {
|
|
2434
|
-
extractAndSaveReport(stdoutPath, meta.agent, runDirPath);
|
|
2435
|
-
archiveRoutineTranscripts(meta, runDirPath);
|
|
2436
|
-
}
|
|
2437
2590
|
}
|
|
2591
|
+
const ours = meta.pid ? await isPidOursAsync(meta.pid, meta.spawnedAt) : false;
|
|
2592
|
+
// Fire-and-forget async emit: the event-log lock must never block the
|
|
2593
|
+
// daemon tick's shared event loop; the long-lived daemon still flushes it.
|
|
2594
|
+
reconcileRunningRecord(meta, jobRunsPath, runDirEntry.name, ours, (m) => { void emitRoutineEndAsync(m); });
|
|
2438
2595
|
}
|
|
2439
2596
|
catch { /* corrupt or unreadable meta.json */ }
|
|
2440
2597
|
}
|