@ours.network/fleet 1.1.5 → 1.2.0-nightly.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -21
- package/dist/agent-ours/bridge.d.ts +3 -0
- package/dist/agent-ours/bridge.js +178 -0
- package/dist/agent-ours/controller.d.ts +14 -0
- package/dist/agent-ours/controller.js +60 -0
- package/dist/agent-ours/file-rpc.d.ts +23 -0
- package/dist/agent-ours/file-rpc.js +123 -0
- package/dist/agent-ours/harness.d.ts +14 -0
- package/dist/agent-ours/harness.js +64 -0
- package/dist/agent-ours/mcp-endpoint.d.ts +15 -0
- package/dist/agent-ours/mcp-endpoint.js +111 -0
- package/dist/agent-ours/runtime.d.ts +53 -0
- package/dist/agent-ours/runtime.js +247 -0
- package/dist/agent-ours/service.d.ts +23 -0
- package/dist/agent-ours/service.js +364 -0
- package/dist/agent-ours/state.d.ts +45 -0
- package/dist/agent-ours/state.js +95 -0
- package/dist/agent-ours/wire.d.ts +14 -0
- package/dist/agent-ours/wire.js +55 -0
- package/dist/agent-recovery-gate.js +1 -1
- package/dist/application/legacy-supervisor-identity.d.ts +7 -0
- package/dist/application/legacy-supervisor-identity.js +43 -0
- package/dist/application/supervisor-ours-tools.d.ts +159 -0
- package/dist/application/supervisor-ours-tools.js +121 -0
- package/dist/application/task-room-service.js +55 -19
- package/dist/briefing.js +10 -59
- package/dist/build-info.json +6 -5
- package/dist/capabilities.d.ts +2 -1
- package/dist/capabilities.js +2 -0
- package/dist/cli.js +75 -8
- package/dist/client-profile.d.ts +18 -0
- package/dist/client-profile.js +130 -0
- package/dist/config.d.ts +1 -0
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +13 -21
- package/dist/daemon-recovery.d.ts +2 -0
- package/dist/daemon-recovery.js +53 -2
- package/dist/docs.d.ts +3 -3
- package/dist/docs.js +32 -23
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.js +70 -14
- package/dist/fleet-command-audit.js +3 -2
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +4 -2
- package/dist/harness/codex-session.js +9 -2
- package/dist/harness/codex.d.ts +3 -2
- package/dist/harness/codex.js +8 -3
- package/dist/harness/hermes-session.js +7 -1
- package/dist/init-wizard.d.ts +11 -6
- package/dist/init-wizard.js +33 -0
- package/dist/monitor.d.ts +34 -3
- package/dist/monitor.js +174 -74
- package/dist/owner-channel/channel.d.ts +3 -0
- package/dist/owner-channel/channel.js +28 -6
- package/dist/owner-channel/ours-client.d.ts +12 -5
- package/dist/owner-channel/ours-client.js +48 -12
- package/dist/rooms-tasks/archived-absence.d.ts +8 -0
- package/dist/rooms-tasks/archived-absence.js +48 -0
- package/dist/rooms-tasks/cli.js +1 -1
- package/dist/rooms-tasks/close.d.ts +3 -0
- package/dist/rooms-tasks/close.js +95 -17
- package/dist/rooms-tasks/cowork-adapter.d.ts +4 -0
- package/dist/rooms-tasks/cowork-adapter.js +10 -2
- package/dist/rooms-tasks/cowork-http.d.ts +3 -0
- package/dist/rooms-tasks/cowork-http.js +70 -0
- package/dist/rooms-tasks/deletion.js +40 -3
- package/dist/rooms-tasks/member-startup.d.ts +1 -0
- package/dist/rooms-tasks/member-startup.js +2 -0
- package/dist/rooms-tasks/provision.d.ts +11 -0
- package/dist/rooms-tasks/provision.js +138 -26
- package/dist/rooms-tasks/room-state.js +13 -0
- package/dist/rooms-tasks/task-state.d.ts +13 -3
- package/dist/rooms-tasks/task-state.js +42 -6
- package/dist/rooms-tasks/types.d.ts +12 -0
- package/dist/rooms-tasks/workspace-artifacts.d.ts +6 -0
- package/dist/rooms-tasks/workspace-artifacts.js +93 -0
- package/dist/rooms-tasks/workspace.d.ts +26 -0
- package/dist/rooms-tasks/workspace.js +195 -0
- package/dist/runner.d.ts +14 -0
- package/dist/runner.js +540 -494
- package/dist/session/codex-app-server-transport.js +1 -1
- package/dist/session/codex-app-server.js +31 -5
- package/dist/spawn.js +19 -16
- package/dist/startup-readiness.d.ts +13 -0
- package/dist/startup-readiness.js +38 -0
- package/dist/supervisor/launchd.js +8 -1
- package/dist/supervisor/systemd.js +5 -0
- package/dist/temp-supervisor-recovery.d.ts +3 -0
- package/dist/temp-supervisor-recovery.js +46 -0
- package/dist/watchdog/briefing.js +3 -16
- package/dist/watchdog/run.js +19 -5
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +18 -1
- package/docs/task-workspaces.md +98 -0
- package/package.json +8 -4
- package/presets/fleet/roles/Coordinator.yaml +22 -0
package/dist/runner.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { prepareManagedAgent, releaseManagedAgent } from './agent-ours/service.js';
|
|
2
|
+
import { prepareManagedHarness } from './agent-ours/harness.js';
|
|
1
3
|
import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'node:fs';
|
|
2
4
|
import { join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
3
6
|
import { randomUUID } from 'node:crypto';
|
|
4
7
|
import { parse } from 'yaml';
|
|
5
8
|
import { agentDir, stateRoot } from './paths.js';
|
|
@@ -7,7 +10,6 @@ import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolv
|
|
|
7
10
|
import { getAdapter } from './harness/registry.js';
|
|
8
11
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
12
|
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
-
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
11
13
|
import { realExec } from './exec.js';
|
|
12
14
|
import { resolveIsolation } from './isolation/policy.js';
|
|
13
15
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
@@ -36,6 +38,8 @@ export class SupervisorRecycleRequiredError extends Error {
|
|
|
36
38
|
}
|
|
37
39
|
}
|
|
38
40
|
const defaultDeps = () => ({
|
|
41
|
+
prepareAgentOurs: prepareManagedAgent,
|
|
42
|
+
releaseAgentOurs: releaseManagedAgent,
|
|
39
43
|
exec: realExec,
|
|
40
44
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
41
45
|
isAlive: pid => { try {
|
|
@@ -423,7 +427,7 @@ export function loadTempRole(name) {
|
|
|
423
427
|
* with "no such role". An empty/missing marker means "use the default", same as
|
|
424
428
|
* no -c was ever given.
|
|
425
429
|
*/
|
|
426
|
-
function resolveConfigPath(dir, explicit) {
|
|
430
|
+
export function resolveConfigPath(dir, explicit) {
|
|
427
431
|
if (explicit)
|
|
428
432
|
return explicit;
|
|
429
433
|
const marker = join(dir, '.config-path');
|
|
@@ -497,11 +501,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
497
501
|
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
498
502
|
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
499
503
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
504
|
+
role = { ...role, monitor: { ...role.monitor, mode: 'fleet' } };
|
|
500
505
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
501
506
|
const sessionBackend = role.session ?? 'acp';
|
|
502
507
|
const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
|
|
503
508
|
let launch = adapter.agentSession.prepareLaunch(role, prep);
|
|
504
|
-
//
|
|
509
|
+
// Preserve the role's existing isolation policy.
|
|
505
510
|
let wrappedArgv = launch.argv;
|
|
506
511
|
if (role.isolation) {
|
|
507
512
|
// Start with the same durable context that config validation and doctor judged,
|
|
@@ -511,7 +516,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
511
516
|
launch = { ...launch, argv: runtime.argv };
|
|
512
517
|
const ctx = {
|
|
513
518
|
...isolationContextFor(role), stateDir: dir, runCwd,
|
|
514
|
-
runtimeReadPaths: runtime.readPaths,
|
|
519
|
+
runtimeReadPaths: [...runtime.readPaths, ...resolveLaunchRuntime([process.execPath, fileURLToPath(new URL('./agent-ours/bridge.js', import.meta.url))]).readPaths],
|
|
515
520
|
};
|
|
516
521
|
const policy = resolveIsolation(role.isolation, ctx);
|
|
517
522
|
const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
|
|
@@ -533,507 +538,527 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
533
538
|
if (rprefix.length)
|
|
534
539
|
wrappedArgv = [...rprefix, ...wrappedArgv];
|
|
535
540
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
if (
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
// (backlog before the tip is the SessionStart hook's job). Native-mode roles
|
|
553
|
-
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
554
|
-
// treated as native (monitor may be undefined on an old role.yaml).
|
|
555
|
-
let sessionHandle;
|
|
556
|
-
let modelRecovery;
|
|
557
|
-
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
558
|
-
resolvedMonitorDeps.onFailureEvidence = evidence => {
|
|
559
|
-
if (!role.model_chain)
|
|
560
|
-
return false;
|
|
561
|
-
const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
|
|
562
|
-
if (action.kind === 'advance') {
|
|
563
|
-
modelRecovery = 'advance';
|
|
564
|
-
deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
|
|
565
|
-
void sessionHandle?.close();
|
|
566
|
-
return true;
|
|
567
|
-
}
|
|
568
|
-
else if (action.kind === 'hold') {
|
|
569
|
-
modelRecovery = 'hold';
|
|
570
|
-
deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
|
|
571
|
-
void sessionHandle?.close();
|
|
572
|
-
return true;
|
|
541
|
+
const managedService = await deps.prepareAgentOurs(role, dir, opts.identityLifetime ? opts.identityLifetime === 'temporary' : temp);
|
|
542
|
+
try {
|
|
543
|
+
const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, harnessChildEnv(role, launch.env, dir));
|
|
544
|
+
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
545
|
+
// agent launch across the whole host, so a burst of boots (systemd starts every
|
|
546
|
+
// user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
|
|
547
|
+
// hit the harness/API rate limit at once. Time-based via a shared launch gate, so
|
|
548
|
+
// a lone start or a solo crash-restart waits zero. Applied right before the harness
|
|
549
|
+
// agent-session start; the cheap monitor prime still runs immediately after.
|
|
550
|
+
if (staggerMs > 0) {
|
|
551
|
+
const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
|
|
552
|
+
const wait = slot - deps.now();
|
|
553
|
+
if (wait > 0) {
|
|
554
|
+
deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
|
|
555
|
+
await deps.sleep(wait);
|
|
556
|
+
}
|
|
573
557
|
}
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
let unsubscribeRecovery;
|
|
592
|
-
let monitorLoop;
|
|
593
|
-
let sessionStartupComplete = false;
|
|
594
|
-
let sessionClosed = false;
|
|
595
|
-
let ownerChannel;
|
|
596
|
-
let ownerBinder;
|
|
597
|
-
let loopManager;
|
|
598
|
-
let arbiter;
|
|
599
|
-
let recoveryController;
|
|
600
|
-
let reloadLoopConfig;
|
|
601
|
-
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
602
|
-
loop.name, loop.definitionHash, loop.promptHash,
|
|
603
|
-
]));
|
|
604
|
-
{
|
|
605
|
-
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
606
|
-
// Say once, at startup, that this role will decide permission requests by
|
|
607
|
-
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
608
|
-
// that quietly did less than it was asked to.
|
|
609
|
-
if (perms.unattended === 'deny')
|
|
610
|
-
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
611
|
-
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
612
|
-
agentSession = await deps.startAgentSession(adapter.agentSession, {
|
|
613
|
-
role, prep,
|
|
614
|
-
launch: { ...launch, argv: wrappedArgv, env: harnessChildEnv(role, launch.env, dir) },
|
|
615
|
-
cwd: runCwd, stateDir: dir, mode, permissions: perms,
|
|
616
|
-
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
617
|
-
});
|
|
618
|
-
pid = agentSession.pid;
|
|
619
|
-
arbiter = new RoleTurnArbiter(agentSession);
|
|
620
|
-
sessionHandle = arbiter;
|
|
621
|
-
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
622
|
-
if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
|
|
623
|
-
return;
|
|
624
|
-
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
625
|
-
if (evidence)
|
|
626
|
-
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
627
|
-
});
|
|
628
|
-
if (role.owner_channel) {
|
|
629
|
-
try {
|
|
630
|
-
ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
|
|
558
|
+
// Prime the supervisor mail monitor's notification cursor at the
|
|
559
|
+
// stream tip BEFORE the session launches so no arrival is missed during boot
|
|
560
|
+
// (backlog before the tip is the SessionStart hook's job). Native-mode roles
|
|
561
|
+
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
562
|
+
// treated as native (monitor may be undefined on an old role.yaml).
|
|
563
|
+
let sessionHandle;
|
|
564
|
+
let modelRecovery;
|
|
565
|
+
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
566
|
+
resolvedMonitorDeps.onFailureEvidence = evidence => {
|
|
567
|
+
if (!role.model_chain)
|
|
568
|
+
return false;
|
|
569
|
+
const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
|
|
570
|
+
if (action.kind === 'advance') {
|
|
571
|
+
modelRecovery = 'advance';
|
|
572
|
+
deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
|
|
573
|
+
void sessionHandle?.close();
|
|
574
|
+
return true;
|
|
631
575
|
}
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
}
|
|
638
|
-
catch (notifyError) {
|
|
639
|
-
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
640
|
-
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
await agentSession.close();
|
|
644
|
-
unsubscribeRecovery?.();
|
|
645
|
-
throw new Error(`[${name}] owner channel failed to start: `
|
|
646
|
-
+ `${error?.message ?? String(error)}`);
|
|
576
|
+
else if (action.kind === 'hold') {
|
|
577
|
+
modelRecovery = 'hold';
|
|
578
|
+
deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
|
|
579
|
+
void sessionHandle?.close();
|
|
580
|
+
return true;
|
|
647
581
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
582
|
+
return false;
|
|
583
|
+
};
|
|
584
|
+
const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
|
|
585
|
+
const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
|
|
586
|
+
const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
|
|
587
|
+
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
588
|
+
deps: resolvedMonitorDeps,
|
|
589
|
+
}) : null;
|
|
590
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
591
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
592
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
593
|
+
if (monitor)
|
|
594
|
+
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
595
|
+
rmSync(exitFile, { force: true });
|
|
596
|
+
let pid;
|
|
597
|
+
let agentSession;
|
|
598
|
+
let control;
|
|
599
|
+
let unsubscribeRecovery;
|
|
600
|
+
let monitorLoop;
|
|
601
|
+
let sessionStartupComplete = false;
|
|
602
|
+
let sessionClosed = false;
|
|
603
|
+
let ownerChannel;
|
|
604
|
+
let ownerBinder;
|
|
605
|
+
let loopManager;
|
|
606
|
+
let arbiter;
|
|
607
|
+
let recoveryController;
|
|
608
|
+
let reloadLoopConfig;
|
|
609
|
+
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
610
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
611
|
+
]));
|
|
612
|
+
{
|
|
613
|
+
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
614
|
+
// Say once, at startup, that this role will decide permission requests by
|
|
615
|
+
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
616
|
+
// that quietly did less than it was asked to.
|
|
617
|
+
if (perms.unattended === 'deny')
|
|
618
|
+
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
619
|
+
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
620
|
+
agentSession = await managedService.runtime.startHarness(() => deps.startAgentSession(adapter.agentSession, {
|
|
621
|
+
managedOurs: managedHarness.ours,
|
|
622
|
+
role, prep,
|
|
623
|
+
launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
|
|
624
|
+
cwd: runCwd, stateDir: dir, mode, permissions: perms,
|
|
625
|
+
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
626
|
+
}));
|
|
627
|
+
pid = agentSession.pid;
|
|
628
|
+
arbiter = new RoleTurnArbiter(agentSession);
|
|
629
|
+
sessionHandle = arbiter;
|
|
630
|
+
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
631
|
+
if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
|
|
632
|
+
return;
|
|
633
|
+
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
634
|
+
if (evidence)
|
|
635
|
+
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
636
|
+
});
|
|
637
|
+
if (role.owner_channel) {
|
|
638
|
+
try {
|
|
639
|
+
ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
|
|
640
|
+
}
|
|
641
|
+
catch (error) {
|
|
642
|
+
if (error instanceof OwnerBinderHandoffTimeoutError) {
|
|
643
|
+
try {
|
|
644
|
+
const status = await deps.reportOwnerStartupFailure(dir);
|
|
645
|
+
deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
|
|
646
|
+
}
|
|
647
|
+
catch (notifyError) {
|
|
648
|
+
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
649
|
+
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
650
|
+
}
|
|
689
651
|
}
|
|
652
|
+
await agentSession.close();
|
|
653
|
+
unsubscribeRecovery?.();
|
|
654
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
655
|
+
+ `${error?.message ?? String(error)}`);
|
|
690
656
|
}
|
|
691
|
-
return event;
|
|
692
|
-
});
|
|
693
|
-
resolvedMonitorDeps.delivery = {
|
|
694
|
-
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
695
|
-
// refusal or a cancellation reached the agent and was not acted on, so
|
|
696
|
-
// the monitor must keep its cursor and try again.
|
|
697
|
-
submit: async (text, options) => {
|
|
698
|
-
// Cancelling the runner-owned startup prompt makes startup look failed
|
|
699
|
-
// and closes the session before the wake turn can run. During startup,
|
|
700
|
-
// steer into the live turn instead; after it completes, honor the
|
|
701
|
-
// configured interrupt policy normally.
|
|
702
|
-
const policy = options?.interrupt;
|
|
703
|
-
const interrupt = policy === true && sessionStartupComplete;
|
|
704
|
-
const promptOptions = {
|
|
705
|
-
interrupt, steer: true,
|
|
706
|
-
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
707
|
-
origin: { kind: 'fleet-monitor' },
|
|
708
|
-
};
|
|
709
|
-
// Startup is already a protected boundary: as with immediate mode,
|
|
710
|
-
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
711
|
-
const result = policy === 'after_tool' && sessionStartupComplete
|
|
712
|
-
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
713
|
-
: await arbiter.submitPrompt(text, promptOptions);
|
|
714
|
-
const steered = result.accepted
|
|
715
|
-
&& (result.detail === 'injected' || result.detail === 'startedNewTurn');
|
|
716
|
-
const boundary = result.safeBoundary;
|
|
717
|
-
const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
|
|
718
|
-
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
719
|
-
? boundary.state === 'timeout'
|
|
720
|
-
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
721
|
-
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
722
|
-
: boundary
|
|
723
|
-
? boundary.state === 'timeout'
|
|
724
|
-
? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
|
|
725
|
-
: boundary.state === 'unsupported'
|
|
726
|
-
? 'after_tool unsupported; used non-cancelling queued delivery'
|
|
727
|
-
: `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
|
|
728
|
-
: result.detail;
|
|
729
|
-
return {
|
|
730
|
-
succeeded: result.succeeded || steered,
|
|
731
|
-
outcome: steered ? result.detail : result.outcome,
|
|
732
|
-
detail: result.succeeded || steered || !boundary
|
|
733
|
-
? boundaryDetail
|
|
734
|
-
: [result.detail, boundaryDetail].filter(Boolean).join('; '),
|
|
735
|
-
...(boundary ? { safeBoundary: boundary.state } : {}),
|
|
736
|
-
};
|
|
737
|
-
},
|
|
738
|
-
};
|
|
739
|
-
const firstPrompt = mode === 'fresh'
|
|
740
|
-
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
741
|
-
: adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
|
|
742
|
-
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
743
|
-
// startup prompt and then refuses it has not started; logging the role as
|
|
744
|
-
// up would hide a role that never read its briefing.
|
|
745
|
-
const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
|
|
746
|
-
// Monitoring starts immediately. The delivery adapter above downgrades
|
|
747
|
-
// interruption to steering until this startup turn reaches a terminal
|
|
748
|
-
// success, so there is neither a deaf gap nor a boot-cancellation loop.
|
|
749
|
-
monitorLoop = monitor?.run(pid);
|
|
750
|
-
const started = await starting;
|
|
751
|
-
// A temporary role's first turn can be the active turn when an ours wake
|
|
752
|
-
// needs immediate attention. A typed console/monitor cancellation ends
|
|
753
|
-
// only that turn: the already-live agent session and any queued wake remain
|
|
754
|
-
// valid. Keep every unproven cancellation, refusal, shutdown, and genuine
|
|
755
|
-
// failure terminal so a role that never accepted its briefing is not
|
|
756
|
-
// silently reported as healthy.
|
|
757
|
-
const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
|
|
758
|
-
|| (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
|
|
759
|
-
if (!started.succeeded && !interruptedForWake) {
|
|
760
|
-
monitor?.stop();
|
|
761
|
-
await control.close();
|
|
762
|
-
ownerBinder?.release();
|
|
763
|
-
await agentSession.close();
|
|
764
|
-
unsubscribeRecovery?.();
|
|
765
|
-
if (modelRecovery) {
|
|
766
|
-
if (monitorLoop)
|
|
767
|
-
await monitorLoop;
|
|
768
|
-
return {
|
|
769
|
-
elapsedSecs: 0,
|
|
770
|
-
exit: {
|
|
771
|
-
version: 1,
|
|
772
|
-
class: 'program-exit',
|
|
773
|
-
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
774
|
-
},
|
|
775
|
-
rotated: false,
|
|
776
|
-
mode,
|
|
777
|
-
modelRecovery,
|
|
778
|
-
};
|
|
779
657
|
}
|
|
780
|
-
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
781
|
-
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
782
|
-
}
|
|
783
|
-
if (started.cancellationSource === 'stall-watchdog')
|
|
784
|
-
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
785
|
-
else if (interruptedForWake)
|
|
786
|
-
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
787
|
-
+ 'keeping temporary supervisor alive');
|
|
788
|
-
sessionStartupComplete = true;
|
|
789
|
-
if (ownerChannel) {
|
|
790
658
|
try {
|
|
791
|
-
|
|
659
|
+
if (role.owner_channel)
|
|
660
|
+
ownerChannel = deps.createOwnerChannel({
|
|
661
|
+
role: name,
|
|
662
|
+
harness: role.harness,
|
|
663
|
+
config: role.owner_channel,
|
|
664
|
+
session: arbiter,
|
|
665
|
+
startupPending: () => !sessionStartupComplete,
|
|
666
|
+
stateDir: dir,
|
|
667
|
+
env: role.env,
|
|
668
|
+
log: deps.log,
|
|
669
|
+
...(ownerBinder ? { binderLease: ownerBinder } : {}),
|
|
670
|
+
...(configPath ? { configPath } : {}),
|
|
671
|
+
});
|
|
672
|
+
control = deps.createControlServer(dir, arbiter, deps.log);
|
|
673
|
+
control.setFleetAuditor(ownerChannel ? {
|
|
674
|
+
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
675
|
+
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
676
|
+
present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
|
|
677
|
+
} : localFleetAuditor(dir, name, deps.log));
|
|
678
|
+
await control.start();
|
|
792
679
|
}
|
|
793
680
|
catch (error) {
|
|
794
|
-
|
|
795
|
-
if (monitorLoop)
|
|
796
|
-
await monitorLoop;
|
|
797
|
-
await ownerChannel.close().catch(() => undefined);
|
|
798
|
-
await control.close();
|
|
681
|
+
await ownerChannel?.close().catch(() => undefined);
|
|
799
682
|
ownerBinder?.release();
|
|
800
683
|
await agentSession.close();
|
|
801
684
|
unsubscribeRecovery?.();
|
|
802
|
-
throw
|
|
803
|
-
+ `${error?.message ?? String(error)}`);
|
|
804
|
-
}
|
|
805
|
-
}
|
|
806
|
-
if (ownerChannel) {
|
|
807
|
-
control.setOwnerChannel(ownerChannel);
|
|
808
|
-
}
|
|
809
|
-
reloadLoopConfig = async () => {
|
|
810
|
-
const nextRole = findRole(loadConfig(configPath), name);
|
|
811
|
-
const definitions = nextRole.loops ?? [];
|
|
812
|
-
const generation = JSON.stringify(definitions.map(loop => [
|
|
813
|
-
loop.name, loop.definitionHash, loop.promptHash,
|
|
814
|
-
]));
|
|
815
|
-
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
816
|
-
return { changed: false, loops: definitions.length };
|
|
817
|
-
if (!loopManager && definitions.length) {
|
|
818
|
-
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
819
|
-
now: deps.now,
|
|
820
|
-
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
821
|
-
clearTimer: timer => clearTimeout(timer),
|
|
822
|
-
log: deps.log,
|
|
823
|
-
});
|
|
824
|
-
control.setLoopManager(loopManager);
|
|
825
|
-
loopManager.start();
|
|
685
|
+
throw error;
|
|
826
686
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
clearTimer: timer => clearTimeout(timer),
|
|
843
|
-
log: deps.log,
|
|
687
|
+
if (!role.roomMemberStartup)
|
|
688
|
+
control.setFleetSpawner(async (requested) => {
|
|
689
|
+
const event = await executeManagedSpawn(role, configPath, requested, deps.log);
|
|
690
|
+
// Room-member launches are collected with their Task/Room transaction so
|
|
691
|
+
// causal delivery is Task → Room → Agents → Room active → Task active.
|
|
692
|
+
if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
|
|
693
|
+
try {
|
|
694
|
+
await ownerChannel.notifyFleetSpawn(event);
|
|
695
|
+
}
|
|
696
|
+
catch (error) {
|
|
697
|
+
deps.log(`[${name}] Agent lifecycle notice delivery failed: `
|
|
698
|
+
+ `${error?.message ?? String(error)}`);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
return event;
|
|
844
702
|
});
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
703
|
+
resolvedMonitorDeps.delivery = {
|
|
704
|
+
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
705
|
+
// refusal or a cancellation reached the agent and was not acted on, so
|
|
706
|
+
// the monitor must keep its cursor and try again.
|
|
707
|
+
submit: async (text, options) => {
|
|
708
|
+
// Cancelling the runner-owned startup prompt makes startup look failed
|
|
709
|
+
// and closes the session before the wake turn can run. During startup,
|
|
710
|
+
// steer into the live turn instead; after it completes, honor the
|
|
711
|
+
// configured interrupt policy normally.
|
|
712
|
+
const policy = options?.interrupt;
|
|
713
|
+
const interrupt = policy === true && sessionStartupComplete;
|
|
714
|
+
const promptOptions = {
|
|
715
|
+
interrupt, steer: true,
|
|
716
|
+
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
717
|
+
origin: { kind: 'fleet-monitor' },
|
|
718
|
+
};
|
|
719
|
+
// Startup is already a protected boundary: as with immediate mode,
|
|
720
|
+
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
721
|
+
const result = policy === 'after_tool' && sessionStartupComplete
|
|
722
|
+
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
723
|
+
: await arbiter.submitPrompt(text, promptOptions);
|
|
724
|
+
const steered = result.accepted
|
|
725
|
+
&& (result.detail === 'injected' || result.detail === 'startedNewTurn');
|
|
726
|
+
const boundary = result.safeBoundary;
|
|
727
|
+
const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
|
|
728
|
+
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
729
|
+
? boundary.state === 'timeout'
|
|
730
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
731
|
+
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
732
|
+
: boundary
|
|
733
|
+
? boundary.state === 'timeout'
|
|
734
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
|
|
735
|
+
: boundary.state === 'unsupported'
|
|
736
|
+
? 'after_tool unsupported; used non-cancelling queued delivery'
|
|
737
|
+
: `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
|
|
738
|
+
: result.detail;
|
|
739
|
+
return {
|
|
740
|
+
succeeded: result.succeeded || steered,
|
|
741
|
+
outcome: steered ? result.detail : result.outcome,
|
|
742
|
+
detail: result.succeeded || steered || !boundary
|
|
743
|
+
? boundaryDetail
|
|
744
|
+
: [result.detail, boundaryDetail].filter(Boolean).join('; '),
|
|
745
|
+
...(boundary ? { safeBoundary: boundary.state } : {}),
|
|
746
|
+
};
|
|
747
|
+
},
|
|
748
|
+
};
|
|
749
|
+
const firstPrompt = mode === 'fresh'
|
|
750
|
+
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
751
|
+
: `Your supervisor has verified your assigned identity and room readiness. Read ${join(dir, 'WORKLOG.md')} and ${join(dir, 'briefing.md')}, then continue using the available ours tools.`;
|
|
752
|
+
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
753
|
+
// startup prompt and then refuses it has not started; logging the role as
|
|
754
|
+
// up would hide a role that never read its briefing.
|
|
755
|
+
const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
|
|
756
|
+
// Monitoring starts immediately. The delivery adapter above downgrades
|
|
757
|
+
// interruption to steering until this startup turn reaches a terminal
|
|
758
|
+
// success, so there is neither a deaf gap nor a boot-cancellation loop.
|
|
759
|
+
monitorLoop = monitor?.run(pid);
|
|
760
|
+
// Owner traffic and agent replies must be observed during a long first
|
|
761
|
+
// turn. Ordinary owner input steers until startup has proved successful.
|
|
762
|
+
if (ownerChannel) {
|
|
763
|
+
try {
|
|
764
|
+
await ownerChannel.start();
|
|
765
|
+
control.setOwnerChannel(ownerChannel);
|
|
766
|
+
}
|
|
767
|
+
catch (error) {
|
|
850
768
|
monitor?.stop();
|
|
851
769
|
if (monitorLoop)
|
|
852
770
|
await monitorLoop;
|
|
853
771
|
await control.close();
|
|
772
|
+
await ownerChannel.close().catch(() => undefined);
|
|
854
773
|
ownerBinder?.release();
|
|
855
774
|
await agentSession.close();
|
|
856
775
|
unsubscribeRecovery?.();
|
|
857
|
-
throw new Error(`[${name}]
|
|
776
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
858
777
|
+ `${error?.message ?? String(error)}`);
|
|
859
778
|
}
|
|
860
|
-
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
}
|
|
864
|
-
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
865
|
-
// The monitor loop lives exactly as long as the session: it starts once the
|
|
866
|
-
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
867
|
-
monitorLoop ??= monitor?.run(pid);
|
|
868
|
-
recoveryController = new RoleRecoveryController({
|
|
869
|
-
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
870
|
-
log: deps.log,
|
|
871
|
-
recoverAgent: async () => {
|
|
872
|
-
const evidence = await recoverAgentIdentity(arbiter, role.identity);
|
|
873
|
-
return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
|
|
874
|
-
},
|
|
875
|
-
recoverOwner: async (epoch) => {
|
|
876
|
-
if (!ownerChannel?.recover)
|
|
877
|
-
return { ok: true };
|
|
878
|
-
try {
|
|
879
|
-
await ownerChannel.recover(epoch);
|
|
880
|
-
return { ok: true };
|
|
881
779
|
}
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
780
|
+
const started = await starting;
|
|
781
|
+
// A temporary role's first turn can be the active turn when an ours wake
|
|
782
|
+
// needs immediate attention. A typed console/monitor cancellation ends
|
|
783
|
+
// only that turn: the already-live agent session and any queued wake remain
|
|
784
|
+
// valid. Keep every unproven cancellation, refusal, shutdown, and genuine
|
|
785
|
+
// failure terminal so a role that never accepted its briefing is not
|
|
786
|
+
// silently reported as healthy.
|
|
787
|
+
const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
|
|
788
|
+
|| (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
|
|
789
|
+
if (!started.succeeded && !interruptedForWake) {
|
|
790
|
+
monitor?.stop();
|
|
791
|
+
await control.close();
|
|
792
|
+
await ownerChannel?.close().catch(() => undefined);
|
|
793
|
+
ownerBinder?.release();
|
|
794
|
+
await agentSession.close();
|
|
795
|
+
unsubscribeRecovery?.();
|
|
796
|
+
if (modelRecovery) {
|
|
797
|
+
if (monitorLoop)
|
|
798
|
+
await monitorLoop;
|
|
799
|
+
return {
|
|
800
|
+
elapsedSecs: 0,
|
|
801
|
+
exit: {
|
|
802
|
+
version: 1,
|
|
803
|
+
class: 'program-exit',
|
|
804
|
+
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
805
|
+
},
|
|
806
|
+
rotated: false,
|
|
807
|
+
mode,
|
|
808
|
+
modelRecovery,
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
812
|
+
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
885
813
|
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
814
|
+
if (started.cancellationSource === 'stall-watchdog')
|
|
815
|
+
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
816
|
+
else if (interruptedForWake)
|
|
817
|
+
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
818
|
+
+ 'keeping temporary supervisor alive');
|
|
819
|
+
sessionStartupComplete = true;
|
|
820
|
+
reloadLoopConfig = async () => {
|
|
821
|
+
const nextRole = findRole(loadConfig(configPath), name);
|
|
822
|
+
const definitions = nextRole.loops ?? [];
|
|
823
|
+
const generation = JSON.stringify(definitions.map(loop => [
|
|
824
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
825
|
+
]));
|
|
826
|
+
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
827
|
+
return { changed: false, loops: definitions.length };
|
|
828
|
+
if (!loopManager && definitions.length) {
|
|
829
|
+
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
830
|
+
now: deps.now,
|
|
831
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
832
|
+
clearTimer: timer => clearTimeout(timer),
|
|
833
|
+
log: deps.log,
|
|
834
|
+
});
|
|
835
|
+
control.setLoopManager(loopManager);
|
|
836
|
+
loopManager.start();
|
|
837
|
+
}
|
|
838
|
+
else {
|
|
839
|
+
loopManager?.reconcile(definitions);
|
|
840
|
+
}
|
|
841
|
+
loopGeneration = generation;
|
|
842
|
+
deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
|
|
843
|
+
return { changed: true, loops: definitions.length };
|
|
844
|
+
};
|
|
845
|
+
// Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
|
|
846
|
+
if (!temp)
|
|
847
|
+
control.setConfigReloader(reloadLoopConfig);
|
|
848
|
+
if (role.loops?.length) {
|
|
849
|
+
try {
|
|
850
|
+
loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
|
|
851
|
+
now: deps.now,
|
|
852
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
853
|
+
clearTimer: timer => clearTimeout(timer),
|
|
854
|
+
log: deps.log,
|
|
855
|
+
});
|
|
856
|
+
control.setLoopManager(loopManager);
|
|
857
|
+
loopManager.start();
|
|
858
|
+
}
|
|
859
|
+
catch (error) {
|
|
860
|
+
if (temp) {
|
|
861
|
+
monitor?.stop();
|
|
862
|
+
if (monitorLoop)
|
|
863
|
+
await monitorLoop;
|
|
864
|
+
await control.close();
|
|
865
|
+
ownerBinder?.release();
|
|
866
|
+
await agentSession.close();
|
|
867
|
+
unsubscribeRecovery?.();
|
|
868
|
+
throw new Error(`[${name}] configured temporary loop manager failed to start: `
|
|
869
|
+
+ `${error?.message ?? String(error)}`);
|
|
870
|
+
}
|
|
871
|
+
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
872
|
+
}
|
|
909
873
|
}
|
|
910
874
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
875
|
+
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
876
|
+
// The monitor loop lives exactly as long as the session: it starts once the
|
|
877
|
+
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
878
|
+
monitorLoop ??= monitor?.run(pid);
|
|
879
|
+
recoveryController = new RoleRecoveryController({
|
|
880
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
881
|
+
log: deps.log,
|
|
882
|
+
recoverAgent: async () => {
|
|
883
|
+
try {
|
|
884
|
+
const release = await managedService.runtime.admit();
|
|
885
|
+
release();
|
|
886
|
+
return { ok: true };
|
|
887
|
+
}
|
|
888
|
+
catch {
|
|
889
|
+
return { ok: false, reason: 'SUPERVISOR_IDENTITY_NOT_READY' };
|
|
890
|
+
}
|
|
891
|
+
},
|
|
892
|
+
recoverOwner: async (epoch) => {
|
|
893
|
+
if (!ownerChannel?.recover)
|
|
894
|
+
return { ok: true };
|
|
895
|
+
try {
|
|
896
|
+
await ownerChannel.recover(epoch);
|
|
897
|
+
return { ok: true };
|
|
898
|
+
}
|
|
899
|
+
catch (error) {
|
|
900
|
+
return { ok: false, reason: error instanceof Error
|
|
901
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
902
|
+
}
|
|
903
|
+
},
|
|
904
|
+
});
|
|
905
|
+
const start = deps.now();
|
|
906
|
+
let nextLoopReloadAt = deps.now() + 30_000;
|
|
907
|
+
let nextIdentityPollAt = deps.now();
|
|
908
|
+
let nextDaemonProbeAt = deps.now();
|
|
909
|
+
let lastReloadError = '';
|
|
910
|
+
let identityObserved = false;
|
|
911
|
+
let identityAbsentSince;
|
|
912
|
+
let retirementReason;
|
|
913
|
+
let supervisorRecycleRequired = false;
|
|
914
|
+
while (sessionHandle.isAlive()) {
|
|
915
|
+
await deps.sleep(temp ? 500 : 2000);
|
|
916
|
+
const now = deps.now();
|
|
917
|
+
if (now >= nextDaemonProbeAt) {
|
|
918
|
+
nextDaemonProbeAt = now + 2_000;
|
|
919
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
920
|
+
if (observation.kind === 'lost')
|
|
921
|
+
recoveryController.noteLoss(observation.reason);
|
|
922
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
923
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
924
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
925
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
926
|
+
}
|
|
924
927
|
}
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
// starts may spend minutes loading the harness and briefing before the
|
|
930
|
-
// agent creates/binds its identity, and absence before then is not a
|
|
931
|
-
// close event. After readiness, debounce a real disappearance.
|
|
932
|
-
if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
933
|
-
retirementReason = 'identity-closed';
|
|
934
|
-
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
935
|
-
await sessionHandle.close();
|
|
936
|
-
sessionClosed = true;
|
|
937
|
-
break;
|
|
928
|
+
if (deps.shouldStop?.()) {
|
|
929
|
+
if (temp) {
|
|
930
|
+
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
931
|
+
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
938
932
|
}
|
|
933
|
+
await sessionHandle.close();
|
|
934
|
+
sessionClosed = true;
|
|
935
|
+
break;
|
|
939
936
|
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
937
|
+
if (temp && now >= nextIdentityPollAt) {
|
|
938
|
+
nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
|
|
939
|
+
const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
|
|
940
|
+
if (presence.state === 'present') {
|
|
941
|
+
identityObserved = true;
|
|
942
|
+
identityAbsentSince = undefined;
|
|
943
|
+
}
|
|
944
|
+
else if (presence.state === 'absent' && identityObserved) {
|
|
945
|
+
identityAbsentSince ??= now;
|
|
946
|
+
// Require a continuous, time-bounded run of authoritative absence.
|
|
947
|
+
// The first positive observation is the readiness gate: cold harness
|
|
948
|
+
// starts may spend minutes loading the harness and briefing before the
|
|
949
|
+
// agent creates/binds its identity, and absence before then is not a
|
|
950
|
+
// close event. After readiness, debounce a real disappearance.
|
|
951
|
+
if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
952
|
+
retirementReason = 'identity-closed';
|
|
953
|
+
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
954
|
+
await sessionHandle.close();
|
|
955
|
+
sessionClosed = true;
|
|
956
|
+
break;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
else if (presence.state === 'unknown')
|
|
960
|
+
identityAbsentSince = undefined;
|
|
948
961
|
}
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
962
|
+
if (reloadLoopConfig && now >= nextLoopReloadAt) {
|
|
963
|
+
nextLoopReloadAt = now + 30_000;
|
|
964
|
+
try {
|
|
965
|
+
await reloadLoopConfig();
|
|
966
|
+
lastReloadError = '';
|
|
967
|
+
}
|
|
968
|
+
catch (error) {
|
|
969
|
+
const message = error?.message ?? String(error);
|
|
970
|
+
if (message !== lastReloadError)
|
|
971
|
+
deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
|
|
972
|
+
lastReloadError = message;
|
|
973
|
+
}
|
|
954
974
|
}
|
|
955
975
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
}
|
|
980
|
-
else
|
|
981
|
-
ownerBinder?.release();
|
|
982
|
-
if (monitor) {
|
|
983
|
-
monitor.stop();
|
|
984
|
-
await monitorLoop;
|
|
985
|
-
}
|
|
986
|
-
unsubscribeRecovery?.();
|
|
987
|
-
if (agentSession && !sessionClosed)
|
|
988
|
-
await agentSession.close();
|
|
989
|
-
const elapsed = (deps.now() - start) / 1000;
|
|
990
|
-
// Establish what actually happened before deciding anything. Absence of a
|
|
991
|
-
// record is `unknown` — except when the console itself is gone, which is a
|
|
992
|
-
// different event with a different consequence.
|
|
993
|
-
const exitRecord = agentSession.exitResult()
|
|
994
|
-
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
995
|
-
writeFileSync(exitFile, JSON.stringify({
|
|
996
|
-
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
997
|
-
}) + '\n');
|
|
998
|
-
let rotated = false;
|
|
999
|
-
const rotate = (why) => {
|
|
1000
|
-
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1001
|
-
rmSync(bootedFile, { force: true });
|
|
1002
|
-
rotated = true;
|
|
1003
|
-
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1004
|
-
};
|
|
1005
|
-
if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1006
|
-
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1007
|
-
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1008
|
-
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1009
|
-
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1010
|
-
// opens its circuit after the configured number of consecutive failures.
|
|
1011
|
-
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1012
|
-
`-> next start RESUMES context`);
|
|
1013
|
-
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1014
|
-
rotate(`clean exit (code 0)`);
|
|
1015
|
-
else if (exitRecord.class === 'session-destroyed')
|
|
1016
|
-
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1017
|
-
// would discard a live conversation for an operator action.
|
|
1018
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1019
|
-
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1020
|
-
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1021
|
-
// on every attempt would discard the conversation again and again while the
|
|
1022
|
-
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1023
|
-
if (opts.allowResumeRotation === false)
|
|
1024
|
-
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1025
|
-
`-> resume state was already discarded once; keeping it`);
|
|
976
|
+
if (loopManager) {
|
|
977
|
+
control?.setLoopManager(undefined);
|
|
978
|
+
await loopManager.stop();
|
|
979
|
+
}
|
|
980
|
+
// Let already-settled path operations publish their aggregate result before
|
|
981
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
982
|
+
await Promise.resolve();
|
|
983
|
+
await Promise.resolve();
|
|
984
|
+
recoveryController?.cancel();
|
|
985
|
+
control?.setConfigReloader(undefined);
|
|
986
|
+
// Close the authenticated control route before releasing the binder lease;
|
|
987
|
+
// otherwise the predecessor can unlink the replacement's new socket.
|
|
988
|
+
if (control) {
|
|
989
|
+
control.setOwnerChannel(undefined);
|
|
990
|
+
await control.close();
|
|
991
|
+
control = undefined;
|
|
992
|
+
}
|
|
993
|
+
if (ownerChannel)
|
|
994
|
+
await ownerChannel.close();
|
|
995
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
996
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
997
|
+
supervisorRecycleRequired = true;
|
|
998
|
+
}
|
|
1026
999
|
else
|
|
1027
|
-
|
|
1000
|
+
ownerBinder?.release();
|
|
1001
|
+
if (monitor) {
|
|
1002
|
+
monitor.stop();
|
|
1003
|
+
await monitorLoop;
|
|
1004
|
+
}
|
|
1005
|
+
unsubscribeRecovery?.();
|
|
1006
|
+
if (agentSession && !sessionClosed)
|
|
1007
|
+
await agentSession.close();
|
|
1008
|
+
const elapsed = (deps.now() - start) / 1000;
|
|
1009
|
+
// Establish what actually happened before deciding anything. Absence of a
|
|
1010
|
+
// record is `unknown` — except when the console itself is gone, which is a
|
|
1011
|
+
// different event with a different consequence.
|
|
1012
|
+
const exitRecord = agentSession.exitResult()
|
|
1013
|
+
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
1014
|
+
writeFileSync(exitFile, JSON.stringify({
|
|
1015
|
+
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
1016
|
+
}) + '\n');
|
|
1017
|
+
let rotated = false;
|
|
1018
|
+
const rotate = (why) => {
|
|
1019
|
+
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1020
|
+
rmSync(bootedFile, { force: true });
|
|
1021
|
+
rotated = true;
|
|
1022
|
+
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1023
|
+
};
|
|
1024
|
+
if (!temp && deps.shouldStop?.())
|
|
1025
|
+
deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
|
|
1026
|
+
else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1027
|
+
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1028
|
+
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1029
|
+
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1030
|
+
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1031
|
+
// opens its circuit after the configured number of consecutive failures.
|
|
1032
|
+
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1033
|
+
`-> next start RESUMES context`);
|
|
1034
|
+
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1035
|
+
rotate(`clean exit (code 0)`);
|
|
1036
|
+
else if (exitRecord.class === 'session-destroyed')
|
|
1037
|
+
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1038
|
+
// would discard a live conversation for an operator action.
|
|
1039
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1040
|
+
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1041
|
+
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1042
|
+
// on every attempt would discard the conversation again and again while the
|
|
1043
|
+
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1044
|
+
if (opts.allowResumeRotation === false)
|
|
1045
|
+
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1046
|
+
`-> resume state was already discarded once; keeping it`);
|
|
1047
|
+
else
|
|
1048
|
+
rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
|
|
1049
|
+
}
|
|
1050
|
+
else
|
|
1051
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1052
|
+
if (supervisorRecycleRequired)
|
|
1053
|
+
throw new SupervisorRecycleRequiredError();
|
|
1054
|
+
return {
|
|
1055
|
+
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1056
|
+
...(retirementReason ? { retirementReason } : {}),
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
finally {
|
|
1060
|
+
await managedService.close(false);
|
|
1028
1061
|
}
|
|
1029
|
-
else
|
|
1030
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1031
|
-
if (supervisorRecycleRequired)
|
|
1032
|
-
throw new SupervisorRecycleRequiredError();
|
|
1033
|
-
return {
|
|
1034
|
-
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1035
|
-
...(retirementReason ? { retirementReason } : {}),
|
|
1036
|
-
};
|
|
1037
1062
|
}
|
|
1038
1063
|
/**
|
|
1039
1064
|
* The persistent supervisor for one permanent role: run child sessions in a
|
|
@@ -1049,7 +1074,10 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1049
1074
|
const deps = { ...defaultDeps(), ...partialDeps };
|
|
1050
1075
|
const dir = agentDir(name);
|
|
1051
1076
|
mkdirSync(dir, { recursive: true });
|
|
1052
|
-
|
|
1077
|
+
let stopping = false;
|
|
1078
|
+
const requestStop = () => { stopping = true; };
|
|
1079
|
+
const shouldStop = () => stopping || (partialDeps.shouldStop?.() ?? false);
|
|
1080
|
+
deps.shouldStop = shouldStop;
|
|
1053
1081
|
const stamp = () => new Date(deps.now()).toISOString();
|
|
1054
1082
|
// Record how the PREVIOUS supervisor process ended before doing anything
|
|
1055
1083
|
// else. An external kill writes nothing itself, and the next ledger write is
|
|
@@ -1071,6 +1099,9 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1071
1099
|
deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
|
|
1072
1100
|
+ `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
|
|
1073
1101
|
}
|
|
1102
|
+
// Service-manager stops must run the same cleanup as a completed session.
|
|
1103
|
+
process.on('SIGTERM', requestStop);
|
|
1104
|
+
process.on('SIGINT', requestStop);
|
|
1074
1105
|
try {
|
|
1075
1106
|
while (!shouldStop()) {
|
|
1076
1107
|
let ledger = readRestartLedger(dir);
|
|
@@ -1125,6 +1156,8 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1125
1156
|
mode: 'fresh',
|
|
1126
1157
|
};
|
|
1127
1158
|
}
|
|
1159
|
+
if (stopping)
|
|
1160
|
+
break;
|
|
1128
1161
|
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1129
1162
|
// have reset the ledger meanwhile.
|
|
1130
1163
|
ledger = readRestartLedger(dir);
|
|
@@ -1189,8 +1222,12 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1189
1222
|
return readRestartLedger(dir);
|
|
1190
1223
|
}
|
|
1191
1224
|
finally {
|
|
1192
|
-
|
|
1193
|
-
|
|
1225
|
+
process.off('SIGTERM', requestStop);
|
|
1226
|
+
process.off('SIGINT', requestStop);
|
|
1227
|
+
// An orderly shutdown clears the marker; an unhandled signal or OOM-kill
|
|
1228
|
+
// leaves it so the successor can identify an abrupt termination.
|
|
1229
|
+
if (shouldStop())
|
|
1230
|
+
await deps.releaseAgentOurs(findRole(loadConfig(opts.configPath), name));
|
|
1194
1231
|
releaseSupervisorRun(dir);
|
|
1195
1232
|
}
|
|
1196
1233
|
}
|
|
@@ -1220,10 +1257,15 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1220
1257
|
let result;
|
|
1221
1258
|
let failure;
|
|
1222
1259
|
try {
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1260
|
+
for (let recoveryAttempt = 0;; recoveryAttempt++) {
|
|
1261
|
+
result = await attempt(name, { temp: true }, {
|
|
1262
|
+
...deps,
|
|
1263
|
+
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1264
|
+
});
|
|
1265
|
+
if (result.exit.class === 'clean' || result.retirementReason || signal || deps.shouldStop?.() || recoveryAttempt >= 2)
|
|
1266
|
+
break;
|
|
1267
|
+
await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(Math.min(5000, 1000 * (recoveryAttempt + 1)));
|
|
1268
|
+
}
|
|
1227
1269
|
}
|
|
1228
1270
|
catch (error) {
|
|
1229
1271
|
failure = error;
|
|
@@ -1231,23 +1273,27 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1231
1273
|
finally {
|
|
1232
1274
|
process.off('SIGTERM', onTerm);
|
|
1233
1275
|
process.off('SIGINT', onInt);
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
?
|
|
1247
|
-
:
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1276
|
+
// A recycle replaces the supervisor process, not the logical temporary agent.
|
|
1277
|
+
if (!(failure instanceof SupervisorRecycleRequiredError) || signal || deps.shouldStop?.()) {
|
|
1278
|
+
const requested = requestedTempStopReason(dir);
|
|
1279
|
+
const reason = requested
|
|
1280
|
+
?? result?.retirementReason
|
|
1281
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1282
|
+
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1283
|
+
// A service-manager stop can make the child connection close before the
|
|
1284
|
+
// runner reaches its normal loop. That is still a successful requested
|
|
1285
|
+
// retirement, not a startup failure.
|
|
1286
|
+
const outcome = failure && !requested && !signal ? 'failed' : 'retired';
|
|
1287
|
+
const detail = failure
|
|
1288
|
+
? (failure instanceof Error ? failure.message : String(failure))
|
|
1289
|
+
: result
|
|
1290
|
+
? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
|
|
1291
|
+
: 'temporary supervisor ended without an attempt result';
|
|
1292
|
+
await (deps.releaseAgentOurs ?? releaseManagedAgent)(loadTempRole(name));
|
|
1293
|
+
const archived = archiveTempState(name, reason, outcome, detail);
|
|
1294
|
+
deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
|
|
1295
|
+
+ `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
|
|
1296
|
+
}
|
|
1251
1297
|
}
|
|
1252
1298
|
if (failure)
|
|
1253
1299
|
throw failure;
|