@ours.network/fleet 1.2.0-nightly.2 → 1.2.0-nightly.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -21
- package/dist/agent-ours/bridge.d.ts +3 -0
- package/dist/agent-ours/bridge.js +169 -0
- package/dist/agent-ours/controller.d.ts +14 -0
- package/dist/agent-ours/controller.js +60 -0
- package/dist/agent-ours/file-rpc.d.ts +23 -0
- package/dist/agent-ours/file-rpc.js +123 -0
- package/dist/agent-ours/harness.d.ts +14 -0
- package/dist/agent-ours/harness.js +64 -0
- package/dist/agent-ours/mcp-endpoint.d.ts +15 -0
- package/dist/agent-ours/mcp-endpoint.js +111 -0
- package/dist/agent-ours/runtime.d.ts +53 -0
- package/dist/agent-ours/runtime.js +247 -0
- package/dist/agent-ours/service.d.ts +23 -0
- package/dist/agent-ours/service.js +352 -0
- package/dist/agent-ours/state.d.ts +45 -0
- package/dist/agent-ours/state.js +95 -0
- package/dist/agent-ours/wire.d.ts +14 -0
- package/dist/agent-ours/wire.js +55 -0
- package/dist/agent-recovery-gate.js +1 -1
- package/dist/briefing.js +10 -59
- package/dist/build-info.json +4 -4
- package/dist/cli.js +15 -2
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +3 -14
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +15 -20
- package/dist/doctor.js +2 -2
- package/dist/fleet-command-audit.js +1 -1
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +4 -2
- package/dist/harness/codex-session.js +9 -2
- package/dist/harness/hermes-session.js +7 -1
- package/dist/rooms-tasks/provision.js +9 -2
- package/dist/runner.d.ts +4 -0
- package/dist/runner.js +522 -492
- package/dist/session/codex-app-server-transport.js +1 -1
- package/dist/spawn.js +12 -16
- package/dist/temp-supervisor-recovery.d.ts +3 -0
- package/dist/temp-supervisor-recovery.js +46 -0
- package/dist/watchdog/briefing.js +3 -16
- package/dist/watchdog/run.js +19 -5
- package/package.json +5 -3
package/dist/runner.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { prepareManagedAgent, releaseManagedAgent } from './agent-ours/service.js';
|
|
2
|
+
import { prepareManagedHarness } from './agent-ours/harness.js';
|
|
1
3
|
import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'node:fs';
|
|
2
4
|
import { join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
3
6
|
import { randomUUID } from 'node:crypto';
|
|
4
7
|
import { parse } from 'yaml';
|
|
5
8
|
import { agentDir, stateRoot } from './paths.js';
|
|
@@ -7,7 +10,6 @@ import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolv
|
|
|
7
10
|
import { getAdapter } from './harness/registry.js';
|
|
8
11
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
12
|
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
-
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
11
13
|
import { realExec } from './exec.js';
|
|
12
14
|
import { resolveIsolation } from './isolation/policy.js';
|
|
13
15
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
@@ -36,6 +38,8 @@ export class SupervisorRecycleRequiredError extends Error {
|
|
|
36
38
|
}
|
|
37
39
|
}
|
|
38
40
|
const defaultDeps = () => ({
|
|
41
|
+
prepareAgentOurs: prepareManagedAgent,
|
|
42
|
+
releaseAgentOurs: releaseManagedAgent,
|
|
39
43
|
exec: realExec,
|
|
40
44
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
41
45
|
isAlive: pid => { try {
|
|
@@ -497,11 +501,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
497
501
|
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
498
502
|
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
499
503
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
504
|
+
role = { ...role, monitor: { ...role.monitor, mode: 'fleet' } };
|
|
500
505
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
501
506
|
const sessionBackend = role.session ?? 'acp';
|
|
502
507
|
const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
|
|
503
508
|
let launch = adapter.agentSession.prepareLaunch(role, prep);
|
|
504
|
-
//
|
|
509
|
+
// Preserve the role's existing isolation policy.
|
|
505
510
|
let wrappedArgv = launch.argv;
|
|
506
511
|
if (role.isolation) {
|
|
507
512
|
// Start with the same durable context that config validation and doctor judged,
|
|
@@ -511,7 +516,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
511
516
|
launch = { ...launch, argv: runtime.argv };
|
|
512
517
|
const ctx = {
|
|
513
518
|
...isolationContextFor(role), stateDir: dir, runCwd,
|
|
514
|
-
runtimeReadPaths: runtime.readPaths,
|
|
519
|
+
runtimeReadPaths: [...runtime.readPaths, ...resolveLaunchRuntime([process.execPath, fileURLToPath(new URL('./agent-ours/bridge.js', import.meta.url))]).readPaths],
|
|
515
520
|
};
|
|
516
521
|
const policy = resolveIsolation(role.isolation, ctx);
|
|
517
522
|
const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
|
|
@@ -533,511 +538,525 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
533
538
|
if (rprefix.length)
|
|
534
539
|
wrappedArgv = [...rprefix, ...wrappedArgv];
|
|
535
540
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
if (
|
|
546
|
-
|
|
547
|
-
|
|
541
|
+
const managedService = await deps.prepareAgentOurs(role, dir, opts.identityLifetime ? opts.identityLifetime === 'temporary' : temp);
|
|
542
|
+
try {
|
|
543
|
+
const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, harnessChildEnv(role, launch.env, dir));
|
|
544
|
+
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
545
|
+
// agent launch across the whole host, so a burst of boots (systemd starts every
|
|
546
|
+
// user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
|
|
547
|
+
// hit the harness/API rate limit at once. Time-based via a shared launch gate, so
|
|
548
|
+
// a lone start or a solo crash-restart waits zero. Applied right before the harness
|
|
549
|
+
// agent-session start; the cheap monitor prime still runs immediately after.
|
|
550
|
+
if (staggerMs > 0) {
|
|
551
|
+
const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
|
|
552
|
+
const wait = slot - deps.now();
|
|
553
|
+
if (wait > 0) {
|
|
554
|
+
deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
|
|
555
|
+
await deps.sleep(wait);
|
|
556
|
+
}
|
|
548
557
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
558
|
+
// Prime the supervisor mail monitor's notification cursor at the
|
|
559
|
+
// stream tip BEFORE the session launches so no arrival is missed during boot
|
|
560
|
+
// (backlog before the tip is the SessionStart hook's job). Native-mode roles
|
|
561
|
+
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
562
|
+
// treated as native (monitor may be undefined on an old role.yaml).
|
|
563
|
+
let sessionHandle;
|
|
564
|
+
let modelRecovery;
|
|
565
|
+
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
566
|
+
resolvedMonitorDeps.onFailureEvidence = evidence => {
|
|
567
|
+
if (!role.model_chain)
|
|
568
|
+
return false;
|
|
569
|
+
const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
|
|
570
|
+
if (action.kind === 'advance') {
|
|
571
|
+
modelRecovery = 'advance';
|
|
572
|
+
deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
|
|
573
|
+
void sessionHandle?.close();
|
|
574
|
+
return true;
|
|
575
|
+
}
|
|
576
|
+
else if (action.kind === 'hold') {
|
|
577
|
+
modelRecovery = 'hold';
|
|
578
|
+
deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
|
|
579
|
+
void sessionHandle?.close();
|
|
580
|
+
return true;
|
|
581
|
+
}
|
|
560
582
|
return false;
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
583
|
+
};
|
|
584
|
+
const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
|
|
585
|
+
const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
|
|
586
|
+
const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
|
|
587
|
+
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
588
|
+
deps: resolvedMonitorDeps,
|
|
589
|
+
}) : null;
|
|
590
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
591
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
592
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
593
|
+
if (monitor)
|
|
594
|
+
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
595
|
+
rmSync(exitFile, { force: true });
|
|
596
|
+
let pid;
|
|
597
|
+
let agentSession;
|
|
598
|
+
let control;
|
|
599
|
+
let unsubscribeRecovery;
|
|
600
|
+
let monitorLoop;
|
|
601
|
+
let sessionStartupComplete = false;
|
|
602
|
+
let sessionClosed = false;
|
|
603
|
+
let ownerChannel;
|
|
604
|
+
let ownerBinder;
|
|
605
|
+
let loopManager;
|
|
606
|
+
let arbiter;
|
|
607
|
+
let recoveryController;
|
|
608
|
+
let reloadLoopConfig;
|
|
609
|
+
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
610
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
611
|
+
]));
|
|
612
|
+
{
|
|
613
|
+
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
614
|
+
// Say once, at startup, that this role will decide permission requests by
|
|
615
|
+
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
616
|
+
// that quietly did less than it was asked to.
|
|
617
|
+
if (perms.unattended === 'deny')
|
|
618
|
+
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
619
|
+
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
620
|
+
agentSession = await managedService.runtime.startHarness(() => deps.startAgentSession(adapter.agentSession, {
|
|
621
|
+
managedOurs: managedHarness.ours,
|
|
622
|
+
role, prep,
|
|
623
|
+
launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
|
|
624
|
+
cwd: runCwd, stateDir: dir, mode, permissions: perms,
|
|
625
|
+
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
626
|
+
}));
|
|
627
|
+
pid = agentSession.pid;
|
|
628
|
+
arbiter = new RoleTurnArbiter(agentSession);
|
|
629
|
+
sessionHandle = arbiter;
|
|
630
|
+
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
631
|
+
if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
|
|
632
|
+
return;
|
|
633
|
+
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
634
|
+
if (evidence)
|
|
635
|
+
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
636
|
+
});
|
|
637
|
+
if (role.owner_channel) {
|
|
638
|
+
try {
|
|
639
|
+
ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
|
|
640
|
+
}
|
|
641
|
+
catch (error) {
|
|
642
|
+
if (error instanceof OwnerBinderHandoffTimeoutError) {
|
|
643
|
+
try {
|
|
644
|
+
const status = await deps.reportOwnerStartupFailure(dir);
|
|
645
|
+
deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
|
|
646
|
+
}
|
|
647
|
+
catch (notifyError) {
|
|
648
|
+
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
649
|
+
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
await agentSession.close();
|
|
653
|
+
unsubscribeRecovery?.();
|
|
654
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
655
|
+
+ `${error?.message ?? String(error)}`);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
629
658
|
try {
|
|
630
|
-
|
|
659
|
+
if (role.owner_channel)
|
|
660
|
+
ownerChannel = deps.createOwnerChannel({
|
|
661
|
+
role: name,
|
|
662
|
+
harness: role.harness,
|
|
663
|
+
config: role.owner_channel,
|
|
664
|
+
session: arbiter,
|
|
665
|
+
stateDir: dir,
|
|
666
|
+
env: role.env,
|
|
667
|
+
log: deps.log,
|
|
668
|
+
...(ownerBinder ? { binderLease: ownerBinder } : {}),
|
|
669
|
+
...(configPath ? { configPath } : {}),
|
|
670
|
+
});
|
|
671
|
+
control = deps.createControlServer(dir, arbiter, deps.log);
|
|
672
|
+
control.setFleetAuditor(ownerChannel ? {
|
|
673
|
+
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
674
|
+
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
675
|
+
present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
|
|
676
|
+
} : localFleetAuditor(dir, name, deps.log));
|
|
677
|
+
await control.start();
|
|
631
678
|
}
|
|
632
679
|
catch (error) {
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
const status = await deps.reportOwnerStartupFailure(dir);
|
|
636
|
-
deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
|
|
637
|
-
}
|
|
638
|
-
catch (notifyError) {
|
|
639
|
-
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
640
|
-
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
641
|
-
}
|
|
642
|
-
}
|
|
680
|
+
await ownerChannel?.close().catch(() => undefined);
|
|
681
|
+
ownerBinder?.release();
|
|
643
682
|
await agentSession.close();
|
|
644
683
|
unsubscribeRecovery?.();
|
|
645
|
-
throw
|
|
646
|
-
+ `${error?.message ?? String(error)}`);
|
|
684
|
+
throw error;
|
|
647
685
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
});
|
|
662
|
-
control = deps.createControlServer(dir, arbiter, deps.log);
|
|
663
|
-
control.setFleetAuditor(ownerChannel ? {
|
|
664
|
-
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
665
|
-
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
666
|
-
present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
|
|
667
|
-
} : localFleetAuditor(dir, name, deps.log));
|
|
668
|
-
await control.start();
|
|
669
|
-
}
|
|
670
|
-
catch (error) {
|
|
671
|
-
await ownerChannel?.close().catch(() => undefined);
|
|
672
|
-
ownerBinder?.release();
|
|
673
|
-
await agentSession.close();
|
|
674
|
-
unsubscribeRecovery?.();
|
|
675
|
-
throw error;
|
|
676
|
-
}
|
|
677
|
-
if (!role.roomMemberStartup)
|
|
678
|
-
control.setFleetSpawner(async (requested) => {
|
|
679
|
-
const event = await executeManagedSpawn(role, configPath, requested, deps.log);
|
|
680
|
-
// Room-member launches are collected with their Task/Room transaction so
|
|
681
|
-
// causal delivery is Task → Room → Agents → Room active → Task active.
|
|
682
|
-
if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
|
|
683
|
-
try {
|
|
684
|
-
await ownerChannel.notifyFleetSpawn(event);
|
|
685
|
-
}
|
|
686
|
-
catch (error) {
|
|
687
|
-
deps.log(`[${name}] Agent lifecycle notice delivery failed: `
|
|
688
|
-
+ `${error?.message ?? String(error)}`);
|
|
686
|
+
if (!role.roomMemberStartup)
|
|
687
|
+
control.setFleetSpawner(async (requested) => {
|
|
688
|
+
const event = await executeManagedSpawn(role, configPath, requested, deps.log);
|
|
689
|
+
// Room-member launches are collected with their Task/Room transaction so
|
|
690
|
+
// causal delivery is Task → Room → Agents → Room active → Task active.
|
|
691
|
+
if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
|
|
692
|
+
try {
|
|
693
|
+
await ownerChannel.notifyFleetSpawn(event);
|
|
694
|
+
}
|
|
695
|
+
catch (error) {
|
|
696
|
+
deps.log(`[${name}] Agent lifecycle notice delivery failed: `
|
|
697
|
+
+ `${error?.message ?? String(error)}`);
|
|
698
|
+
}
|
|
689
699
|
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
719
|
-
? boundary.state === 'timeout'
|
|
720
|
-
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
721
|
-
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
722
|
-
: boundary
|
|
700
|
+
return event;
|
|
701
|
+
});
|
|
702
|
+
resolvedMonitorDeps.delivery = {
|
|
703
|
+
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
704
|
+
// refusal or a cancellation reached the agent and was not acted on, so
|
|
705
|
+
// the monitor must keep its cursor and try again.
|
|
706
|
+
submit: async (text, options) => {
|
|
707
|
+
// Cancelling the runner-owned startup prompt makes startup look failed
|
|
708
|
+
// and closes the session before the wake turn can run. During startup,
|
|
709
|
+
// steer into the live turn instead; after it completes, honor the
|
|
710
|
+
// configured interrupt policy normally.
|
|
711
|
+
const policy = options?.interrupt;
|
|
712
|
+
const interrupt = policy === true && sessionStartupComplete;
|
|
713
|
+
const promptOptions = {
|
|
714
|
+
interrupt, steer: true,
|
|
715
|
+
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
716
|
+
origin: { kind: 'fleet-monitor' },
|
|
717
|
+
};
|
|
718
|
+
// Startup is already a protected boundary: as with immediate mode,
|
|
719
|
+
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
720
|
+
const result = policy === 'after_tool' && sessionStartupComplete
|
|
721
|
+
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
722
|
+
: await arbiter.submitPrompt(text, promptOptions);
|
|
723
|
+
const steered = result.accepted
|
|
724
|
+
&& (result.detail === 'injected' || result.detail === 'startedNewTurn');
|
|
725
|
+
const boundary = result.safeBoundary;
|
|
726
|
+
const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
|
|
727
|
+
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
723
728
|
? boundary.state === 'timeout'
|
|
724
|
-
? `after_tool timed out after ${boundary.waitedMs}ms;
|
|
725
|
-
: boundary.state
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
:
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
unsubscribeRecovery?.();
|
|
765
|
-
if (modelRecovery) {
|
|
766
|
-
if (monitorLoop)
|
|
767
|
-
await monitorLoop;
|
|
768
|
-
return {
|
|
769
|
-
elapsedSecs: 0,
|
|
770
|
-
exit: {
|
|
771
|
-
version: 1,
|
|
772
|
-
class: 'program-exit',
|
|
773
|
-
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
774
|
-
},
|
|
775
|
-
rotated: false,
|
|
776
|
-
mode,
|
|
777
|
-
modelRecovery,
|
|
778
|
-
};
|
|
779
|
-
}
|
|
780
|
-
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
781
|
-
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
782
|
-
}
|
|
783
|
-
if (started.cancellationSource === 'stall-watchdog')
|
|
784
|
-
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
785
|
-
else if (interruptedForWake)
|
|
786
|
-
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
787
|
-
+ 'keeping temporary supervisor alive');
|
|
788
|
-
sessionStartupComplete = true;
|
|
789
|
-
if (ownerChannel) {
|
|
790
|
-
try {
|
|
791
|
-
await ownerChannel.start();
|
|
792
|
-
}
|
|
793
|
-
catch (error) {
|
|
729
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
730
|
+
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
731
|
+
: boundary
|
|
732
|
+
? boundary.state === 'timeout'
|
|
733
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
|
|
734
|
+
: boundary.state === 'unsupported'
|
|
735
|
+
? 'after_tool unsupported; used non-cancelling queued delivery'
|
|
736
|
+
: `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
|
|
737
|
+
: result.detail;
|
|
738
|
+
return {
|
|
739
|
+
succeeded: result.succeeded || steered,
|
|
740
|
+
outcome: steered ? result.detail : result.outcome,
|
|
741
|
+
detail: result.succeeded || steered || !boundary
|
|
742
|
+
? boundaryDetail
|
|
743
|
+
: [result.detail, boundaryDetail].filter(Boolean).join('; '),
|
|
744
|
+
...(boundary ? { safeBoundary: boundary.state } : {}),
|
|
745
|
+
};
|
|
746
|
+
},
|
|
747
|
+
};
|
|
748
|
+
const firstPrompt = mode === 'fresh'
|
|
749
|
+
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
750
|
+
: `Your supervisor has verified your assigned identity and room readiness. Read ${join(dir, 'WORKLOG.md')} and ${join(dir, 'briefing.md')}, then continue using the available ours tools.`;
|
|
751
|
+
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
752
|
+
// startup prompt and then refuses it has not started; logging the role as
|
|
753
|
+
// up would hide a role that never read its briefing.
|
|
754
|
+
const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
|
|
755
|
+
// Monitoring starts immediately. The delivery adapter above downgrades
|
|
756
|
+
// interruption to steering until this startup turn reaches a terminal
|
|
757
|
+
// success, so there is neither a deaf gap nor a boot-cancellation loop.
|
|
758
|
+
monitorLoop = monitor?.run(pid);
|
|
759
|
+
const started = await starting;
|
|
760
|
+
// A temporary role's first turn can be the active turn when an ours wake
|
|
761
|
+
// needs immediate attention. A typed console/monitor cancellation ends
|
|
762
|
+
// only that turn: the already-live agent session and any queued wake remain
|
|
763
|
+
// valid. Keep every unproven cancellation, refusal, shutdown, and genuine
|
|
764
|
+
// failure terminal so a role that never accepted its briefing is not
|
|
765
|
+
// silently reported as healthy.
|
|
766
|
+
const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
|
|
767
|
+
|| (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
|
|
768
|
+
if (!started.succeeded && !interruptedForWake) {
|
|
794
769
|
monitor?.stop();
|
|
795
|
-
if (monitorLoop)
|
|
796
|
-
await monitorLoop;
|
|
797
|
-
await ownerChannel.close().catch(() => undefined);
|
|
798
770
|
await control.close();
|
|
799
771
|
ownerBinder?.release();
|
|
800
772
|
await agentSession.close();
|
|
801
773
|
unsubscribeRecovery?.();
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
now: deps.now,
|
|
820
|
-
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
821
|
-
clearTimer: timer => clearTimeout(timer),
|
|
822
|
-
log: deps.log,
|
|
823
|
-
});
|
|
824
|
-
control.setLoopManager(loopManager);
|
|
825
|
-
loopManager.start();
|
|
826
|
-
}
|
|
827
|
-
else {
|
|
828
|
-
loopManager?.reconcile(definitions);
|
|
829
|
-
}
|
|
830
|
-
loopGeneration = generation;
|
|
831
|
-
deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
|
|
832
|
-
return { changed: true, loops: definitions.length };
|
|
833
|
-
};
|
|
834
|
-
// Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
|
|
835
|
-
if (!temp)
|
|
836
|
-
control.setConfigReloader(reloadLoopConfig);
|
|
837
|
-
if (role.loops?.length) {
|
|
838
|
-
try {
|
|
839
|
-
loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
|
|
840
|
-
now: deps.now,
|
|
841
|
-
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
842
|
-
clearTimer: timer => clearTimeout(timer),
|
|
843
|
-
log: deps.log,
|
|
844
|
-
});
|
|
845
|
-
control.setLoopManager(loopManager);
|
|
846
|
-
loopManager.start();
|
|
774
|
+
if (modelRecovery) {
|
|
775
|
+
if (monitorLoop)
|
|
776
|
+
await monitorLoop;
|
|
777
|
+
return {
|
|
778
|
+
elapsedSecs: 0,
|
|
779
|
+
exit: {
|
|
780
|
+
version: 1,
|
|
781
|
+
class: 'program-exit',
|
|
782
|
+
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
783
|
+
},
|
|
784
|
+
rotated: false,
|
|
785
|
+
mode,
|
|
786
|
+
modelRecovery,
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
790
|
+
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
847
791
|
}
|
|
848
|
-
|
|
849
|
-
|
|
792
|
+
if (started.cancellationSource === 'stall-watchdog')
|
|
793
|
+
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
794
|
+
else if (interruptedForWake)
|
|
795
|
+
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
796
|
+
+ 'keeping temporary supervisor alive');
|
|
797
|
+
sessionStartupComplete = true;
|
|
798
|
+
if (ownerChannel) {
|
|
799
|
+
try {
|
|
800
|
+
await ownerChannel.start();
|
|
801
|
+
}
|
|
802
|
+
catch (error) {
|
|
850
803
|
monitor?.stop();
|
|
851
804
|
if (monitorLoop)
|
|
852
805
|
await monitorLoop;
|
|
806
|
+
await ownerChannel.close().catch(() => undefined);
|
|
853
807
|
await control.close();
|
|
854
808
|
ownerBinder?.release();
|
|
855
809
|
await agentSession.close();
|
|
856
810
|
unsubscribeRecovery?.();
|
|
857
|
-
throw new Error(`[${name}]
|
|
811
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
858
812
|
+ `${error?.message ?? String(error)}`);
|
|
859
813
|
}
|
|
860
|
-
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
861
814
|
}
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
865
|
-
// The monitor loop lives exactly as long as the session: it starts once the
|
|
866
|
-
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
867
|
-
monitorLoop ??= monitor?.run(pid);
|
|
868
|
-
recoveryController = new RoleRecoveryController({
|
|
869
|
-
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
870
|
-
log: deps.log,
|
|
871
|
-
recoverAgent: async () => {
|
|
872
|
-
const evidence = await recoverAgentIdentity(arbiter, role.identity);
|
|
873
|
-
return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
|
|
874
|
-
},
|
|
875
|
-
recoverOwner: async (epoch) => {
|
|
876
|
-
if (!ownerChannel?.recover)
|
|
877
|
-
return { ok: true };
|
|
878
|
-
try {
|
|
879
|
-
await ownerChannel.recover(epoch);
|
|
880
|
-
return { ok: true };
|
|
881
|
-
}
|
|
882
|
-
catch (error) {
|
|
883
|
-
return { ok: false, reason: error instanceof Error
|
|
884
|
-
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
815
|
+
if (ownerChannel) {
|
|
816
|
+
control.setOwnerChannel(ownerChannel);
|
|
885
817
|
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
if (temp)
|
|
913
|
-
|
|
914
|
-
|
|
818
|
+
reloadLoopConfig = async () => {
|
|
819
|
+
const nextRole = findRole(loadConfig(configPath), name);
|
|
820
|
+
const definitions = nextRole.loops ?? [];
|
|
821
|
+
const generation = JSON.stringify(definitions.map(loop => [
|
|
822
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
823
|
+
]));
|
|
824
|
+
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
825
|
+
return { changed: false, loops: definitions.length };
|
|
826
|
+
if (!loopManager && definitions.length) {
|
|
827
|
+
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
828
|
+
now: deps.now,
|
|
829
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
830
|
+
clearTimer: timer => clearTimeout(timer),
|
|
831
|
+
log: deps.log,
|
|
832
|
+
});
|
|
833
|
+
control.setLoopManager(loopManager);
|
|
834
|
+
loopManager.start();
|
|
835
|
+
}
|
|
836
|
+
else {
|
|
837
|
+
loopManager?.reconcile(definitions);
|
|
838
|
+
}
|
|
839
|
+
loopGeneration = generation;
|
|
840
|
+
deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
|
|
841
|
+
return { changed: true, loops: definitions.length };
|
|
842
|
+
};
|
|
843
|
+
// Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
|
|
844
|
+
if (!temp)
|
|
845
|
+
control.setConfigReloader(reloadLoopConfig);
|
|
846
|
+
if (role.loops?.length) {
|
|
847
|
+
try {
|
|
848
|
+
loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
|
|
849
|
+
now: deps.now,
|
|
850
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
851
|
+
clearTimer: timer => clearTimeout(timer),
|
|
852
|
+
log: deps.log,
|
|
853
|
+
});
|
|
854
|
+
control.setLoopManager(loopManager);
|
|
855
|
+
loopManager.start();
|
|
856
|
+
}
|
|
857
|
+
catch (error) {
|
|
858
|
+
if (temp) {
|
|
859
|
+
monitor?.stop();
|
|
860
|
+
if (monitorLoop)
|
|
861
|
+
await monitorLoop;
|
|
862
|
+
await control.close();
|
|
863
|
+
ownerBinder?.release();
|
|
864
|
+
await agentSession.close();
|
|
865
|
+
unsubscribeRecovery?.();
|
|
866
|
+
throw new Error(`[${name}] configured temporary loop manager failed to start: `
|
|
867
|
+
+ `${error?.message ?? String(error)}`);
|
|
868
|
+
}
|
|
869
|
+
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
870
|
+
}
|
|
915
871
|
}
|
|
916
|
-
await sessionHandle.close();
|
|
917
|
-
sessionClosed = true;
|
|
918
|
-
break;
|
|
919
872
|
}
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
873
|
+
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
874
|
+
// The monitor loop lives exactly as long as the session: it starts once the
|
|
875
|
+
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
876
|
+
monitorLoop ??= monitor?.run(pid);
|
|
877
|
+
recoveryController = new RoleRecoveryController({
|
|
878
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
879
|
+
log: deps.log,
|
|
880
|
+
recoverAgent: async () => {
|
|
881
|
+
try {
|
|
882
|
+
const release = await managedService.runtime.admit();
|
|
883
|
+
release();
|
|
884
|
+
return { ok: true };
|
|
885
|
+
}
|
|
886
|
+
catch {
|
|
887
|
+
return { ok: false, reason: 'SUPERVISOR_IDENTITY_NOT_READY' };
|
|
888
|
+
}
|
|
889
|
+
},
|
|
890
|
+
recoverOwner: async (epoch) => {
|
|
891
|
+
if (!ownerChannel?.recover)
|
|
892
|
+
return { ok: true };
|
|
893
|
+
try {
|
|
894
|
+
await ownerChannel.recover(epoch);
|
|
895
|
+
return { ok: true };
|
|
896
|
+
}
|
|
897
|
+
catch (error) {
|
|
898
|
+
return { ok: false, reason: error instanceof Error
|
|
899
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
900
|
+
}
|
|
901
|
+
},
|
|
902
|
+
});
|
|
903
|
+
const start = deps.now();
|
|
904
|
+
let nextLoopReloadAt = deps.now() + 30_000;
|
|
905
|
+
let nextIdentityPollAt = deps.now();
|
|
906
|
+
let nextDaemonProbeAt = deps.now();
|
|
907
|
+
let lastReloadError = '';
|
|
908
|
+
let identityObserved = false;
|
|
909
|
+
let identityAbsentSince;
|
|
910
|
+
let retirementReason;
|
|
911
|
+
let supervisorRecycleRequired = false;
|
|
912
|
+
while (sessionHandle.isAlive()) {
|
|
913
|
+
await deps.sleep(temp ? 500 : 2000);
|
|
914
|
+
const now = deps.now();
|
|
915
|
+
if (now >= nextDaemonProbeAt) {
|
|
916
|
+
nextDaemonProbeAt = now + 2_000;
|
|
917
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
918
|
+
if (observation.kind === 'lost')
|
|
919
|
+
recoveryController.noteLoss(observation.reason);
|
|
920
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
921
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
922
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
923
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
924
|
+
}
|
|
926
925
|
}
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
// starts may spend minutes loading the harness and briefing before the
|
|
932
|
-
// agent creates/binds its identity, and absence before then is not a
|
|
933
|
-
// close event. After readiness, debounce a real disappearance.
|
|
934
|
-
if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
935
|
-
retirementReason = 'identity-closed';
|
|
936
|
-
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
937
|
-
await sessionHandle.close();
|
|
938
|
-
sessionClosed = true;
|
|
939
|
-
break;
|
|
926
|
+
if (deps.shouldStop?.()) {
|
|
927
|
+
if (temp) {
|
|
928
|
+
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
929
|
+
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
940
930
|
}
|
|
931
|
+
await sessionHandle.close();
|
|
932
|
+
sessionClosed = true;
|
|
933
|
+
break;
|
|
941
934
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
935
|
+
if (temp && now >= nextIdentityPollAt) {
|
|
936
|
+
nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
|
|
937
|
+
const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
|
|
938
|
+
if (presence.state === 'present') {
|
|
939
|
+
identityObserved = true;
|
|
940
|
+
identityAbsentSince = undefined;
|
|
941
|
+
}
|
|
942
|
+
else if (presence.state === 'absent' && identityObserved) {
|
|
943
|
+
identityAbsentSince ??= now;
|
|
944
|
+
// Require a continuous, time-bounded run of authoritative absence.
|
|
945
|
+
// The first positive observation is the readiness gate: cold harness
|
|
946
|
+
// starts may spend minutes loading the harness and briefing before the
|
|
947
|
+
// agent creates/binds its identity, and absence before then is not a
|
|
948
|
+
// close event. After readiness, debounce a real disappearance.
|
|
949
|
+
if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
950
|
+
retirementReason = 'identity-closed';
|
|
951
|
+
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
952
|
+
await sessionHandle.close();
|
|
953
|
+
sessionClosed = true;
|
|
954
|
+
break;
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
else if (presence.state === 'unknown')
|
|
958
|
+
identityAbsentSince = undefined;
|
|
950
959
|
}
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
960
|
+
if (reloadLoopConfig && now >= nextLoopReloadAt) {
|
|
961
|
+
nextLoopReloadAt = now + 30_000;
|
|
962
|
+
try {
|
|
963
|
+
await reloadLoopConfig();
|
|
964
|
+
lastReloadError = '';
|
|
965
|
+
}
|
|
966
|
+
catch (error) {
|
|
967
|
+
const message = error?.message ?? String(error);
|
|
968
|
+
if (message !== lastReloadError)
|
|
969
|
+
deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
|
|
970
|
+
lastReloadError = message;
|
|
971
|
+
}
|
|
956
972
|
}
|
|
957
973
|
}
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
}
|
|
982
|
-
else
|
|
983
|
-
ownerBinder?.release();
|
|
984
|
-
if (monitor) {
|
|
985
|
-
monitor.stop();
|
|
986
|
-
await monitorLoop;
|
|
987
|
-
}
|
|
988
|
-
unsubscribeRecovery?.();
|
|
989
|
-
if (agentSession && !sessionClosed)
|
|
990
|
-
await agentSession.close();
|
|
991
|
-
const elapsed = (deps.now() - start) / 1000;
|
|
992
|
-
// Establish what actually happened before deciding anything. Absence of a
|
|
993
|
-
// record is `unknown` — except when the console itself is gone, which is a
|
|
994
|
-
// different event with a different consequence.
|
|
995
|
-
const exitRecord = agentSession.exitResult()
|
|
996
|
-
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
997
|
-
writeFileSync(exitFile, JSON.stringify({
|
|
998
|
-
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
999
|
-
}) + '\n');
|
|
1000
|
-
let rotated = false;
|
|
1001
|
-
const rotate = (why) => {
|
|
1002
|
-
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1003
|
-
rmSync(bootedFile, { force: true });
|
|
1004
|
-
rotated = true;
|
|
1005
|
-
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1006
|
-
};
|
|
1007
|
-
if (!temp && deps.shouldStop?.())
|
|
1008
|
-
deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
|
|
1009
|
-
else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1010
|
-
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1011
|
-
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1012
|
-
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1013
|
-
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1014
|
-
// opens its circuit after the configured number of consecutive failures.
|
|
1015
|
-
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1016
|
-
`-> next start RESUMES context`);
|
|
1017
|
-
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1018
|
-
rotate(`clean exit (code 0)`);
|
|
1019
|
-
else if (exitRecord.class === 'session-destroyed')
|
|
1020
|
-
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1021
|
-
// would discard a live conversation for an operator action.
|
|
1022
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1023
|
-
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1024
|
-
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1025
|
-
// on every attempt would discard the conversation again and again while the
|
|
1026
|
-
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1027
|
-
if (opts.allowResumeRotation === false)
|
|
1028
|
-
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1029
|
-
`-> resume state was already discarded once; keeping it`);
|
|
974
|
+
if (loopManager) {
|
|
975
|
+
control?.setLoopManager(undefined);
|
|
976
|
+
await loopManager.stop();
|
|
977
|
+
}
|
|
978
|
+
// Let already-settled path operations publish their aggregate result before
|
|
979
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
980
|
+
await Promise.resolve();
|
|
981
|
+
await Promise.resolve();
|
|
982
|
+
recoveryController?.cancel();
|
|
983
|
+
control?.setConfigReloader(undefined);
|
|
984
|
+
// Close the authenticated control route before releasing the binder lease;
|
|
985
|
+
// otherwise the predecessor can unlink the replacement's new socket.
|
|
986
|
+
if (control) {
|
|
987
|
+
control.setOwnerChannel(undefined);
|
|
988
|
+
await control.close();
|
|
989
|
+
control = undefined;
|
|
990
|
+
}
|
|
991
|
+
if (ownerChannel)
|
|
992
|
+
await ownerChannel.close();
|
|
993
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
994
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
995
|
+
supervisorRecycleRequired = true;
|
|
996
|
+
}
|
|
1030
997
|
else
|
|
1031
|
-
|
|
998
|
+
ownerBinder?.release();
|
|
999
|
+
if (monitor) {
|
|
1000
|
+
monitor.stop();
|
|
1001
|
+
await monitorLoop;
|
|
1002
|
+
}
|
|
1003
|
+
unsubscribeRecovery?.();
|
|
1004
|
+
if (agentSession && !sessionClosed)
|
|
1005
|
+
await agentSession.close();
|
|
1006
|
+
const elapsed = (deps.now() - start) / 1000;
|
|
1007
|
+
// Establish what actually happened before deciding anything. Absence of a
|
|
1008
|
+
// record is `unknown` — except when the console itself is gone, which is a
|
|
1009
|
+
// different event with a different consequence.
|
|
1010
|
+
const exitRecord = agentSession.exitResult()
|
|
1011
|
+
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
1012
|
+
writeFileSync(exitFile, JSON.stringify({
|
|
1013
|
+
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
1014
|
+
}) + '\n');
|
|
1015
|
+
let rotated = false;
|
|
1016
|
+
const rotate = (why) => {
|
|
1017
|
+
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1018
|
+
rmSync(bootedFile, { force: true });
|
|
1019
|
+
rotated = true;
|
|
1020
|
+
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1021
|
+
};
|
|
1022
|
+
if (!temp && deps.shouldStop?.())
|
|
1023
|
+
deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
|
|
1024
|
+
else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1025
|
+
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1026
|
+
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1027
|
+
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1028
|
+
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1029
|
+
// opens its circuit after the configured number of consecutive failures.
|
|
1030
|
+
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1031
|
+
`-> next start RESUMES context`);
|
|
1032
|
+
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1033
|
+
rotate(`clean exit (code 0)`);
|
|
1034
|
+
else if (exitRecord.class === 'session-destroyed')
|
|
1035
|
+
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1036
|
+
// would discard a live conversation for an operator action.
|
|
1037
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1038
|
+
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1039
|
+
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1040
|
+
// on every attempt would discard the conversation again and again while the
|
|
1041
|
+
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1042
|
+
if (opts.allowResumeRotation === false)
|
|
1043
|
+
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1044
|
+
`-> resume state was already discarded once; keeping it`);
|
|
1045
|
+
else
|
|
1046
|
+
rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
|
|
1047
|
+
}
|
|
1048
|
+
else
|
|
1049
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1050
|
+
if (supervisorRecycleRequired)
|
|
1051
|
+
throw new SupervisorRecycleRequiredError();
|
|
1052
|
+
return {
|
|
1053
|
+
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1054
|
+
...(retirementReason ? { retirementReason } : {}),
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
finally {
|
|
1058
|
+
await managedService.close(false);
|
|
1032
1059
|
}
|
|
1033
|
-
else
|
|
1034
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1035
|
-
if (supervisorRecycleRequired)
|
|
1036
|
-
throw new SupervisorRecycleRequiredError();
|
|
1037
|
-
return {
|
|
1038
|
-
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1039
|
-
...(retirementReason ? { retirementReason } : {}),
|
|
1040
|
-
};
|
|
1041
1060
|
}
|
|
1042
1061
|
/**
|
|
1043
1062
|
* The persistent supervisor for one permanent role: run child sessions in a
|
|
@@ -1205,6 +1224,8 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1205
1224
|
process.off('SIGINT', requestStop);
|
|
1206
1225
|
// An orderly shutdown clears the marker; an unhandled signal or OOM-kill
|
|
1207
1226
|
// leaves it so the successor can identify an abrupt termination.
|
|
1227
|
+
if (shouldStop())
|
|
1228
|
+
await deps.releaseAgentOurs(findRole(loadConfig(opts.configPath), name));
|
|
1208
1229
|
releaseSupervisorRun(dir);
|
|
1209
1230
|
}
|
|
1210
1231
|
}
|
|
@@ -1234,10 +1255,15 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1234
1255
|
let result;
|
|
1235
1256
|
let failure;
|
|
1236
1257
|
try {
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1258
|
+
for (let recoveryAttempt = 0;; recoveryAttempt++) {
|
|
1259
|
+
result = await attempt(name, { temp: true }, {
|
|
1260
|
+
...deps,
|
|
1261
|
+
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1262
|
+
});
|
|
1263
|
+
if (result.exit.class === 'clean' || result.retirementReason || signal || deps.shouldStop?.() || recoveryAttempt >= 2)
|
|
1264
|
+
break;
|
|
1265
|
+
await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(Math.min(5000, 1000 * (recoveryAttempt + 1)));
|
|
1266
|
+
}
|
|
1241
1267
|
}
|
|
1242
1268
|
catch (error) {
|
|
1243
1269
|
failure = error;
|
|
@@ -1245,23 +1271,27 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1245
1271
|
finally {
|
|
1246
1272
|
process.off('SIGTERM', onTerm);
|
|
1247
1273
|
process.off('SIGINT', onInt);
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
?
|
|
1261
|
-
:
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1274
|
+
// A recycle replaces the supervisor process, not the logical temporary agent.
|
|
1275
|
+
if (!(failure instanceof SupervisorRecycleRequiredError) || signal || deps.shouldStop?.()) {
|
|
1276
|
+
const requested = requestedTempStopReason(dir);
|
|
1277
|
+
const reason = requested
|
|
1278
|
+
?? result?.retirementReason
|
|
1279
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1280
|
+
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1281
|
+
// A service-manager stop can make the child connection close before the
|
|
1282
|
+
// runner reaches its normal loop. That is still a successful requested
|
|
1283
|
+
// retirement, not a startup failure.
|
|
1284
|
+
const outcome = failure && !requested && !signal ? 'failed' : 'retired';
|
|
1285
|
+
const detail = failure
|
|
1286
|
+
? (failure instanceof Error ? failure.message : String(failure))
|
|
1287
|
+
: result
|
|
1288
|
+
? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
|
|
1289
|
+
: 'temporary supervisor ended without an attempt result';
|
|
1290
|
+
await (deps.releaseAgentOurs ?? releaseManagedAgent)(loadTempRole(name));
|
|
1291
|
+
const archived = archiveTempState(name, reason, outcome, detail);
|
|
1292
|
+
deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
|
|
1293
|
+
+ `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
|
|
1294
|
+
}
|
|
1265
1295
|
}
|
|
1266
1296
|
if (failure)
|
|
1267
1297
|
throw failure;
|