@ours.network/fleet 1.2.0-nightly.2 → 1.2.0-nightly.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +18 -21
  2. package/dist/agent-ours/bridge.d.ts +3 -0
  3. package/dist/agent-ours/bridge.js +169 -0
  4. package/dist/agent-ours/controller.d.ts +14 -0
  5. package/dist/agent-ours/controller.js +60 -0
  6. package/dist/agent-ours/file-rpc.d.ts +23 -0
  7. package/dist/agent-ours/file-rpc.js +123 -0
  8. package/dist/agent-ours/harness.d.ts +14 -0
  9. package/dist/agent-ours/harness.js +64 -0
  10. package/dist/agent-ours/mcp-endpoint.d.ts +15 -0
  11. package/dist/agent-ours/mcp-endpoint.js +111 -0
  12. package/dist/agent-ours/runtime.d.ts +53 -0
  13. package/dist/agent-ours/runtime.js +247 -0
  14. package/dist/agent-ours/service.d.ts +23 -0
  15. package/dist/agent-ours/service.js +352 -0
  16. package/dist/agent-ours/state.d.ts +45 -0
  17. package/dist/agent-ours/state.js +95 -0
  18. package/dist/agent-ours/wire.d.ts +14 -0
  19. package/dist/agent-ours/wire.js +55 -0
  20. package/dist/agent-recovery-gate.js +1 -1
  21. package/dist/briefing.js +10 -59
  22. package/dist/build-info.json +4 -4
  23. package/dist/cli.js +15 -2
  24. package/dist/creation.d.ts +1 -1
  25. package/dist/creation.js +3 -14
  26. package/dist/docs.d.ts +1 -1
  27. package/dist/docs.js +15 -20
  28. package/dist/doctor.js +2 -2
  29. package/dist/fleet-command-audit.js +1 -1
  30. package/dist/harness/agent-session.d.ts +2 -0
  31. package/dist/harness/claude-code-session.js +4 -2
  32. package/dist/harness/codex-session.js +9 -2
  33. package/dist/harness/hermes-session.js +7 -1
  34. package/dist/rooms-tasks/provision.js +9 -2
  35. package/dist/runner.d.ts +4 -0
  36. package/dist/runner.js +522 -492
  37. package/dist/session/codex-app-server-transport.js +1 -1
  38. package/dist/spawn.js +12 -16
  39. package/dist/temp-supervisor-recovery.d.ts +3 -0
  40. package/dist/temp-supervisor-recovery.js +46 -0
  41. package/dist/watchdog/briefing.js +3 -16
  42. package/dist/watchdog/run.js +19 -5
  43. package/package.json +5 -3
package/dist/runner.js CHANGED
@@ -1,5 +1,8 @@
1
+ import { prepareManagedAgent, releaseManagedAgent } from './agent-ours/service.js';
2
+ import { prepareManagedHarness } from './agent-ours/harness.js';
1
3
  import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'node:fs';
2
4
  import { join } from 'node:path';
5
+ import { fileURLToPath } from 'node:url';
3
6
  import { randomUUID } from 'node:crypto';
4
7
  import { parse } from 'yaml';
5
8
  import { agentDir, stateRoot } from './paths.js';
@@ -7,7 +10,6 @@ import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolv
7
10
  import { getAdapter } from './harness/registry.js';
8
11
  import { createMonitor, probeIdentityPresence, } from './monitor.js';
9
12
  import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
10
- import { recoverAgentIdentity } from './agent-recovery-gate.js';
11
13
  import { realExec } from './exec.js';
12
14
  import { resolveIsolation } from './isolation/policy.js';
13
15
  import { selectIsolationBackend } from './isolation/registry.js';
@@ -36,6 +38,8 @@ export class SupervisorRecycleRequiredError extends Error {
36
38
  }
37
39
  }
38
40
  const defaultDeps = () => ({
41
+ prepareAgentOurs: prepareManagedAgent,
42
+ releaseAgentOurs: releaseManagedAgent,
39
43
  exec: realExec,
40
44
  cpuDelegated: () => cpuControllerDelegated(),
41
45
  isAlive: pid => { try {
@@ -497,11 +501,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
497
501
  // already ran, so rewriting cannot change the fresh/resume decision.
498
502
  writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
499
503
  const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
504
+ role = { ...role, monitor: { ...role.monitor, mode: 'fleet' } };
500
505
  const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
501
506
  const sessionBackend = role.session ?? 'acp';
502
507
  const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
503
508
  let launch = adapter.agentSession.prepareLaunch(role, prep);
504
- // Isolation is additive: only roles that declare `isolation:` are wrapped.
509
+ // Preserve the role's existing isolation policy.
505
510
  let wrappedArgv = launch.argv;
506
511
  if (role.isolation) {
507
512
  // Start with the same durable context that config validation and doctor judged,
@@ -511,7 +516,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
511
516
  launch = { ...launch, argv: runtime.argv };
512
517
  const ctx = {
513
518
  ...isolationContextFor(role), stateDir: dir, runCwd,
514
- runtimeReadPaths: runtime.readPaths,
519
+ runtimeReadPaths: [...runtime.readPaths, ...resolveLaunchRuntime([process.execPath, fileURLToPath(new URL('./agent-ours/bridge.js', import.meta.url))]).readPaths],
515
520
  };
516
521
  const policy = resolveIsolation(role.isolation, ctx);
517
522
  const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
@@ -533,511 +538,525 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
533
538
  if (rprefix.length)
534
539
  wrappedArgv = [...rprefix, ...wrappedArgv];
535
540
  }
536
- // Start-stagger: space this launch at least start_stagger_ms after the previous
537
- // agent launch across the whole host, so a burst of boots (systemd starts every
538
- // user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
539
- // hit the harness/API rate limit at once. Time-based via a shared launch gate, so
540
- // a lone start or a solo crash-restart waits zero. Applied right before the harness
541
- // agent-session start; the cheap monitor prime still runs immediately after.
542
- if (staggerMs > 0) {
543
- const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
544
- const wait = slot - deps.now();
545
- if (wait > 0) {
546
- deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
547
- await deps.sleep(wait);
541
+ const managedService = await deps.prepareAgentOurs(role, dir, opts.identityLifetime ? opts.identityLifetime === 'temporary' : temp);
542
+ try {
543
+ const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, harnessChildEnv(role, launch.env, dir));
544
+ // Start-stagger: space this launch at least start_stagger_ms after the previous
545
+ // agent launch across the whole host, so a burst of boots (systemd starts every
546
+ // user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
547
+ // hit the harness/API rate limit at once. Time-based via a shared launch gate, so
548
+ // a lone start or a solo crash-restart waits zero. Applied right before the harness
549
+ // agent-session start; the cheap monitor prime still runs immediately after.
550
+ if (staggerMs > 0) {
551
+ const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
552
+ const wait = slot - deps.now();
553
+ if (wait > 0) {
554
+ deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
555
+ await deps.sleep(wait);
556
+ }
548
557
  }
549
- }
550
- // Prime the supervisor mail monitor's notification cursor at the
551
- // stream tip BEFORE the session launches so no arrival is missed during boot
552
- // (backlog before the tip is the SessionStart hook's job). Native-mode roles
553
- // leave wake ownership to the harness. Temp snapshots predating `monitor:` are
554
- // treated as native (monitor may be undefined on an old role.yaml).
555
- let sessionHandle;
556
- let modelRecovery;
557
- const resolvedMonitorDeps = monitorDeps(deps, role.env);
558
- resolvedMonitorDeps.onFailureEvidence = evidence => {
559
- if (!role.model_chain)
558
+ // Prime the supervisor mail monitor's notification cursor at the
559
+ // stream tip BEFORE the session launches so no arrival is missed during boot
560
+ // (backlog before the tip is the SessionStart hook's job). Native-mode roles
561
+ // leave wake ownership to the harness. Temp snapshots predating `monitor:` are
562
+ // treated as native (monitor may be undefined on an old role.yaml).
563
+ let sessionHandle;
564
+ let modelRecovery;
565
+ const resolvedMonitorDeps = monitorDeps(deps, role.env);
566
+ resolvedMonitorDeps.onFailureEvidence = evidence => {
567
+ if (!role.model_chain)
568
+ return false;
569
+ const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
570
+ if (action.kind === 'advance') {
571
+ modelRecovery = 'advance';
572
+ deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
573
+ void sessionHandle?.close();
574
+ return true;
575
+ }
576
+ else if (action.kind === 'hold') {
577
+ modelRecovery = 'hold';
578
+ deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
579
+ void sessionHandle?.close();
580
+ return true;
581
+ }
560
582
  return false;
561
- const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
562
- if (action.kind === 'advance') {
563
- modelRecovery = 'advance';
564
- deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
565
- void sessionHandle?.close();
566
- return true;
567
- }
568
- else if (action.kind === 'hold') {
569
- modelRecovery = 'hold';
570
- deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
571
- void sessionHandle?.close();
572
- return true;
573
- }
574
- return false;
575
- };
576
- const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
577
- const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
578
- const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
579
- name, identity: role.identity, agentDir: dir, cfg: role.monitor,
580
- deps: resolvedMonitorDeps,
581
- }) : null;
582
- const daemonObserver = new DaemonGenerationObserver();
583
- const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
584
- let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
585
- if (monitor)
586
- await monitor.prime({ resetCursor: resetMonitorCursor });
587
- rmSync(exitFile, { force: true });
588
- let pid;
589
- let agentSession;
590
- let control;
591
- let unsubscribeRecovery;
592
- let monitorLoop;
593
- let sessionStartupComplete = false;
594
- let sessionClosed = false;
595
- let ownerChannel;
596
- let ownerBinder;
597
- let loopManager;
598
- let arbiter;
599
- let recoveryController;
600
- let reloadLoopConfig;
601
- let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
602
- loop.name, loop.definitionHash, loop.promptHash,
603
- ]));
604
- {
605
- const perms = role.permissions ?? resolvePermissions(undefined, undefined);
606
- // Say once, at startup, that this role will decide permission requests by
607
- // itself. Without it the only trace of an auto-denied tool call is a turn
608
- // that quietly did less than it was asked to.
609
- if (perms.unattended === 'deny')
610
- deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
611
- `permission requests are automatically denied once each (reject_once) and the turn continues`);
612
- agentSession = await deps.startAgentSession(adapter.agentSession, {
613
- role, prep,
614
- launch: { ...launch, argv: wrappedArgv, env: harnessChildEnv(role, launch.env, dir) },
615
- cwd: runCwd, stateDir: dir, mode, permissions: perms,
616
- permissionMode: effectivePermissionMode(role), log: deps.log,
617
- });
618
- pid = agentSession.pid;
619
- arbiter = new RoleTurnArbiter(agentSession);
620
- sessionHandle = arbiter;
621
- unsubscribeRecovery = agentSession.subscribe(event => {
622
- if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
623
- return;
624
- const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
625
- if (evidence)
626
- resolvedMonitorDeps.onFailureEvidence?.(evidence);
627
- });
628
- if (role.owner_channel) {
583
+ };
584
+ const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
585
+ const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
586
+ const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
587
+ name, identity: role.identity, agentDir: dir, cfg: role.monitor,
588
+ deps: resolvedMonitorDeps,
589
+ }) : null;
590
+ const daemonObserver = new DaemonGenerationObserver();
591
+ const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
592
+ let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
593
+ if (monitor)
594
+ await monitor.prime({ resetCursor: resetMonitorCursor });
595
+ rmSync(exitFile, { force: true });
596
+ let pid;
597
+ let agentSession;
598
+ let control;
599
+ let unsubscribeRecovery;
600
+ let monitorLoop;
601
+ let sessionStartupComplete = false;
602
+ let sessionClosed = false;
603
+ let ownerChannel;
604
+ let ownerBinder;
605
+ let loopManager;
606
+ let arbiter;
607
+ let recoveryController;
608
+ let reloadLoopConfig;
609
+ let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
610
+ loop.name, loop.definitionHash, loop.promptHash,
611
+ ]));
612
+ {
613
+ const perms = role.permissions ?? resolvePermissions(undefined, undefined);
614
+ // Say once, at startup, that this role will decide permission requests by
615
+ // itself. Without it the only trace of an auto-denied tool call is a turn
616
+ // that quietly did less than it was asked to.
617
+ if (perms.unattended === 'deny')
618
+ deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
619
+ `permission requests are automatically denied once each (reject_once) and the turn continues`);
620
+ agentSession = await managedService.runtime.startHarness(() => deps.startAgentSession(adapter.agentSession, {
621
+ managedOurs: managedHarness.ours,
622
+ role, prep,
623
+ launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
624
+ cwd: runCwd, stateDir: dir, mode, permissions: perms,
625
+ permissionMode: effectivePermissionMode(role), log: deps.log,
626
+ }));
627
+ pid = agentSession.pid;
628
+ arbiter = new RoleTurnArbiter(agentSession);
629
+ sessionHandle = arbiter;
630
+ unsubscribeRecovery = agentSession.subscribe(event => {
631
+ if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
632
+ return;
633
+ const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
634
+ if (evidence)
635
+ resolvedMonitorDeps.onFailureEvidence?.(evidence);
636
+ });
637
+ if (role.owner_channel) {
638
+ try {
639
+ ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
640
+ }
641
+ catch (error) {
642
+ if (error instanceof OwnerBinderHandoffTimeoutError) {
643
+ try {
644
+ const status = await deps.reportOwnerStartupFailure(dir);
645
+ deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
646
+ }
647
+ catch (notifyError) {
648
+ deps.log(`[${name}] owner channel startup recovery notice unavailable: `
649
+ + `${notifyError?.message ?? String(notifyError)}`);
650
+ }
651
+ }
652
+ await agentSession.close();
653
+ unsubscribeRecovery?.();
654
+ throw new Error(`[${name}] owner channel failed to start: `
655
+ + `${error?.message ?? String(error)}`);
656
+ }
657
+ }
629
658
  try {
630
- ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
659
+ if (role.owner_channel)
660
+ ownerChannel = deps.createOwnerChannel({
661
+ role: name,
662
+ harness: role.harness,
663
+ config: role.owner_channel,
664
+ session: arbiter,
665
+ stateDir: dir,
666
+ env: role.env,
667
+ log: deps.log,
668
+ ...(ownerBinder ? { binderLease: ownerBinder } : {}),
669
+ ...(configPath ? { configPath } : {}),
670
+ });
671
+ control = deps.createControlServer(dir, arbiter, deps.log);
672
+ control.setFleetAuditor(ownerChannel ? {
673
+ begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
674
+ finish: input => ownerChannel.finishFleetCommandAudit(input),
675
+ present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
676
+ } : localFleetAuditor(dir, name, deps.log));
677
+ await control.start();
631
678
  }
632
679
  catch (error) {
633
- if (error instanceof OwnerBinderHandoffTimeoutError) {
634
- try {
635
- const status = await deps.reportOwnerStartupFailure(dir);
636
- deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
637
- }
638
- catch (notifyError) {
639
- deps.log(`[${name}] owner channel startup recovery notice unavailable: `
640
- + `${notifyError?.message ?? String(notifyError)}`);
641
- }
642
- }
680
+ await ownerChannel?.close().catch(() => undefined);
681
+ ownerBinder?.release();
643
682
  await agentSession.close();
644
683
  unsubscribeRecovery?.();
645
- throw new Error(`[${name}] owner channel failed to start: `
646
- + `${error?.message ?? String(error)}`);
684
+ throw error;
647
685
  }
648
- }
649
- try {
650
- if (role.owner_channel)
651
- ownerChannel = deps.createOwnerChannel({
652
- role: name,
653
- harness: role.harness,
654
- config: role.owner_channel,
655
- session: arbiter,
656
- stateDir: dir,
657
- env: role.env,
658
- log: deps.log,
659
- ...(ownerBinder ? { binderLease: ownerBinder } : {}),
660
- ...(configPath ? { configPath } : {}),
661
- });
662
- control = deps.createControlServer(dir, arbiter, deps.log);
663
- control.setFleetAuditor(ownerChannel ? {
664
- begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
665
- finish: input => ownerChannel.finishFleetCommandAudit(input),
666
- present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
667
- } : localFleetAuditor(dir, name, deps.log));
668
- await control.start();
669
- }
670
- catch (error) {
671
- await ownerChannel?.close().catch(() => undefined);
672
- ownerBinder?.release();
673
- await agentSession.close();
674
- unsubscribeRecovery?.();
675
- throw error;
676
- }
677
- if (!role.roomMemberStartup)
678
- control.setFleetSpawner(async (requested) => {
679
- const event = await executeManagedSpawn(role, configPath, requested, deps.log);
680
- // Room-member launches are collected with their Task/Room transaction so
681
- // causal delivery is Task → Room → Agents → Room active → Task active.
682
- if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
683
- try {
684
- await ownerChannel.notifyFleetSpawn(event);
685
- }
686
- catch (error) {
687
- deps.log(`[${name}] Agent lifecycle notice delivery failed: `
688
- + `${error?.message ?? String(error)}`);
686
+ if (!role.roomMemberStartup)
687
+ control.setFleetSpawner(async (requested) => {
688
+ const event = await executeManagedSpawn(role, configPath, requested, deps.log);
689
+ // Room-member launches are collected with their Task/Room transaction so
690
+ // causal delivery is Task → Room → Agents → Room active → Task active.
691
+ if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
692
+ try {
693
+ await ownerChannel.notifyFleetSpawn(event);
694
+ }
695
+ catch (error) {
696
+ deps.log(`[${name}] Agent lifecycle notice delivery failed: `
697
+ + `${error?.message ?? String(error)}`);
698
+ }
689
699
  }
690
- }
691
- return event;
692
- });
693
- resolvedMonitorDeps.delivery = {
694
- // A wake is only delivered when its turn TERMINATES successfully. A
695
- // refusal or a cancellation reached the agent and was not acted on, so
696
- // the monitor must keep its cursor and try again.
697
- submit: async (text, options) => {
698
- // Cancelling the runner-owned startup prompt makes startup look failed
699
- // and closes the session before the wake turn can run. During startup,
700
- // steer into the live turn instead; after it completes, honor the
701
- // configured interrupt policy normally.
702
- const policy = options?.interrupt;
703
- const interrupt = policy === true && sessionStartupComplete;
704
- const promptOptions = {
705
- interrupt, steer: true,
706
- ...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
707
- origin: { kind: 'fleet-monitor' },
708
- };
709
- // Startup is already a protected boundary: as with immediate mode,
710
- // steer rather than waiting on/cancelling the runner-owned first turn.
711
- const result = policy === 'after_tool' && sessionStartupComplete
712
- ? await arbiter.submitPromptAfterTool(text, promptOptions)
713
- : await arbiter.submitPrompt(text, promptOptions);
714
- const steered = result.accepted
715
- && (result.detail === 'injected' || result.detail === 'startedNewTurn');
716
- const boundary = result.safeBoundary;
717
- const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
718
- const boundaryDetail = boundary && queuedAfterSteeringFailure
719
- ? boundary.state === 'timeout'
720
- ? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
721
- : `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
722
- : boundary
700
+ return event;
701
+ });
702
+ resolvedMonitorDeps.delivery = {
703
+ // A wake is only delivered when its turn TERMINATES successfully. A
704
+ // refusal or a cancellation reached the agent and was not acted on, so
705
+ // the monitor must keep its cursor and try again.
706
+ submit: async (text, options) => {
707
+ // Cancelling the runner-owned startup prompt makes startup look failed
708
+ // and closes the session before the wake turn can run. During startup,
709
+ // steer into the live turn instead; after it completes, honor the
710
+ // configured interrupt policy normally.
711
+ const policy = options?.interrupt;
712
+ const interrupt = policy === true && sessionStartupComplete;
713
+ const promptOptions = {
714
+ interrupt, steer: true,
715
+ ...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
716
+ origin: { kind: 'fleet-monitor' },
717
+ };
718
+ // Startup is already a protected boundary: as with immediate mode,
719
+ // steer rather than waiting on/cancelling the runner-owned first turn.
720
+ const result = policy === 'after_tool' && sessionStartupComplete
721
+ ? await arbiter.submitPromptAfterTool(text, promptOptions)
722
+ : await arbiter.submitPrompt(text, promptOptions);
723
+ const steered = result.accepted
724
+ && (result.detail === 'injected' || result.detail === 'startedNewTurn');
725
+ const boundary = result.safeBoundary;
726
+ const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
727
+ const boundaryDetail = boundary && queuedAfterSteeringFailure
723
728
  ? boundary.state === 'timeout'
724
- ? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
725
- : boundary.state === 'unsupported'
726
- ? 'after_tool unsupported; used non-cancelling queued delivery'
727
- : `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
728
- : result.detail;
729
- return {
730
- succeeded: result.succeeded || steered,
731
- outcome: steered ? result.detail : result.outcome,
732
- detail: result.succeeded || steered || !boundary
733
- ? boundaryDetail
734
- : [result.detail, boundaryDetail].filter(Boolean).join('; '),
735
- ...(boundary ? { safeBoundary: boundary.state } : {}),
736
- };
737
- },
738
- };
739
- const firstPrompt = mode === 'fresh'
740
- ? `Read and follow ${join(dir, 'briefing.md')} now.`
741
- : adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
742
- // Wait for the first turn's TERMINAL result. An agent that accepts the
743
- // startup prompt and then refuses it has not started; logging the role as
744
- // up would hide a role that never read its briefing.
745
- const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
746
- // Monitoring starts immediately. The delivery adapter above downgrades
747
- // interruption to steering until this startup turn reaches a terminal
748
- // success, so there is neither a deaf gap nor a boot-cancellation loop.
749
- monitorLoop = monitor?.run(pid);
750
- const started = await starting;
751
- // A temporary role's first turn can be the active turn when an ours wake
752
- // needs immediate attention. A typed console/monitor cancellation ends
753
- // only that turn: the already-live agent session and any queued wake remain
754
- // valid. Keep every unproven cancellation, refusal, shutdown, and genuine
755
- // failure terminal so a role that never accepted its briefing is not
756
- // silently reported as healthy.
757
- const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
758
- || (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
759
- if (!started.succeeded && !interruptedForWake) {
760
- monitor?.stop();
761
- await control.close();
762
- ownerBinder?.release();
763
- await agentSession.close();
764
- unsubscribeRecovery?.();
765
- if (modelRecovery) {
766
- if (monitorLoop)
767
- await monitorLoop;
768
- return {
769
- elapsedSecs: 0,
770
- exit: {
771
- version: 1,
772
- class: 'program-exit',
773
- detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
774
- },
775
- rotated: false,
776
- mode,
777
- modelRecovery,
778
- };
779
- }
780
- throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
781
- `${started.detail ? `: ${started.detail}` : ''}`);
782
- }
783
- if (started.cancellationSource === 'stall-watchdog')
784
- deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
785
- else if (interruptedForWake)
786
- deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
787
- + 'keeping temporary supervisor alive');
788
- sessionStartupComplete = true;
789
- if (ownerChannel) {
790
- try {
791
- await ownerChannel.start();
792
- }
793
- catch (error) {
729
+ ? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
730
+ : `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
731
+ : boundary
732
+ ? boundary.state === 'timeout'
733
+ ? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
734
+ : boundary.state === 'unsupported'
735
+ ? 'after_tool unsupported; used non-cancelling queued delivery'
736
+ : `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
737
+ : result.detail;
738
+ return {
739
+ succeeded: result.succeeded || steered,
740
+ outcome: steered ? result.detail : result.outcome,
741
+ detail: result.succeeded || steered || !boundary
742
+ ? boundaryDetail
743
+ : [result.detail, boundaryDetail].filter(Boolean).join('; '),
744
+ ...(boundary ? { safeBoundary: boundary.state } : {}),
745
+ };
746
+ },
747
+ };
748
+ const firstPrompt = mode === 'fresh'
749
+ ? `Read and follow ${join(dir, 'briefing.md')} now.`
750
+ : `Your supervisor has verified your assigned identity and room readiness. Read ${join(dir, 'WORKLOG.md')} and ${join(dir, 'briefing.md')}, then continue using the available ours tools.`;
751
+ // Wait for the first turn's TERMINAL result. An agent that accepts the
752
+ // startup prompt and then refuses it has not started; logging the role as
753
+ // up would hide a role that never read its briefing.
754
+ const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
755
+ // Monitoring starts immediately. The delivery adapter above downgrades
756
+ // interruption to steering until this startup turn reaches a terminal
757
+ // success, so there is neither a deaf gap nor a boot-cancellation loop.
758
+ monitorLoop = monitor?.run(pid);
759
+ const started = await starting;
760
+ // A temporary role's first turn can be the active turn when an ours wake
761
+ // needs immediate attention. A typed console/monitor cancellation ends
762
+ // only that turn: the already-live agent session and any queued wake remain
763
+ // valid. Keep every unproven cancellation, refusal, shutdown, and genuine
764
+ // failure terminal so a role that never accepted its briefing is not
765
+ // silently reported as healthy.
766
+ const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
767
+ || (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
768
+ if (!started.succeeded && !interruptedForWake) {
794
769
  monitor?.stop();
795
- if (monitorLoop)
796
- await monitorLoop;
797
- await ownerChannel.close().catch(() => undefined);
798
770
  await control.close();
799
771
  ownerBinder?.release();
800
772
  await agentSession.close();
801
773
  unsubscribeRecovery?.();
802
- throw new Error(`[${name}] owner channel failed to start: `
803
- + `${error?.message ?? String(error)}`);
804
- }
805
- }
806
- if (ownerChannel) {
807
- control.setOwnerChannel(ownerChannel);
808
- }
809
- reloadLoopConfig = async () => {
810
- const nextRole = findRole(loadConfig(configPath), name);
811
- const definitions = nextRole.loops ?? [];
812
- const generation = JSON.stringify(definitions.map(loop => [
813
- loop.name, loop.definitionHash, loop.promptHash,
814
- ]));
815
- if (generation === loopGeneration && (loopManager || !definitions.length))
816
- return { changed: false, loops: definitions.length };
817
- if (!loopManager && definitions.length) {
818
- loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
819
- now: deps.now,
820
- setTimer: (callback, ms) => setTimeout(callback, ms),
821
- clearTimer: timer => clearTimeout(timer),
822
- log: deps.log,
823
- });
824
- control.setLoopManager(loopManager);
825
- loopManager.start();
826
- }
827
- else {
828
- loopManager?.reconcile(definitions);
829
- }
830
- loopGeneration = generation;
831
- deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
832
- return { changed: true, loops: definitions.length };
833
- };
834
- // Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
835
- if (!temp)
836
- control.setConfigReloader(reloadLoopConfig);
837
- if (role.loops?.length) {
838
- try {
839
- loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
840
- now: deps.now,
841
- setTimer: (callback, ms) => setTimeout(callback, ms),
842
- clearTimer: timer => clearTimeout(timer),
843
- log: deps.log,
844
- });
845
- control.setLoopManager(loopManager);
846
- loopManager.start();
774
+ if (modelRecovery) {
775
+ if (monitorLoop)
776
+ await monitorLoop;
777
+ return {
778
+ elapsedSecs: 0,
779
+ exit: {
780
+ version: 1,
781
+ class: 'program-exit',
782
+ detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
783
+ },
784
+ rotated: false,
785
+ mode,
786
+ modelRecovery,
787
+ };
788
+ }
789
+ throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
790
+ `${started.detail ? `: ${started.detail}` : ''}`);
847
791
  }
848
- catch (error) {
849
- if (temp) {
792
+ if (started.cancellationSource === 'stall-watchdog')
793
+ deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
794
+ else if (interruptedForWake)
795
+ deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
796
+ + 'keeping temporary supervisor alive');
797
+ sessionStartupComplete = true;
798
+ if (ownerChannel) {
799
+ try {
800
+ await ownerChannel.start();
801
+ }
802
+ catch (error) {
850
803
  monitor?.stop();
851
804
  if (monitorLoop)
852
805
  await monitorLoop;
806
+ await ownerChannel.close().catch(() => undefined);
853
807
  await control.close();
854
808
  ownerBinder?.release();
855
809
  await agentSession.close();
856
810
  unsubscribeRecovery?.();
857
- throw new Error(`[${name}] configured temporary loop manager failed to start: `
811
+ throw new Error(`[${name}] owner channel failed to start: `
858
812
  + `${error?.message ?? String(error)}`);
859
813
  }
860
- deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
861
814
  }
862
- }
863
- }
864
- deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
865
- // The monitor loop lives exactly as long as the session: it starts once the
866
- // pane pid is known and is stopped when that pid dies (task dies with runner).
867
- monitorLoop ??= monitor?.run(pid);
868
- recoveryController = new RoleRecoveryController({
869
- role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
870
- log: deps.log,
871
- recoverAgent: async () => {
872
- const evidence = await recoverAgentIdentity(arbiter, role.identity);
873
- return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
874
- },
875
- recoverOwner: async (epoch) => {
876
- if (!ownerChannel?.recover)
877
- return { ok: true };
878
- try {
879
- await ownerChannel.recover(epoch);
880
- return { ok: true };
881
- }
882
- catch (error) {
883
- return { ok: false, reason: error instanceof Error
884
- ? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
815
+ if (ownerChannel) {
816
+ control.setOwnerChannel(ownerChannel);
885
817
  }
886
- },
887
- });
888
- const start = deps.now();
889
- let nextLoopReloadAt = deps.now() + 30_000;
890
- let nextIdentityPollAt = deps.now();
891
- let nextDaemonProbeAt = deps.now();
892
- let lastReloadError = '';
893
- let identityObserved = false;
894
- let identityAbsentSince;
895
- let retirementReason;
896
- let supervisorRecycleRequired = false;
897
- while (sessionHandle.isAlive()) {
898
- await deps.sleep(temp ? 500 : 2000);
899
- const now = deps.now();
900
- if (now >= nextDaemonProbeAt) {
901
- nextDaemonProbeAt = now + 2_000;
902
- const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
903
- if (observation.kind === 'lost')
904
- recoveryController.noteLoss(observation.reason);
905
- if (observation.kind === 'changed' || observation.kind === 'available'
906
- || (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
907
- sessionStartedWithoutDaemonBaseline = false;
908
- void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
909
- }
910
- }
911
- if (deps.shouldStop?.()) {
912
- if (temp) {
913
- retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
914
- deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
818
+ reloadLoopConfig = async () => {
819
+ const nextRole = findRole(loadConfig(configPath), name);
820
+ const definitions = nextRole.loops ?? [];
821
+ const generation = JSON.stringify(definitions.map(loop => [
822
+ loop.name, loop.definitionHash, loop.promptHash,
823
+ ]));
824
+ if (generation === loopGeneration && (loopManager || !definitions.length))
825
+ return { changed: false, loops: definitions.length };
826
+ if (!loopManager && definitions.length) {
827
+ loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
828
+ now: deps.now,
829
+ setTimer: (callback, ms) => setTimeout(callback, ms),
830
+ clearTimer: timer => clearTimeout(timer),
831
+ log: deps.log,
832
+ });
833
+ control.setLoopManager(loopManager);
834
+ loopManager.start();
835
+ }
836
+ else {
837
+ loopManager?.reconcile(definitions);
838
+ }
839
+ loopGeneration = generation;
840
+ deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
841
+ return { changed: true, loops: definitions.length };
842
+ };
843
+ // Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
844
+ if (!temp)
845
+ control.setConfigReloader(reloadLoopConfig);
846
+ if (role.loops?.length) {
847
+ try {
848
+ loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
849
+ now: deps.now,
850
+ setTimer: (callback, ms) => setTimeout(callback, ms),
851
+ clearTimer: timer => clearTimeout(timer),
852
+ log: deps.log,
853
+ });
854
+ control.setLoopManager(loopManager);
855
+ loopManager.start();
856
+ }
857
+ catch (error) {
858
+ if (temp) {
859
+ monitor?.stop();
860
+ if (monitorLoop)
861
+ await monitorLoop;
862
+ await control.close();
863
+ ownerBinder?.release();
864
+ await agentSession.close();
865
+ unsubscribeRecovery?.();
866
+ throw new Error(`[${name}] configured temporary loop manager failed to start: `
867
+ + `${error?.message ?? String(error)}`);
868
+ }
869
+ deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
870
+ }
915
871
  }
916
- await sessionHandle.close();
917
- sessionClosed = true;
918
- break;
919
872
  }
920
- if (temp && now >= nextIdentityPollAt) {
921
- nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
922
- const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
923
- if (presence.state === 'present') {
924
- identityObserved = true;
925
- identityAbsentSince = undefined;
873
+ deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
874
+ // The monitor loop lives exactly as long as the session: it starts once the
875
+ // pane pid is known and is stopped when that pid dies (task dies with runner).
876
+ monitorLoop ??= monitor?.run(pid);
877
+ recoveryController = new RoleRecoveryController({
878
+ role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
879
+ log: deps.log,
880
+ recoverAgent: async () => {
881
+ try {
882
+ const release = await managedService.runtime.admit();
883
+ release();
884
+ return { ok: true };
885
+ }
886
+ catch {
887
+ return { ok: false, reason: 'SUPERVISOR_IDENTITY_NOT_READY' };
888
+ }
889
+ },
890
+ recoverOwner: async (epoch) => {
891
+ if (!ownerChannel?.recover)
892
+ return { ok: true };
893
+ try {
894
+ await ownerChannel.recover(epoch);
895
+ return { ok: true };
896
+ }
897
+ catch (error) {
898
+ return { ok: false, reason: error instanceof Error
899
+ ? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
900
+ }
901
+ },
902
+ });
903
+ const start = deps.now();
904
+ let nextLoopReloadAt = deps.now() + 30_000;
905
+ let nextIdentityPollAt = deps.now();
906
+ let nextDaemonProbeAt = deps.now();
907
+ let lastReloadError = '';
908
+ let identityObserved = false;
909
+ let identityAbsentSince;
910
+ let retirementReason;
911
+ let supervisorRecycleRequired = false;
912
+ while (sessionHandle.isAlive()) {
913
+ await deps.sleep(temp ? 500 : 2000);
914
+ const now = deps.now();
915
+ if (now >= nextDaemonProbeAt) {
916
+ nextDaemonProbeAt = now + 2_000;
917
+ const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
918
+ if (observation.kind === 'lost')
919
+ recoveryController.noteLoss(observation.reason);
920
+ if (observation.kind === 'changed' || observation.kind === 'available'
921
+ || (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
922
+ sessionStartedWithoutDaemonBaseline = false;
923
+ void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
924
+ }
926
925
  }
927
- else if (presence.state === 'absent' && identityObserved) {
928
- identityAbsentSince ??= now;
929
- // Require a continuous, time-bounded run of authoritative absence.
930
- // The first positive observation is the readiness gate: cold harness
931
- // starts may spend minutes loading the harness and briefing before the
932
- // agent creates/binds its identity, and absence before then is not a
933
- // close event. After readiness, debounce a real disappearance.
934
- if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
935
- retirementReason = 'identity-closed';
936
- deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
937
- await sessionHandle.close();
938
- sessionClosed = true;
939
- break;
926
+ if (deps.shouldStop?.()) {
927
+ if (temp) {
928
+ retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
929
+ deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
940
930
  }
931
+ await sessionHandle.close();
932
+ sessionClosed = true;
933
+ break;
941
934
  }
942
- else if (presence.state === 'unknown')
943
- identityAbsentSince = undefined;
944
- }
945
- if (reloadLoopConfig && now >= nextLoopReloadAt) {
946
- nextLoopReloadAt = now + 30_000;
947
- try {
948
- await reloadLoopConfig();
949
- lastReloadError = '';
935
+ if (temp && now >= nextIdentityPollAt) {
936
+ nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
937
+ const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
938
+ if (presence.state === 'present') {
939
+ identityObserved = true;
940
+ identityAbsentSince = undefined;
941
+ }
942
+ else if (presence.state === 'absent' && identityObserved) {
943
+ identityAbsentSince ??= now;
944
+ // Require a continuous, time-bounded run of authoritative absence.
945
+ // The first positive observation is the readiness gate: cold harness
946
+ // starts may spend minutes loading the harness and briefing before the
947
+ // agent creates/binds its identity, and absence before then is not a
948
+ // close event. After readiness, debounce a real disappearance.
949
+ if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
950
+ retirementReason = 'identity-closed';
951
+ deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
952
+ await sessionHandle.close();
953
+ sessionClosed = true;
954
+ break;
955
+ }
956
+ }
957
+ else if (presence.state === 'unknown')
958
+ identityAbsentSince = undefined;
950
959
  }
951
- catch (error) {
952
- const message = error?.message ?? String(error);
953
- if (message !== lastReloadError)
954
- deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
955
- lastReloadError = message;
960
+ if (reloadLoopConfig && now >= nextLoopReloadAt) {
961
+ nextLoopReloadAt = now + 30_000;
962
+ try {
963
+ await reloadLoopConfig();
964
+ lastReloadError = '';
965
+ }
966
+ catch (error) {
967
+ const message = error?.message ?? String(error);
968
+ if (message !== lastReloadError)
969
+ deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
970
+ lastReloadError = message;
971
+ }
956
972
  }
957
973
  }
958
- }
959
- if (loopManager) {
960
- control?.setLoopManager(undefined);
961
- await loopManager.stop();
962
- }
963
- // Let already-settled path operations publish their aggregate result before
964
- // the shutdown fence invalidates the epoch. This never waits on I/O.
965
- await Promise.resolve();
966
- await Promise.resolve();
967
- recoveryController?.cancel();
968
- control?.setConfigReloader(undefined);
969
- // Close the authenticated control route before releasing the binder lease;
970
- // otherwise the predecessor can unlink the replacement's new socket.
971
- if (control) {
972
- control.setOwnerChannel(undefined);
973
- await control.close();
974
- control = undefined;
975
- }
976
- if (ownerChannel)
977
- await ownerChannel.close();
978
- if (ownerChannel?.binderReleaseSafe?.() === false) {
979
- deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
980
- supervisorRecycleRequired = true;
981
- }
982
- else
983
- ownerBinder?.release();
984
- if (monitor) {
985
- monitor.stop();
986
- await monitorLoop;
987
- }
988
- unsubscribeRecovery?.();
989
- if (agentSession && !sessionClosed)
990
- await agentSession.close();
991
- const elapsed = (deps.now() - start) / 1000;
992
- // Establish what actually happened before deciding anything. Absence of a
993
- // record is `unknown` — except when the console itself is gone, which is a
994
- // different event with a different consequence.
995
- const exitRecord = agentSession.exitResult()
996
- ?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
997
- writeFileSync(exitFile, JSON.stringify({
998
- ...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
999
- }) + '\n');
1000
- let rotated = false;
1001
- const rotate = (why) => {
1002
- writeFileSync(sidFile, randomUUID() + '\n');
1003
- rmSync(bootedFile, { force: true });
1004
- rotated = true;
1005
- deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
1006
- };
1007
- if (!temp && deps.shouldStop?.())
1008
- deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
1009
- else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
1010
- || exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
1011
- // This is a deliberate adapter reclamation, not evidence that resume state
1012
- // is poisoned. Preserve the context even when the resumed generation hits
1013
- // the same bound immediately; runSupervised still counts the fast exit and
1014
- // opens its circuit after the configured number of consecutive failures.
1015
- deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1016
- `-> next start RESUMES context`);
1017
- else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
1018
- rotate(`clean exit (code 0)`);
1019
- else if (exitRecord.class === 'session-destroyed')
1020
- // Someone tore the console down; the agent did not fail. Rotating here
1021
- // would discard a live conversation for an operator action.
1022
- deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1023
- else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
1024
- // Self-heal a poisoned resume — but only once per failure sequence. Rotating
1025
- // on every attempt would discard the conversation again and again while the
1026
- // real cause (a broken command, a missing binary) went unaddressed.
1027
- if (opts.allowResumeRotation === false)
1028
- deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1029
- `-> resume state was already discarded once; keeping it`);
974
+ if (loopManager) {
975
+ control?.setLoopManager(undefined);
976
+ await loopManager.stop();
977
+ }
978
+ // Let already-settled path operations publish their aggregate result before
979
+ // the shutdown fence invalidates the epoch. This never waits on I/O.
980
+ await Promise.resolve();
981
+ await Promise.resolve();
982
+ recoveryController?.cancel();
983
+ control?.setConfigReloader(undefined);
984
+ // Close the authenticated control route before releasing the binder lease;
985
+ // otherwise the predecessor can unlink the replacement's new socket.
986
+ if (control) {
987
+ control.setOwnerChannel(undefined);
988
+ await control.close();
989
+ control = undefined;
990
+ }
991
+ if (ownerChannel)
992
+ await ownerChannel.close();
993
+ if (ownerChannel?.binderReleaseSafe?.() === false) {
994
+ deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
995
+ supervisorRecycleRequired = true;
996
+ }
1030
997
  else
1031
- rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
998
+ ownerBinder?.release();
999
+ if (monitor) {
1000
+ monitor.stop();
1001
+ await monitorLoop;
1002
+ }
1003
+ unsubscribeRecovery?.();
1004
+ if (agentSession && !sessionClosed)
1005
+ await agentSession.close();
1006
+ const elapsed = (deps.now() - start) / 1000;
1007
+ // Establish what actually happened before deciding anything. Absence of a
1008
+ // record is `unknown` — except when the console itself is gone, which is a
1009
+ // different event with a different consequence.
1010
+ const exitRecord = agentSession.exitResult()
1011
+ ?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
1012
+ writeFileSync(exitFile, JSON.stringify({
1013
+ ...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
1014
+ }) + '\n');
1015
+ let rotated = false;
1016
+ const rotate = (why) => {
1017
+ writeFileSync(sidFile, randomUUID() + '\n');
1018
+ rmSync(bootedFile, { force: true });
1019
+ rotated = true;
1020
+ deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
1021
+ };
1022
+ if (!temp && deps.shouldStop?.())
1023
+ deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
1024
+ else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
1025
+ || exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
1026
+ // This is a deliberate adapter reclamation, not evidence that resume state
1027
+ // is poisoned. Preserve the context even when the resumed generation hits
1028
+ // the same bound immediately; runSupervised still counts the fast exit and
1029
+ // opens its circuit after the configured number of consecutive failures.
1030
+ deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1031
+ `-> next start RESUMES context`);
1032
+ else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
1033
+ rotate(`clean exit (code 0)`);
1034
+ else if (exitRecord.class === 'session-destroyed')
1035
+ // Someone tore the console down; the agent did not fail. Rotating here
1036
+ // would discard a live conversation for an operator action.
1037
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1038
+ else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
1039
+ // Self-heal a poisoned resume — but only once per failure sequence. Rotating
1040
+ // on every attempt would discard the conversation again and again while the
1041
+ // real cause (a broken command, a missing binary) went unaddressed.
1042
+ if (opts.allowResumeRotation === false)
1043
+ deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1044
+ `-> resume state was already discarded once; keeping it`);
1045
+ else
1046
+ rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
1047
+ }
1048
+ else
1049
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1050
+ if (supervisorRecycleRequired)
1051
+ throw new SupervisorRecycleRequiredError();
1052
+ return {
1053
+ elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
1054
+ ...(retirementReason ? { retirementReason } : {}),
1055
+ };
1056
+ }
1057
+ finally {
1058
+ await managedService.close(false);
1032
1059
  }
1033
- else
1034
- deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1035
- if (supervisorRecycleRequired)
1036
- throw new SupervisorRecycleRequiredError();
1037
- return {
1038
- elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
1039
- ...(retirementReason ? { retirementReason } : {}),
1040
- };
1041
1060
  }
1042
1061
  /**
1043
1062
  * The persistent supervisor for one permanent role: run child sessions in a
@@ -1205,6 +1224,8 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
1205
1224
  process.off('SIGINT', requestStop);
1206
1225
  // An orderly shutdown clears the marker; an unhandled signal or OOM-kill
1207
1226
  // leaves it so the successor can identify an abrupt termination.
1227
+ if (shouldStop())
1228
+ await deps.releaseAgentOurs(findRole(loadConfig(opts.configPath), name));
1208
1229
  releaseSupervisorRun(dir);
1209
1230
  }
1210
1231
  }
@@ -1234,10 +1255,15 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
1234
1255
  let result;
1235
1256
  let failure;
1236
1257
  try {
1237
- result = await attempt(name, { temp: true }, {
1238
- ...deps,
1239
- shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
1240
- });
1258
+ for (let recoveryAttempt = 0;; recoveryAttempt++) {
1259
+ result = await attempt(name, { temp: true }, {
1260
+ ...deps,
1261
+ shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
1262
+ });
1263
+ if (result.exit.class === 'clean' || result.retirementReason || signal || deps.shouldStop?.() || recoveryAttempt >= 2)
1264
+ break;
1265
+ await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(Math.min(5000, 1000 * (recoveryAttempt + 1)));
1266
+ }
1241
1267
  }
1242
1268
  catch (error) {
1243
1269
  failure = error;
@@ -1245,23 +1271,27 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
1245
1271
  finally {
1246
1272
  process.off('SIGTERM', onTerm);
1247
1273
  process.off('SIGINT', onInt);
1248
- const requested = requestedTempStopReason(dir);
1249
- const reason = requested
1250
- ?? result?.retirementReason
1251
- ?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
1252
- ?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
1253
- // A service-manager stop can make the child connection close before the
1254
- // runner reaches its normal loop. That is still a successful requested
1255
- // retirement, not a startup failure.
1256
- const outcome = failure && !requested && !signal ? 'failed' : 'retired';
1257
- const detail = failure
1258
- ? (failure instanceof Error ? failure.message : String(failure))
1259
- : result
1260
- ? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
1261
- : 'temporary supervisor ended without an attempt result';
1262
- const archived = archiveTempState(name, reason, outcome, detail);
1263
- deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
1264
- + `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
1274
+ // A recycle replaces the supervisor process, not the logical temporary agent.
1275
+ if (!(failure instanceof SupervisorRecycleRequiredError) || signal || deps.shouldStop?.()) {
1276
+ const requested = requestedTempStopReason(dir);
1277
+ const reason = requested
1278
+ ?? result?.retirementReason
1279
+ ?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
1280
+ ?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
1281
+ // A service-manager stop can make the child connection close before the
1282
+ // runner reaches its normal loop. That is still a successful requested
1283
+ // retirement, not a startup failure.
1284
+ const outcome = failure && !requested && !signal ? 'failed' : 'retired';
1285
+ const detail = failure
1286
+ ? (failure instanceof Error ? failure.message : String(failure))
1287
+ : result
1288
+ ? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
1289
+ : 'temporary supervisor ended without an attempt result';
1290
+ await (deps.releaseAgentOurs ?? releaseManagedAgent)(loadTempRole(name));
1291
+ const archived = archiveTempState(name, reason, outcome, detail);
1292
+ deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
1293
+ + `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
1294
+ }
1265
1295
  }
1266
1296
  if (failure)
1267
1297
  throw failure;