@ours.network/fleet 1.1.5 → 1.2.0-nightly.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +163 -21
  2. package/dist/agent-ours/bridge.d.ts +3 -0
  3. package/dist/agent-ours/bridge.js +178 -0
  4. package/dist/agent-ours/controller.d.ts +14 -0
  5. package/dist/agent-ours/controller.js +60 -0
  6. package/dist/agent-ours/file-rpc.d.ts +23 -0
  7. package/dist/agent-ours/file-rpc.js +123 -0
  8. package/dist/agent-ours/harness.d.ts +14 -0
  9. package/dist/agent-ours/harness.js +64 -0
  10. package/dist/agent-ours/mcp-endpoint.d.ts +15 -0
  11. package/dist/agent-ours/mcp-endpoint.js +111 -0
  12. package/dist/agent-ours/runtime.d.ts +53 -0
  13. package/dist/agent-ours/runtime.js +247 -0
  14. package/dist/agent-ours/service.d.ts +23 -0
  15. package/dist/agent-ours/service.js +364 -0
  16. package/dist/agent-ours/state.d.ts +45 -0
  17. package/dist/agent-ours/state.js +95 -0
  18. package/dist/agent-ours/wire.d.ts +14 -0
  19. package/dist/agent-ours/wire.js +55 -0
  20. package/dist/agent-recovery-gate.js +1 -1
  21. package/dist/application/legacy-supervisor-identity.d.ts +7 -0
  22. package/dist/application/legacy-supervisor-identity.js +43 -0
  23. package/dist/application/supervisor-ours-tools.d.ts +159 -0
  24. package/dist/application/supervisor-ours-tools.js +121 -0
  25. package/dist/application/task-room-service.js +55 -19
  26. package/dist/briefing.js +10 -59
  27. package/dist/build-info.json +6 -5
  28. package/dist/capabilities.d.ts +2 -1
  29. package/dist/capabilities.js +2 -0
  30. package/dist/cli.js +75 -8
  31. package/dist/client-profile.d.ts +18 -0
  32. package/dist/client-profile.js +130 -0
  33. package/dist/config.d.ts +1 -0
  34. package/dist/creation.d.ts +1 -1
  35. package/dist/creation.js +13 -21
  36. package/dist/daemon-recovery.d.ts +2 -0
  37. package/dist/daemon-recovery.js +53 -2
  38. package/dist/docs.d.ts +3 -3
  39. package/dist/docs.js +32 -23
  40. package/dist/doctor.d.ts +1 -1
  41. package/dist/doctor.js +70 -14
  42. package/dist/fleet-command-audit.js +3 -2
  43. package/dist/harness/agent-session.d.ts +2 -0
  44. package/dist/harness/claude-code-session.js +4 -2
  45. package/dist/harness/codex-session.js +9 -2
  46. package/dist/harness/codex.d.ts +3 -2
  47. package/dist/harness/codex.js +8 -3
  48. package/dist/harness/hermes-session.js +7 -1
  49. package/dist/init-wizard.d.ts +11 -6
  50. package/dist/init-wizard.js +33 -0
  51. package/dist/monitor.d.ts +34 -3
  52. package/dist/monitor.js +174 -74
  53. package/dist/owner-channel/channel.d.ts +3 -0
  54. package/dist/owner-channel/channel.js +28 -6
  55. package/dist/owner-channel/ours-client.d.ts +12 -5
  56. package/dist/owner-channel/ours-client.js +48 -12
  57. package/dist/rooms-tasks/archived-absence.d.ts +8 -0
  58. package/dist/rooms-tasks/archived-absence.js +48 -0
  59. package/dist/rooms-tasks/cli.js +1 -1
  60. package/dist/rooms-tasks/close.d.ts +3 -0
  61. package/dist/rooms-tasks/close.js +95 -17
  62. package/dist/rooms-tasks/cowork-adapter.d.ts +4 -0
  63. package/dist/rooms-tasks/cowork-adapter.js +10 -2
  64. package/dist/rooms-tasks/cowork-http.d.ts +3 -0
  65. package/dist/rooms-tasks/cowork-http.js +70 -0
  66. package/dist/rooms-tasks/deletion.js +40 -3
  67. package/dist/rooms-tasks/member-startup.d.ts +1 -0
  68. package/dist/rooms-tasks/member-startup.js +2 -0
  69. package/dist/rooms-tasks/provision.d.ts +11 -0
  70. package/dist/rooms-tasks/provision.js +138 -26
  71. package/dist/rooms-tasks/room-state.js +13 -0
  72. package/dist/rooms-tasks/task-state.d.ts +13 -3
  73. package/dist/rooms-tasks/task-state.js +42 -6
  74. package/dist/rooms-tasks/types.d.ts +12 -0
  75. package/dist/rooms-tasks/workspace-artifacts.d.ts +6 -0
  76. package/dist/rooms-tasks/workspace-artifacts.js +93 -0
  77. package/dist/rooms-tasks/workspace.d.ts +26 -0
  78. package/dist/rooms-tasks/workspace.js +195 -0
  79. package/dist/runner.d.ts +14 -0
  80. package/dist/runner.js +540 -494
  81. package/dist/session/codex-app-server-transport.js +1 -1
  82. package/dist/session/codex-app-server.js +31 -5
  83. package/dist/spawn.js +19 -16
  84. package/dist/startup-readiness.d.ts +13 -0
  85. package/dist/startup-readiness.js +38 -0
  86. package/dist/supervisor/launchd.js +8 -1
  87. package/dist/supervisor/systemd.js +5 -0
  88. package/dist/temp-supervisor-recovery.d.ts +3 -0
  89. package/dist/temp-supervisor-recovery.js +46 -0
  90. package/dist/watchdog/briefing.js +3 -16
  91. package/dist/watchdog/run.js +19 -5
  92. package/dist/web/server.d.ts +2 -0
  93. package/dist/web/server.js +18 -1
  94. package/docs/task-workspaces.md +98 -0
  95. package/package.json +8 -4
  96. package/presets/fleet/roles/Coordinator.yaml +22 -0
package/dist/runner.js CHANGED
@@ -1,5 +1,8 @@
1
+ import { prepareManagedAgent, releaseManagedAgent } from './agent-ours/service.js';
2
+ import { prepareManagedHarness } from './agent-ours/harness.js';
1
3
  import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'node:fs';
2
4
  import { join } from 'node:path';
5
+ import { fileURLToPath } from 'node:url';
3
6
  import { randomUUID } from 'node:crypto';
4
7
  import { parse } from 'yaml';
5
8
  import { agentDir, stateRoot } from './paths.js';
@@ -7,7 +10,6 @@ import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolv
7
10
  import { getAdapter } from './harness/registry.js';
8
11
  import { createMonitor, probeIdentityPresence, } from './monitor.js';
9
12
  import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
10
- import { recoverAgentIdentity } from './agent-recovery-gate.js';
11
13
  import { realExec } from './exec.js';
12
14
  import { resolveIsolation } from './isolation/policy.js';
13
15
  import { selectIsolationBackend } from './isolation/registry.js';
@@ -36,6 +38,8 @@ export class SupervisorRecycleRequiredError extends Error {
36
38
  }
37
39
  }
38
40
  const defaultDeps = () => ({
41
+ prepareAgentOurs: prepareManagedAgent,
42
+ releaseAgentOurs: releaseManagedAgent,
39
43
  exec: realExec,
40
44
  cpuDelegated: () => cpuControllerDelegated(),
41
45
  isAlive: pid => { try {
@@ -423,7 +427,7 @@ export function loadTempRole(name) {
423
427
  * with "no such role". An empty/missing marker means "use the default", same as
424
428
  * no -c was ever given.
425
429
  */
426
- function resolveConfigPath(dir, explicit) {
430
+ export function resolveConfigPath(dir, explicit) {
427
431
  if (explicit)
428
432
  return explicit;
429
433
  const marker = join(dir, '.config-path');
@@ -497,11 +501,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
497
501
  // already ran, so rewriting cannot change the fresh/resume decision.
498
502
  writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
499
503
  const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
504
+ role = { ...role, monitor: { ...role.monitor, mode: 'fleet' } };
500
505
  const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
501
506
  const sessionBackend = role.session ?? 'acp';
502
507
  const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
503
508
  let launch = adapter.agentSession.prepareLaunch(role, prep);
504
- // Isolation is additive: only roles that declare `isolation:` are wrapped.
509
+ // Preserve the role's existing isolation policy.
505
510
  let wrappedArgv = launch.argv;
506
511
  if (role.isolation) {
507
512
  // Start with the same durable context that config validation and doctor judged,
@@ -511,7 +516,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
511
516
  launch = { ...launch, argv: runtime.argv };
512
517
  const ctx = {
513
518
  ...isolationContextFor(role), stateDir: dir, runCwd,
514
- runtimeReadPaths: runtime.readPaths,
519
+ runtimeReadPaths: [...runtime.readPaths, ...resolveLaunchRuntime([process.execPath, fileURLToPath(new URL('./agent-ours/bridge.js', import.meta.url))]).readPaths],
515
520
  };
516
521
  const policy = resolveIsolation(role.isolation, ctx);
517
522
  const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
@@ -533,507 +538,527 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
533
538
  if (rprefix.length)
534
539
  wrappedArgv = [...rprefix, ...wrappedArgv];
535
540
  }
536
- // Start-stagger: space this launch at least start_stagger_ms after the previous
537
- // agent launch across the whole host, so a burst of boots (systemd starts every
538
- // user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
539
- // hit the harness/API rate limit at once. Time-based via a shared launch gate, so
540
- // a lone start or a solo crash-restart waits zero. Applied right before the harness
541
- // agent-session start; the cheap monitor prime still runs immediately after.
542
- if (staggerMs > 0) {
543
- const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
544
- const wait = slot - deps.now();
545
- if (wait > 0) {
546
- deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
547
- await deps.sleep(wait);
548
- }
549
- }
550
- // Prime the supervisor mail monitor's notification cursor at the
551
- // stream tip BEFORE the session launches so no arrival is missed during boot
552
- // (backlog before the tip is the SessionStart hook's job). Native-mode roles
553
- // leave wake ownership to the harness. Temp snapshots predating `monitor:` are
554
- // treated as native (monitor may be undefined on an old role.yaml).
555
- let sessionHandle;
556
- let modelRecovery;
557
- const resolvedMonitorDeps = monitorDeps(deps, role.env);
558
- resolvedMonitorDeps.onFailureEvidence = evidence => {
559
- if (!role.model_chain)
560
- return false;
561
- const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
562
- if (action.kind === 'advance') {
563
- modelRecovery = 'advance';
564
- deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
565
- void sessionHandle?.close();
566
- return true;
567
- }
568
- else if (action.kind === 'hold') {
569
- modelRecovery = 'hold';
570
- deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
571
- void sessionHandle?.close();
572
- return true;
541
+ const managedService = await deps.prepareAgentOurs(role, dir, opts.identityLifetime ? opts.identityLifetime === 'temporary' : temp);
542
+ try {
543
+ const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, harnessChildEnv(role, launch.env, dir));
544
+ // Start-stagger: space this launch at least start_stagger_ms after the previous
545
+ // agent launch across the whole host, so a burst of boots (systemd starts every
546
+ // user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
547
+ // hit the harness/API rate limit at once. Time-based via a shared launch gate, so
548
+ // a lone start or a solo crash-restart waits zero. Applied right before the harness
549
+ // agent-session start; the cheap monitor prime still runs immediately after.
550
+ if (staggerMs > 0) {
551
+ const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
552
+ const wait = slot - deps.now();
553
+ if (wait > 0) {
554
+ deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
555
+ await deps.sleep(wait);
556
+ }
573
557
  }
574
- return false;
575
- };
576
- const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
577
- const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
578
- const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
579
- name, identity: role.identity, agentDir: dir, cfg: role.monitor,
580
- deps: resolvedMonitorDeps,
581
- }) : null;
582
- const daemonObserver = new DaemonGenerationObserver();
583
- const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
584
- let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
585
- if (monitor)
586
- await monitor.prime({ resetCursor: resetMonitorCursor });
587
- rmSync(exitFile, { force: true });
588
- let pid;
589
- let agentSession;
590
- let control;
591
- let unsubscribeRecovery;
592
- let monitorLoop;
593
- let sessionStartupComplete = false;
594
- let sessionClosed = false;
595
- let ownerChannel;
596
- let ownerBinder;
597
- let loopManager;
598
- let arbiter;
599
- let recoveryController;
600
- let reloadLoopConfig;
601
- let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
602
- loop.name, loop.definitionHash, loop.promptHash,
603
- ]));
604
- {
605
- const perms = role.permissions ?? resolvePermissions(undefined, undefined);
606
- // Say once, at startup, that this role will decide permission requests by
607
- // itself. Without it the only trace of an auto-denied tool call is a turn
608
- // that quietly did less than it was asked to.
609
- if (perms.unattended === 'deny')
610
- deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
611
- `permission requests are automatically denied once each (reject_once) and the turn continues`);
612
- agentSession = await deps.startAgentSession(adapter.agentSession, {
613
- role, prep,
614
- launch: { ...launch, argv: wrappedArgv, env: harnessChildEnv(role, launch.env, dir) },
615
- cwd: runCwd, stateDir: dir, mode, permissions: perms,
616
- permissionMode: effectivePermissionMode(role), log: deps.log,
617
- });
618
- pid = agentSession.pid;
619
- arbiter = new RoleTurnArbiter(agentSession);
620
- sessionHandle = arbiter;
621
- unsubscribeRecovery = agentSession.subscribe(event => {
622
- if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
623
- return;
624
- const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
625
- if (evidence)
626
- resolvedMonitorDeps.onFailureEvidence?.(evidence);
627
- });
628
- if (role.owner_channel) {
629
- try {
630
- ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
558
+ // Prime the supervisor mail monitor's notification cursor at the
559
+ // stream tip BEFORE the session launches so no arrival is missed during boot
560
+ // (backlog before the tip is the SessionStart hook's job). Native-mode roles
561
+ // leave wake ownership to the harness. Temp snapshots predating `monitor:` are
562
+ // treated as native (monitor may be undefined on an old role.yaml).
563
+ let sessionHandle;
564
+ let modelRecovery;
565
+ const resolvedMonitorDeps = monitorDeps(deps, role.env);
566
+ resolvedMonitorDeps.onFailureEvidence = evidence => {
567
+ if (!role.model_chain)
568
+ return false;
569
+ const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
570
+ if (action.kind === 'advance') {
571
+ modelRecovery = 'advance';
572
+ deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
573
+ void sessionHandle?.close();
574
+ return true;
631
575
  }
632
- catch (error) {
633
- if (error instanceof OwnerBinderHandoffTimeoutError) {
634
- try {
635
- const status = await deps.reportOwnerStartupFailure(dir);
636
- deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
637
- }
638
- catch (notifyError) {
639
- deps.log(`[${name}] owner channel startup recovery notice unavailable: `
640
- + `${notifyError?.message ?? String(notifyError)}`);
641
- }
642
- }
643
- await agentSession.close();
644
- unsubscribeRecovery?.();
645
- throw new Error(`[${name}] owner channel failed to start: `
646
- + `${error?.message ?? String(error)}`);
576
+ else if (action.kind === 'hold') {
577
+ modelRecovery = 'hold';
578
+ deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
579
+ void sessionHandle?.close();
580
+ return true;
647
581
  }
648
- }
649
- try {
650
- if (role.owner_channel)
651
- ownerChannel = deps.createOwnerChannel({
652
- role: name,
653
- harness: role.harness,
654
- config: role.owner_channel,
655
- session: arbiter,
656
- stateDir: dir,
657
- env: role.env,
658
- log: deps.log,
659
- ...(ownerBinder ? { binderLease: ownerBinder } : {}),
660
- ...(configPath ? { configPath } : {}),
661
- });
662
- control = deps.createControlServer(dir, arbiter, deps.log);
663
- control.setFleetAuditor(ownerChannel ? {
664
- begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
665
- finish: input => ownerChannel.finishFleetCommandAudit(input),
666
- present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
667
- } : localFleetAuditor(dir, name, deps.log));
668
- await control.start();
669
- }
670
- catch (error) {
671
- await ownerChannel?.close().catch(() => undefined);
672
- ownerBinder?.release();
673
- await agentSession.close();
674
- unsubscribeRecovery?.();
675
- throw error;
676
- }
677
- if (!role.roomMemberStartup)
678
- control.setFleetSpawner(async (requested) => {
679
- const event = await executeManagedSpawn(role, configPath, requested, deps.log);
680
- // Room-member launches are collected with their Task/Room transaction so
681
- // causal delivery is Task → Room → Agents → Room active → Task active.
682
- if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
683
- try {
684
- await ownerChannel.notifyFleetSpawn(event);
685
- }
686
- catch (error) {
687
- deps.log(`[${name}] Agent lifecycle notice delivery failed: `
688
- + `${error?.message ?? String(error)}`);
582
+ return false;
583
+ };
584
+ const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
585
+ const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
586
+ const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
587
+ name, identity: role.identity, agentDir: dir, cfg: role.monitor,
588
+ deps: resolvedMonitorDeps,
589
+ }) : null;
590
+ const daemonObserver = new DaemonGenerationObserver();
591
+ const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
592
+ let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
593
+ if (monitor)
594
+ await monitor.prime({ resetCursor: resetMonitorCursor });
595
+ rmSync(exitFile, { force: true });
596
+ let pid;
597
+ let agentSession;
598
+ let control;
599
+ let unsubscribeRecovery;
600
+ let monitorLoop;
601
+ let sessionStartupComplete = false;
602
+ let sessionClosed = false;
603
+ let ownerChannel;
604
+ let ownerBinder;
605
+ let loopManager;
606
+ let arbiter;
607
+ let recoveryController;
608
+ let reloadLoopConfig;
609
+ let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
610
+ loop.name, loop.definitionHash, loop.promptHash,
611
+ ]));
612
+ {
613
+ const perms = role.permissions ?? resolvePermissions(undefined, undefined);
614
+ // Say once, at startup, that this role will decide permission requests by
615
+ // itself. Without it the only trace of an auto-denied tool call is a turn
616
+ // that quietly did less than it was asked to.
617
+ if (perms.unattended === 'deny')
618
+ deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
619
+ `permission requests are automatically denied once each (reject_once) and the turn continues`);
620
+ agentSession = await managedService.runtime.startHarness(() => deps.startAgentSession(adapter.agentSession, {
621
+ managedOurs: managedHarness.ours,
622
+ role, prep,
623
+ launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
624
+ cwd: runCwd, stateDir: dir, mode, permissions: perms,
625
+ permissionMode: effectivePermissionMode(role), log: deps.log,
626
+ }));
627
+ pid = agentSession.pid;
628
+ arbiter = new RoleTurnArbiter(agentSession);
629
+ sessionHandle = arbiter;
630
+ unsubscribeRecovery = agentSession.subscribe(event => {
631
+ if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
632
+ return;
633
+ const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
634
+ if (evidence)
635
+ resolvedMonitorDeps.onFailureEvidence?.(evidence);
636
+ });
637
+ if (role.owner_channel) {
638
+ try {
639
+ ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
640
+ }
641
+ catch (error) {
642
+ if (error instanceof OwnerBinderHandoffTimeoutError) {
643
+ try {
644
+ const status = await deps.reportOwnerStartupFailure(dir);
645
+ deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
646
+ }
647
+ catch (notifyError) {
648
+ deps.log(`[${name}] owner channel startup recovery notice unavailable: `
649
+ + `${notifyError?.message ?? String(notifyError)}`);
650
+ }
689
651
  }
652
+ await agentSession.close();
653
+ unsubscribeRecovery?.();
654
+ throw new Error(`[${name}] owner channel failed to start: `
655
+ + `${error?.message ?? String(error)}`);
690
656
  }
691
- return event;
692
- });
693
- resolvedMonitorDeps.delivery = {
694
- // A wake is only delivered when its turn TERMINATES successfully. A
695
- // refusal or a cancellation reached the agent and was not acted on, so
696
- // the monitor must keep its cursor and try again.
697
- submit: async (text, options) => {
698
- // Cancelling the runner-owned startup prompt makes startup look failed
699
- // and closes the session before the wake turn can run. During startup,
700
- // steer into the live turn instead; after it completes, honor the
701
- // configured interrupt policy normally.
702
- const policy = options?.interrupt;
703
- const interrupt = policy === true && sessionStartupComplete;
704
- const promptOptions = {
705
- interrupt, steer: true,
706
- ...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
707
- origin: { kind: 'fleet-monitor' },
708
- };
709
- // Startup is already a protected boundary: as with immediate mode,
710
- // steer rather than waiting on/cancelling the runner-owned first turn.
711
- const result = policy === 'after_tool' && sessionStartupComplete
712
- ? await arbiter.submitPromptAfterTool(text, promptOptions)
713
- : await arbiter.submitPrompt(text, promptOptions);
714
- const steered = result.accepted
715
- && (result.detail === 'injected' || result.detail === 'startedNewTurn');
716
- const boundary = result.safeBoundary;
717
- const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
718
- const boundaryDetail = boundary && queuedAfterSteeringFailure
719
- ? boundary.state === 'timeout'
720
- ? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
721
- : `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
722
- : boundary
723
- ? boundary.state === 'timeout'
724
- ? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
725
- : boundary.state === 'unsupported'
726
- ? 'after_tool unsupported; used non-cancelling queued delivery'
727
- : `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
728
- : result.detail;
729
- return {
730
- succeeded: result.succeeded || steered,
731
- outcome: steered ? result.detail : result.outcome,
732
- detail: result.succeeded || steered || !boundary
733
- ? boundaryDetail
734
- : [result.detail, boundaryDetail].filter(Boolean).join('; '),
735
- ...(boundary ? { safeBoundary: boundary.state } : {}),
736
- };
737
- },
738
- };
739
- const firstPrompt = mode === 'fresh'
740
- ? `Read and follow ${join(dir, 'briefing.md')} now.`
741
- : adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
742
- // Wait for the first turn's TERMINAL result. An agent that accepts the
743
- // startup prompt and then refuses it has not started; logging the role as
744
- // up would hide a role that never read its briefing.
745
- const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
746
- // Monitoring starts immediately. The delivery adapter above downgrades
747
- // interruption to steering until this startup turn reaches a terminal
748
- // success, so there is neither a deaf gap nor a boot-cancellation loop.
749
- monitorLoop = monitor?.run(pid);
750
- const started = await starting;
751
- // A temporary role's first turn can be the active turn when an ours wake
752
- // needs immediate attention. A typed console/monitor cancellation ends
753
- // only that turn: the already-live agent session and any queued wake remain
754
- // valid. Keep every unproven cancellation, refusal, shutdown, and genuine
755
- // failure terminal so a role that never accepted its briefing is not
756
- // silently reported as healthy.
757
- const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
758
- || (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
759
- if (!started.succeeded && !interruptedForWake) {
760
- monitor?.stop();
761
- await control.close();
762
- ownerBinder?.release();
763
- await agentSession.close();
764
- unsubscribeRecovery?.();
765
- if (modelRecovery) {
766
- if (monitorLoop)
767
- await monitorLoop;
768
- return {
769
- elapsedSecs: 0,
770
- exit: {
771
- version: 1,
772
- class: 'program-exit',
773
- detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
774
- },
775
- rotated: false,
776
- mode,
777
- modelRecovery,
778
- };
779
657
  }
780
- throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
781
- `${started.detail ? `: ${started.detail}` : ''}`);
782
- }
783
- if (started.cancellationSource === 'stall-watchdog')
784
- deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
785
- else if (interruptedForWake)
786
- deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
787
- + 'keeping temporary supervisor alive');
788
- sessionStartupComplete = true;
789
- if (ownerChannel) {
790
658
  try {
791
- await ownerChannel.start();
659
+ if (role.owner_channel)
660
+ ownerChannel = deps.createOwnerChannel({
661
+ role: name,
662
+ harness: role.harness,
663
+ config: role.owner_channel,
664
+ session: arbiter,
665
+ startupPending: () => !sessionStartupComplete,
666
+ stateDir: dir,
667
+ env: role.env,
668
+ log: deps.log,
669
+ ...(ownerBinder ? { binderLease: ownerBinder } : {}),
670
+ ...(configPath ? { configPath } : {}),
671
+ });
672
+ control = deps.createControlServer(dir, arbiter, deps.log);
673
+ control.setFleetAuditor(ownerChannel ? {
674
+ begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
675
+ finish: input => ownerChannel.finishFleetCommandAudit(input),
676
+ present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
677
+ } : localFleetAuditor(dir, name, deps.log));
678
+ await control.start();
792
679
  }
793
680
  catch (error) {
794
- monitor?.stop();
795
- if (monitorLoop)
796
- await monitorLoop;
797
- await ownerChannel.close().catch(() => undefined);
798
- await control.close();
681
+ await ownerChannel?.close().catch(() => undefined);
799
682
  ownerBinder?.release();
800
683
  await agentSession.close();
801
684
  unsubscribeRecovery?.();
802
- throw new Error(`[${name}] owner channel failed to start: `
803
- + `${error?.message ?? String(error)}`);
804
- }
805
- }
806
- if (ownerChannel) {
807
- control.setOwnerChannel(ownerChannel);
808
- }
809
- reloadLoopConfig = async () => {
810
- const nextRole = findRole(loadConfig(configPath), name);
811
- const definitions = nextRole.loops ?? [];
812
- const generation = JSON.stringify(definitions.map(loop => [
813
- loop.name, loop.definitionHash, loop.promptHash,
814
- ]));
815
- if (generation === loopGeneration && (loopManager || !definitions.length))
816
- return { changed: false, loops: definitions.length };
817
- if (!loopManager && definitions.length) {
818
- loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
819
- now: deps.now,
820
- setTimer: (callback, ms) => setTimeout(callback, ms),
821
- clearTimer: timer => clearTimeout(timer),
822
- log: deps.log,
823
- });
824
- control.setLoopManager(loopManager);
825
- loopManager.start();
685
+ throw error;
826
686
  }
827
- else {
828
- loopManager?.reconcile(definitions);
829
- }
830
- loopGeneration = generation;
831
- deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
832
- return { changed: true, loops: definitions.length };
833
- };
834
- // Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
835
- if (!temp)
836
- control.setConfigReloader(reloadLoopConfig);
837
- if (role.loops?.length) {
838
- try {
839
- loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
840
- now: deps.now,
841
- setTimer: (callback, ms) => setTimeout(callback, ms),
842
- clearTimer: timer => clearTimeout(timer),
843
- log: deps.log,
687
+ if (!role.roomMemberStartup)
688
+ control.setFleetSpawner(async (requested) => {
689
+ const event = await executeManagedSpawn(role, configPath, requested, deps.log);
690
+ // Room-member launches are collected with their Task/Room transaction so
691
+ // causal delivery is Task → Room → Agents → Room active → Task active.
692
+ if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
693
+ try {
694
+ await ownerChannel.notifyFleetSpawn(event);
695
+ }
696
+ catch (error) {
697
+ deps.log(`[${name}] Agent lifecycle notice delivery failed: `
698
+ + `${error?.message ?? String(error)}`);
699
+ }
700
+ }
701
+ return event;
844
702
  });
845
- control.setLoopManager(loopManager);
846
- loopManager.start();
847
- }
848
- catch (error) {
849
- if (temp) {
703
+ resolvedMonitorDeps.delivery = {
704
+ // A wake is only delivered when its turn TERMINATES successfully. A
705
+ // refusal or a cancellation reached the agent and was not acted on, so
706
+ // the monitor must keep its cursor and try again.
707
+ submit: async (text, options) => {
708
+ // Cancelling the runner-owned startup prompt makes startup look failed
709
+ // and closes the session before the wake turn can run. During startup,
710
+ // steer into the live turn instead; after it completes, honor the
711
+ // configured interrupt policy normally.
712
+ const policy = options?.interrupt;
713
+ const interrupt = policy === true && sessionStartupComplete;
714
+ const promptOptions = {
715
+ interrupt, steer: true,
716
+ ...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
717
+ origin: { kind: 'fleet-monitor' },
718
+ };
719
+ // Startup is already a protected boundary: as with immediate mode,
720
+ // steer rather than waiting on/cancelling the runner-owned first turn.
721
+ const result = policy === 'after_tool' && sessionStartupComplete
722
+ ? await arbiter.submitPromptAfterTool(text, promptOptions)
723
+ : await arbiter.submitPrompt(text, promptOptions);
724
+ const steered = result.accepted
725
+ && (result.detail === 'injected' || result.detail === 'startedNewTurn');
726
+ const boundary = result.safeBoundary;
727
+ const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
728
+ const boundaryDetail = boundary && queuedAfterSteeringFailure
729
+ ? boundary.state === 'timeout'
730
+ ? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
731
+ : `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
732
+ : boundary
733
+ ? boundary.state === 'timeout'
734
+ ? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
735
+ : boundary.state === 'unsupported'
736
+ ? 'after_tool unsupported; used non-cancelling queued delivery'
737
+ : `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
738
+ : result.detail;
739
+ return {
740
+ succeeded: result.succeeded || steered,
741
+ outcome: steered ? result.detail : result.outcome,
742
+ detail: result.succeeded || steered || !boundary
743
+ ? boundaryDetail
744
+ : [result.detail, boundaryDetail].filter(Boolean).join('; '),
745
+ ...(boundary ? { safeBoundary: boundary.state } : {}),
746
+ };
747
+ },
748
+ };
749
+ const firstPrompt = mode === 'fresh'
750
+ ? `Read and follow ${join(dir, 'briefing.md')} now.`
751
+ : `Your supervisor has verified your assigned identity and room readiness. Read ${join(dir, 'WORKLOG.md')} and ${join(dir, 'briefing.md')}, then continue using the available ours tools.`;
752
+ // Wait for the first turn's TERMINAL result. An agent that accepts the
753
+ // startup prompt and then refuses it has not started; logging the role as
754
+ // up would hide a role that never read its briefing.
755
+ const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
756
+ // Monitoring starts immediately. The delivery adapter above downgrades
757
+ // interruption to steering until this startup turn reaches a terminal
758
+ // success, so there is neither a deaf gap nor a boot-cancellation loop.
759
+ monitorLoop = monitor?.run(pid);
760
+ // Owner traffic and agent replies must be observed during a long first
761
+ // turn. Ordinary owner input steers until startup has proved successful.
762
+ if (ownerChannel) {
763
+ try {
764
+ await ownerChannel.start();
765
+ control.setOwnerChannel(ownerChannel);
766
+ }
767
+ catch (error) {
850
768
  monitor?.stop();
851
769
  if (monitorLoop)
852
770
  await monitorLoop;
853
771
  await control.close();
772
+ await ownerChannel.close().catch(() => undefined);
854
773
  ownerBinder?.release();
855
774
  await agentSession.close();
856
775
  unsubscribeRecovery?.();
857
- throw new Error(`[${name}] configured temporary loop manager failed to start: `
776
+ throw new Error(`[${name}] owner channel failed to start: `
858
777
  + `${error?.message ?? String(error)}`);
859
778
  }
860
- deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
861
- }
862
- }
863
- }
864
- deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
865
- // The monitor loop lives exactly as long as the session: it starts once the
866
- // pane pid is known and is stopped when that pid dies (task dies with runner).
867
- monitorLoop ??= monitor?.run(pid);
868
- recoveryController = new RoleRecoveryController({
869
- role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
870
- log: deps.log,
871
- recoverAgent: async () => {
872
- const evidence = await recoverAgentIdentity(arbiter, role.identity);
873
- return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
874
- },
875
- recoverOwner: async (epoch) => {
876
- if (!ownerChannel?.recover)
877
- return { ok: true };
878
- try {
879
- await ownerChannel.recover(epoch);
880
- return { ok: true };
881
779
  }
882
- catch (error) {
883
- return { ok: false, reason: error instanceof Error
884
- ? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
780
+ const started = await starting;
781
+ // A temporary role's first turn can be the active turn when an ours wake
782
+ // needs immediate attention. A typed console/monitor cancellation ends
783
+ // only that turn: the already-live agent session and any queued wake remain
784
+ // valid. Keep every unproven cancellation, refusal, shutdown, and genuine
785
+ // failure terminal so a role that never accepted its briefing is not
786
+ // silently reported as healthy.
787
+ const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
788
+ || (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
789
+ if (!started.succeeded && !interruptedForWake) {
790
+ monitor?.stop();
791
+ await control.close();
792
+ await ownerChannel?.close().catch(() => undefined);
793
+ ownerBinder?.release();
794
+ await agentSession.close();
795
+ unsubscribeRecovery?.();
796
+ if (modelRecovery) {
797
+ if (monitorLoop)
798
+ await monitorLoop;
799
+ return {
800
+ elapsedSecs: 0,
801
+ exit: {
802
+ version: 1,
803
+ class: 'program-exit',
804
+ detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
805
+ },
806
+ rotated: false,
807
+ mode,
808
+ modelRecovery,
809
+ };
810
+ }
811
+ throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
812
+ `${started.detail ? `: ${started.detail}` : ''}`);
885
813
  }
886
- },
887
- });
888
- const start = deps.now();
889
- let nextLoopReloadAt = deps.now() + 30_000;
890
- let nextIdentityPollAt = deps.now();
891
- let nextDaemonProbeAt = deps.now();
892
- let lastReloadError = '';
893
- let identityObserved = false;
894
- let identityAbsentSince;
895
- let retirementReason;
896
- let supervisorRecycleRequired = false;
897
- while (sessionHandle.isAlive()) {
898
- await deps.sleep(temp ? 500 : 2000);
899
- const now = deps.now();
900
- if (now >= nextDaemonProbeAt) {
901
- nextDaemonProbeAt = now + 2_000;
902
- const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
903
- if (observation.kind === 'lost')
904
- recoveryController.noteLoss(observation.reason);
905
- if (observation.kind === 'changed' || observation.kind === 'available'
906
- || (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
907
- sessionStartedWithoutDaemonBaseline = false;
908
- void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
814
+ if (started.cancellationSource === 'stall-watchdog')
815
+ deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
816
+ else if (interruptedForWake)
817
+ deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
818
+ + 'keeping temporary supervisor alive');
819
+ sessionStartupComplete = true;
820
+ reloadLoopConfig = async () => {
821
+ const nextRole = findRole(loadConfig(configPath), name);
822
+ const definitions = nextRole.loops ?? [];
823
+ const generation = JSON.stringify(definitions.map(loop => [
824
+ loop.name, loop.definitionHash, loop.promptHash,
825
+ ]));
826
+ if (generation === loopGeneration && (loopManager || !definitions.length))
827
+ return { changed: false, loops: definitions.length };
828
+ if (!loopManager && definitions.length) {
829
+ loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
830
+ now: deps.now,
831
+ setTimer: (callback, ms) => setTimeout(callback, ms),
832
+ clearTimer: timer => clearTimeout(timer),
833
+ log: deps.log,
834
+ });
835
+ control.setLoopManager(loopManager);
836
+ loopManager.start();
837
+ }
838
+ else {
839
+ loopManager?.reconcile(definitions);
840
+ }
841
+ loopGeneration = generation;
842
+ deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
843
+ return { changed: true, loops: definitions.length };
844
+ };
845
+ // Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
846
+ if (!temp)
847
+ control.setConfigReloader(reloadLoopConfig);
848
+ if (role.loops?.length) {
849
+ try {
850
+ loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
851
+ now: deps.now,
852
+ setTimer: (callback, ms) => setTimeout(callback, ms),
853
+ clearTimer: timer => clearTimeout(timer),
854
+ log: deps.log,
855
+ });
856
+ control.setLoopManager(loopManager);
857
+ loopManager.start();
858
+ }
859
+ catch (error) {
860
+ if (temp) {
861
+ monitor?.stop();
862
+ if (monitorLoop)
863
+ await monitorLoop;
864
+ await control.close();
865
+ ownerBinder?.release();
866
+ await agentSession.close();
867
+ unsubscribeRecovery?.();
868
+ throw new Error(`[${name}] configured temporary loop manager failed to start: `
869
+ + `${error?.message ?? String(error)}`);
870
+ }
871
+ deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
872
+ }
909
873
  }
910
874
  }
911
- if (temp && deps.shouldStop?.()) {
912
- retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
913
- deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
914
- await sessionHandle.close();
915
- sessionClosed = true;
916
- break;
917
- }
918
- if (temp && now >= nextIdentityPollAt) {
919
- nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
920
- const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
921
- if (presence.state === 'present') {
922
- identityObserved = true;
923
- identityAbsentSince = undefined;
875
+ deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
876
+ // The monitor loop lives exactly as long as the session: it starts once the
877
+ // pane pid is known and is stopped when that pid dies (task dies with runner).
878
+ monitorLoop ??= monitor?.run(pid);
879
+ recoveryController = new RoleRecoveryController({
880
+ role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
881
+ log: deps.log,
882
+ recoverAgent: async () => {
883
+ try {
884
+ const release = await managedService.runtime.admit();
885
+ release();
886
+ return { ok: true };
887
+ }
888
+ catch {
889
+ return { ok: false, reason: 'SUPERVISOR_IDENTITY_NOT_READY' };
890
+ }
891
+ },
892
+ recoverOwner: async (epoch) => {
893
+ if (!ownerChannel?.recover)
894
+ return { ok: true };
895
+ try {
896
+ await ownerChannel.recover(epoch);
897
+ return { ok: true };
898
+ }
899
+ catch (error) {
900
+ return { ok: false, reason: error instanceof Error
901
+ ? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
902
+ }
903
+ },
904
+ });
905
+ const start = deps.now();
906
+ let nextLoopReloadAt = deps.now() + 30_000;
907
+ let nextIdentityPollAt = deps.now();
908
+ let nextDaemonProbeAt = deps.now();
909
+ let lastReloadError = '';
910
+ let identityObserved = false;
911
+ let identityAbsentSince;
912
+ let retirementReason;
913
+ let supervisorRecycleRequired = false;
914
+ while (sessionHandle.isAlive()) {
915
+ await deps.sleep(temp ? 500 : 2000);
916
+ const now = deps.now();
917
+ if (now >= nextDaemonProbeAt) {
918
+ nextDaemonProbeAt = now + 2_000;
919
+ const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
920
+ if (observation.kind === 'lost')
921
+ recoveryController.noteLoss(observation.reason);
922
+ if (observation.kind === 'changed' || observation.kind === 'available'
923
+ || (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
924
+ sessionStartedWithoutDaemonBaseline = false;
925
+ void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
926
+ }
924
927
  }
925
- else if (presence.state === 'absent' && identityObserved) {
926
- identityAbsentSince ??= now;
927
- // Require a continuous, time-bounded run of authoritative absence.
928
- // The first positive observation is the readiness gate: cold harness
929
- // starts may spend minutes loading the harness and briefing before the
930
- // agent creates/binds its identity, and absence before then is not a
931
- // close event. After readiness, debounce a real disappearance.
932
- if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
933
- retirementReason = 'identity-closed';
934
- deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
935
- await sessionHandle.close();
936
- sessionClosed = true;
937
- break;
928
+ if (deps.shouldStop?.()) {
929
+ if (temp) {
930
+ retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
931
+ deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
938
932
  }
933
+ await sessionHandle.close();
934
+ sessionClosed = true;
935
+ break;
939
936
  }
940
- else if (presence.state === 'unknown')
941
- identityAbsentSince = undefined;
942
- }
943
- if (reloadLoopConfig && now >= nextLoopReloadAt) {
944
- nextLoopReloadAt = now + 30_000;
945
- try {
946
- await reloadLoopConfig();
947
- lastReloadError = '';
937
+ if (temp && now >= nextIdentityPollAt) {
938
+ nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
939
+ const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env);
940
+ if (presence.state === 'present') {
941
+ identityObserved = true;
942
+ identityAbsentSince = undefined;
943
+ }
944
+ else if (presence.state === 'absent' && identityObserved) {
945
+ identityAbsentSince ??= now;
946
+ // Require a continuous, time-bounded run of authoritative absence.
947
+ // The first positive observation is the readiness gate: cold harness
948
+ // starts may spend minutes loading the harness and briefing before the
949
+ // agent creates/binds its identity, and absence before then is not a
950
+ // close event. After readiness, debounce a real disappearance.
951
+ if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
952
+ retirementReason = 'identity-closed';
953
+ deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
954
+ await sessionHandle.close();
955
+ sessionClosed = true;
956
+ break;
957
+ }
958
+ }
959
+ else if (presence.state === 'unknown')
960
+ identityAbsentSince = undefined;
948
961
  }
949
- catch (error) {
950
- const message = error?.message ?? String(error);
951
- if (message !== lastReloadError)
952
- deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
953
- lastReloadError = message;
962
+ if (reloadLoopConfig && now >= nextLoopReloadAt) {
963
+ nextLoopReloadAt = now + 30_000;
964
+ try {
965
+ await reloadLoopConfig();
966
+ lastReloadError = '';
967
+ }
968
+ catch (error) {
969
+ const message = error?.message ?? String(error);
970
+ if (message !== lastReloadError)
971
+ deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
972
+ lastReloadError = message;
973
+ }
954
974
  }
955
975
  }
956
- }
957
- if (loopManager) {
958
- control?.setLoopManager(undefined);
959
- await loopManager.stop();
960
- }
961
- // Let already-settled path operations publish their aggregate result before
962
- // the shutdown fence invalidates the epoch. This never waits on I/O.
963
- await Promise.resolve();
964
- await Promise.resolve();
965
- recoveryController?.cancel();
966
- control?.setConfigReloader(undefined);
967
- // Close the authenticated control route before releasing the binder lease;
968
- // otherwise the predecessor can unlink the replacement's new socket.
969
- if (control) {
970
- control.setOwnerChannel(undefined);
971
- await control.close();
972
- control = undefined;
973
- }
974
- if (ownerChannel)
975
- await ownerChannel.close();
976
- if (ownerChannel?.binderReleaseSafe?.() === false) {
977
- deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
978
- supervisorRecycleRequired = true;
979
- }
980
- else
981
- ownerBinder?.release();
982
- if (monitor) {
983
- monitor.stop();
984
- await monitorLoop;
985
- }
986
- unsubscribeRecovery?.();
987
- if (agentSession && !sessionClosed)
988
- await agentSession.close();
989
- const elapsed = (deps.now() - start) / 1000;
990
- // Establish what actually happened before deciding anything. Absence of a
991
- // record is `unknown` — except when the console itself is gone, which is a
992
- // different event with a different consequence.
993
- const exitRecord = agentSession.exitResult()
994
- ?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
995
- writeFileSync(exitFile, JSON.stringify({
996
- ...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
997
- }) + '\n');
998
- let rotated = false;
999
- const rotate = (why) => {
1000
- writeFileSync(sidFile, randomUUID() + '\n');
1001
- rmSync(bootedFile, { force: true });
1002
- rotated = true;
1003
- deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
1004
- };
1005
- if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
1006
- || exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
1007
- // This is a deliberate adapter reclamation, not evidence that resume state
1008
- // is poisoned. Preserve the context even when the resumed generation hits
1009
- // the same bound immediately; runSupervised still counts the fast exit and
1010
- // opens its circuit after the configured number of consecutive failures.
1011
- deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1012
- `-> next start RESUMES context`);
1013
- else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
1014
- rotate(`clean exit (code 0)`);
1015
- else if (exitRecord.class === 'session-destroyed')
1016
- // Someone tore the console down; the agent did not fail. Rotating here
1017
- // would discard a live conversation for an operator action.
1018
- deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1019
- else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
1020
- // Self-heal a poisoned resume — but only once per failure sequence. Rotating
1021
- // on every attempt would discard the conversation again and again while the
1022
- // real cause (a broken command, a missing binary) went unaddressed.
1023
- if (opts.allowResumeRotation === false)
1024
- deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1025
- `-> resume state was already discarded once; keeping it`);
976
+ if (loopManager) {
977
+ control?.setLoopManager(undefined);
978
+ await loopManager.stop();
979
+ }
980
+ // Let already-settled path operations publish their aggregate result before
981
+ // the shutdown fence invalidates the epoch. This never waits on I/O.
982
+ await Promise.resolve();
983
+ await Promise.resolve();
984
+ recoveryController?.cancel();
985
+ control?.setConfigReloader(undefined);
986
+ // Close the authenticated control route before releasing the binder lease;
987
+ // otherwise the predecessor can unlink the replacement's new socket.
988
+ if (control) {
989
+ control.setOwnerChannel(undefined);
990
+ await control.close();
991
+ control = undefined;
992
+ }
993
+ if (ownerChannel)
994
+ await ownerChannel.close();
995
+ if (ownerChannel?.binderReleaseSafe?.() === false) {
996
+ deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
997
+ supervisorRecycleRequired = true;
998
+ }
1026
999
  else
1027
- rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
1000
+ ownerBinder?.release();
1001
+ if (monitor) {
1002
+ monitor.stop();
1003
+ await monitorLoop;
1004
+ }
1005
+ unsubscribeRecovery?.();
1006
+ if (agentSession && !sessionClosed)
1007
+ await agentSession.close();
1008
+ const elapsed = (deps.now() - start) / 1000;
1009
+ // Establish what actually happened before deciding anything. Absence of a
1010
+ // record is `unknown` — except when the console itself is gone, which is a
1011
+ // different event with a different consequence.
1012
+ const exitRecord = agentSession.exitResult()
1013
+ ?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
1014
+ writeFileSync(exitFile, JSON.stringify({
1015
+ ...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
1016
+ }) + '\n');
1017
+ let rotated = false;
1018
+ const rotate = (why) => {
1019
+ writeFileSync(sidFile, randomUUID() + '\n');
1020
+ rmSync(bootedFile, { force: true });
1021
+ rotated = true;
1022
+ deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
1023
+ };
1024
+ if (!temp && deps.shouldStop?.())
1025
+ deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
1026
+ else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
1027
+ || exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
1028
+ // This is a deliberate adapter reclamation, not evidence that resume state
1029
+ // is poisoned. Preserve the context even when the resumed generation hits
1030
+ // the same bound immediately; runSupervised still counts the fast exit and
1031
+ // opens its circuit after the configured number of consecutive failures.
1032
+ deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1033
+ `-> next start RESUMES context`);
1034
+ else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
1035
+ rotate(`clean exit (code 0)`);
1036
+ else if (exitRecord.class === 'session-destroyed')
1037
+ // Someone tore the console down; the agent did not fail. Rotating here
1038
+ // would discard a live conversation for an operator action.
1039
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1040
+ else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
1041
+ // Self-heal a poisoned resume — but only once per failure sequence. Rotating
1042
+ // on every attempt would discard the conversation again and again while the
1043
+ // real cause (a broken command, a missing binary) went unaddressed.
1044
+ if (opts.allowResumeRotation === false)
1045
+ deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
1046
+ `-> resume state was already discarded once; keeping it`);
1047
+ else
1048
+ rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
1049
+ }
1050
+ else
1051
+ deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1052
+ if (supervisorRecycleRequired)
1053
+ throw new SupervisorRecycleRequiredError();
1054
+ return {
1055
+ elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
1056
+ ...(retirementReason ? { retirementReason } : {}),
1057
+ };
1058
+ }
1059
+ finally {
1060
+ await managedService.close(false);
1028
1061
  }
1029
- else
1030
- deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
1031
- if (supervisorRecycleRequired)
1032
- throw new SupervisorRecycleRequiredError();
1033
- return {
1034
- elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
1035
- ...(retirementReason ? { retirementReason } : {}),
1036
- };
1037
1062
  }
1038
1063
  /**
1039
1064
  * The persistent supervisor for one permanent role: run child sessions in a
@@ -1049,7 +1074,10 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
1049
1074
  const deps = { ...defaultDeps(), ...partialDeps };
1050
1075
  const dir = agentDir(name);
1051
1076
  mkdirSync(dir, { recursive: true });
1052
- const shouldStop = deps.shouldStop ?? (() => false);
1077
+ let stopping = false;
1078
+ const requestStop = () => { stopping = true; };
1079
+ const shouldStop = () => stopping || (partialDeps.shouldStop?.() ?? false);
1080
+ deps.shouldStop = shouldStop;
1053
1081
  const stamp = () => new Date(deps.now()).toISOString();
1054
1082
  // Record how the PREVIOUS supervisor process ended before doing anything
1055
1083
  // else. An external kill writes nothing itself, and the next ledger write is
@@ -1071,6 +1099,9 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
1071
1099
  deps.log(`[${name}] previous supervisor run (started ${termination.runStartedAt}) `
1072
1100
  + `ended abruptly: ${termination.detail}; abrupt terminations recorded: ${abrupt}`);
1073
1101
  }
1102
+ // Service-manager stops must run the same cleanup as a completed session.
1103
+ process.on('SIGTERM', requestStop);
1104
+ process.on('SIGINT', requestStop);
1074
1105
  try {
1075
1106
  while (!shouldStop()) {
1076
1107
  let ledger = readRestartLedger(dir);
@@ -1125,6 +1156,8 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
1125
1156
  mode: 'fresh',
1126
1157
  };
1127
1158
  }
1159
+ if (stopping)
1160
+ break;
1128
1161
  // Re-read: the attempt itself may have taken minutes, and an operator may
1129
1162
  // have reset the ledger meanwhile.
1130
1163
  ledger = readRestartLedger(dir);
@@ -1189,8 +1222,12 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
1189
1222
  return readRestartLedger(dir);
1190
1223
  }
1191
1224
  finally {
1192
- // Only an orderly return through this loop clears the marker; a signal or
1193
- // an OOM-kill leaves it, which is exactly how the successor detects them.
1225
+ process.off('SIGTERM', requestStop);
1226
+ process.off('SIGINT', requestStop);
1227
+ // An orderly shutdown clears the marker; an unhandled signal or OOM-kill
1228
+ // leaves it so the successor can identify an abrupt termination.
1229
+ if (shouldStop())
1230
+ await deps.releaseAgentOurs(findRole(loadConfig(opts.configPath), name));
1194
1231
  releaseSupervisorRun(dir);
1195
1232
  }
1196
1233
  }
@@ -1220,10 +1257,15 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
1220
1257
  let result;
1221
1258
  let failure;
1222
1259
  try {
1223
- result = await attempt(name, { temp: true }, {
1224
- ...deps,
1225
- shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
1226
- });
1260
+ for (let recoveryAttempt = 0;; recoveryAttempt++) {
1261
+ result = await attempt(name, { temp: true }, {
1262
+ ...deps,
1263
+ shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
1264
+ });
1265
+ if (result.exit.class === 'clean' || result.retirementReason || signal || deps.shouldStop?.() || recoveryAttempt >= 2)
1266
+ break;
1267
+ await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(Math.min(5000, 1000 * (recoveryAttempt + 1)));
1268
+ }
1227
1269
  }
1228
1270
  catch (error) {
1229
1271
  failure = error;
@@ -1231,23 +1273,27 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
1231
1273
  finally {
1232
1274
  process.off('SIGTERM', onTerm);
1233
1275
  process.off('SIGINT', onInt);
1234
- const requested = requestedTempStopReason(dir);
1235
- const reason = requested
1236
- ?? result?.retirementReason
1237
- ?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
1238
- ?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
1239
- // A service-manager stop can make the child connection close before the
1240
- // runner reaches its normal loop. That is still a successful requested
1241
- // retirement, not a startup failure.
1242
- const outcome = failure && !requested && !signal ? 'failed' : 'retired';
1243
- const detail = failure
1244
- ? (failure instanceof Error ? failure.message : String(failure))
1245
- : result
1246
- ? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
1247
- : 'temporary supervisor ended without an attempt result';
1248
- const archived = archiveTempState(name, reason, outcome, detail);
1249
- deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
1250
- + `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
1276
+ // A recycle replaces the supervisor process, not the logical temporary agent.
1277
+ if (!(failure instanceof SupervisorRecycleRequiredError) || signal || deps.shouldStop?.()) {
1278
+ const requested = requestedTempStopReason(dir);
1279
+ const reason = requested
1280
+ ?? result?.retirementReason
1281
+ ?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
1282
+ ?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
1283
+ // A service-manager stop can make the child connection close before the
1284
+ // runner reaches its normal loop. That is still a successful requested
1285
+ // retirement, not a startup failure.
1286
+ const outcome = failure && !requested && !signal ? 'failed' : 'retired';
1287
+ const detail = failure
1288
+ ? (failure instanceof Error ? failure.message : String(failure))
1289
+ : result
1290
+ ? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
1291
+ : 'temporary supervisor ended without an attempt result';
1292
+ await (deps.releaseAgentOurs ?? releaseManagedAgent)(loadTempRole(name));
1293
+ const archived = archiveTempState(name, reason, outcome, detail);
1294
+ deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
1295
+ + `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
1296
+ }
1251
1297
  }
1252
1298
  if (failure)
1253
1299
  throw failure;