@ours.network/fleet 1.2.0-nightly.2 → 1.2.0-nightly.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -29
- package/dist/agent-ours/bridge.d.ts +3 -0
- package/dist/agent-ours/bridge.js +178 -0
- package/dist/agent-ours/controller.d.ts +14 -0
- package/dist/agent-ours/controller.js +60 -0
- package/dist/agent-ours/correspondence.d.ts +10 -0
- package/dist/agent-ours/correspondence.js +1 -0
- package/dist/agent-ours/file-rpc.d.ts +23 -0
- package/dist/agent-ours/file-rpc.js +123 -0
- package/dist/agent-ours/harness.d.ts +14 -0
- package/dist/agent-ours/harness.js +72 -0
- package/dist/agent-ours/mcp-endpoint.d.ts +15 -0
- package/dist/agent-ours/mcp-endpoint.js +111 -0
- package/dist/agent-ours/runtime.d.ts +59 -0
- package/dist/agent-ours/runtime.js +283 -0
- package/dist/agent-ours/service.d.ts +23 -0
- package/dist/agent-ours/service.js +372 -0
- package/dist/agent-ours/state.d.ts +45 -0
- package/dist/agent-ours/state.js +95 -0
- package/dist/agent-ours/wire.d.ts +14 -0
- package/dist/agent-ours/wire.js +55 -0
- package/dist/agent-recovery-gate.js +1 -1
- package/dist/application/conversation-history.d.ts +6 -0
- package/dist/application/conversation-history.js +65 -0
- package/dist/application/fleet-query-service.d.ts +1 -1
- package/dist/application/fleet-query-service.js +2 -2
- package/dist/application/legacy-supervisor-identity.d.ts +7 -0
- package/dist/application/legacy-supervisor-identity.js +43 -0
- package/dist/application/model-catalog.d.ts +1 -1
- package/dist/application/model-catalog.js +5 -2
- package/dist/application/preset-provenance.d.ts +19 -0
- package/dist/application/preset-provenance.js +59 -0
- package/dist/application/room-layout-definitions.d.ts +45 -0
- package/dist/application/room-layout-definitions.js +130 -0
- package/dist/application/session-control.d.ts +8 -0
- package/dist/application/session-control.js +21 -3
- package/dist/application/supervisor-ours-tools.d.ts +159 -0
- package/dist/application/supervisor-ours-tools.js +113 -0
- package/dist/application/task-layouts.d.ts +89 -0
- package/dist/application/task-layouts.js +248 -0
- package/dist/application/task-room-service.d.ts +99 -3
- package/dist/application/task-room-service.js +296 -31
- package/dist/atomic-file.d.ts +5 -0
- package/dist/atomic-file.js +62 -1
- package/dist/briefing.js +10 -59
- package/dist/build-info.json +5 -4
- package/dist/capabilities.d.ts +2 -1
- package/dist/capabilities.js +2 -0
- package/dist/cli.js +65 -2
- package/dist/client-profile.d.ts +5 -16
- package/dist/client-profile.js +4 -112
- package/dist/config.d.ts +4 -0
- package/dist/config.js +7 -0
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +5 -16
- package/dist/daemon-recovery.d.ts +1 -8
- package/dist/daemon-recovery.js +8 -113
- package/dist/docs.d.ts +3 -3
- package/dist/docs.js +43 -23
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.js +46 -59
- package/dist/erased-resources.d.ts +9 -0
- package/dist/erased-resources.js +87 -0
- package/dist/fleet-command-audit.d.ts +1 -0
- package/dist/fleet-command-audit.js +12 -7
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +4 -2
- package/dist/harness/codex-app-server-proxy.d.ts +4 -1
- package/dist/harness/codex-app-server-proxy.js +41 -5
- package/dist/harness/codex-session.js +9 -2
- package/dist/harness/codex.d.ts +3 -2
- package/dist/harness/codex.js +4 -3
- package/dist/harness/hermes-session.js +7 -1
- package/dist/init-wizard.js +20 -9
- package/dist/monitor.d.ts +1 -34
- package/dist/monitor.js +24 -151
- package/dist/owner-channel/channel.d.ts +3 -0
- package/dist/owner-channel/channel.js +34 -6
- package/dist/owner-channel/commands.d.ts +4 -0
- package/dist/owner-channel/commands.js +8 -0
- package/dist/owner-channel/lifecycle-outbox.js +4 -3
- package/dist/owner-channel/ours-client.js +2 -17
- package/dist/preset-migration.js +15 -0
- package/dist/rooms-tasks/add-member.d.ts +48 -0
- package/dist/rooms-tasks/add-member.js +83 -0
- package/dist/rooms-tasks/archived-absence.d.ts +8 -0
- package/dist/rooms-tasks/archived-absence.js +48 -0
- package/dist/rooms-tasks/cli.js +34 -5
- package/dist/rooms-tasks/close.d.ts +5 -2
- package/dist/rooms-tasks/close.js +208 -28
- package/dist/rooms-tasks/config.js +3 -1
- package/dist/rooms-tasks/cowork-adapter.d.ts +6 -4
- package/dist/rooms-tasks/cowork-adapter.js +11 -117
- package/dist/rooms-tasks/cowork-http.d.ts +3 -0
- package/dist/rooms-tasks/cowork-http.js +70 -0
- package/dist/rooms-tasks/deletion.d.ts +1 -1
- package/dist/rooms-tasks/deletion.js +74 -5
- package/dist/rooms-tasks/erasure.d.ts +6 -0
- package/dist/rooms-tasks/erasure.js +169 -0
- package/dist/rooms-tasks/external-worker.d.ts +1 -1
- package/dist/rooms-tasks/external-worker.js +3 -4
- package/dist/rooms-tasks/layout-cli.d.ts +2 -0
- package/dist/rooms-tasks/layout-cli.js +81 -0
- package/dist/rooms-tasks/layout-config.d.ts +11 -0
- package/dist/rooms-tasks/layout-config.js +89 -0
- package/dist/rooms-tasks/layout-control.d.ts +21 -0
- package/dist/rooms-tasks/layout-control.js +52 -0
- package/dist/rooms-tasks/layout-service.d.ts +39 -0
- package/dist/rooms-tasks/layout-service.js +173 -0
- package/dist/rooms-tasks/layout.d.ts +102 -0
- package/dist/rooms-tasks/layout.js +227 -0
- package/dist/rooms-tasks/live-readiness.d.ts +18 -0
- package/dist/rooms-tasks/live-readiness.js +85 -0
- package/dist/rooms-tasks/member-startup.d.ts +1 -0
- package/dist/rooms-tasks/member-startup.js +2 -0
- package/dist/rooms-tasks/provision.d.ts +14 -0
- package/dist/rooms-tasks/provision.js +182 -48
- package/dist/rooms-tasks/room-state.d.ts +4 -2
- package/dist/rooms-tasks/room-state.js +53 -8
- package/dist/rooms-tasks/task-state.d.ts +42 -7
- package/dist/rooms-tasks/task-state.js +153 -23
- package/dist/rooms-tasks/types.d.ts +28 -0
- package/dist/rooms-tasks/workspace-artifacts.d.ts +6 -0
- package/dist/rooms-tasks/workspace-artifacts.js +93 -0
- package/dist/rooms-tasks/workspace.d.ts +26 -0
- package/dist/rooms-tasks/workspace.js +195 -0
- package/dist/runner.d.ts +23 -3
- package/dist/runner.js +574 -494
- package/dist/session/acp.d.ts +3 -0
- package/dist/session/acp.js +7 -0
- package/dist/session/arbiter.d.ts +3 -0
- package/dist/session/arbiter.js +20 -2
- package/dist/session/codex-app-server-transport.js +1 -1
- package/dist/session/codex-app-server.d.ts +3 -0
- package/dist/session/codex-app-server.js +25 -1
- package/dist/session/control.d.ts +10 -1
- package/dist/session/control.js +36 -2
- package/dist/session/conversation-normalizer.d.ts +2 -1
- package/dist/session/conversation-normalizer.js +1 -1
- package/dist/session/conversation-types.d.ts +5 -3
- package/dist/session/task-notice.d.ts +14 -0
- package/dist/session/task-notice.js +13 -0
- package/dist/session/types.d.ts +3 -0
- package/dist/socket-path.d.ts +2 -0
- package/dist/socket-path.js +22 -0
- package/dist/spawn.js +19 -16
- package/dist/startup-readiness.d.ts +13 -0
- package/dist/startup-readiness.js +38 -0
- package/dist/supervisor/launchd.js +2 -1
- package/dist/supervisor/systemd.js +9 -23
- package/dist/temp-idle.d.ts +17 -0
- package/dist/temp-idle.js +42 -0
- package/dist/temp-lifecycle.d.ts +6 -1
- package/dist/temp-lifecycle.js +29 -13
- package/dist/temp-supervisor-recovery.d.ts +3 -0
- package/dist/temp-supervisor-recovery.js +46 -0
- package/dist/watchdog/briefing.js +3 -16
- package/dist/watchdog/run.js +19 -5
- package/dist/web/agent-attachments.d.ts +29 -0
- package/dist/web/agent-attachments.js +162 -0
- package/dist/web/auth.d.ts +4 -1
- package/dist/web/auth.js +23 -2
- package/dist/web/control.js +2 -1
- package/dist/web/fleet-config-service.d.ts +4 -1
- package/dist/web/fleet-config-service.js +46 -15
- package/dist/web/prefix-gateway.d.ts +20 -0
- package/dist/web/prefix-gateway.js +150 -0
- package/dist/web/runtime.d.ts +4 -0
- package/dist/web/runtime.js +19 -14
- package/dist/web/server.d.ts +7 -0
- package/dist/web/server.js +293 -25
- package/docs/room-layouts.md +92 -0
- package/docs/task-workspaces.md +130 -0
- package/examples/room-layouts/fresh.yaml +27 -0
- package/examples/room-layouts/shared.yaml +19 -0
- package/package.json +17 -13
- package/presets/README.md +78 -0
- package/presets/brain-catalog.json +8 -2
- package/presets/fleet/agent_templates/Engineer.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-1-high.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-1-low.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-1-max.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-1-medium.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-1-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-high.yaml +1 -1
- package/presets/fleet/brains/claude-fable-5-low.yaml +1 -1
- package/presets/fleet/brains/claude-fable-5-max.yaml +1 -1
- package/presets/fleet/brains/claude-fable-5-medium.yaml +1 -1
- package/presets/fleet/brains/claude-fable-5-xhigh.yaml +1 -1
- package/presets/fleet/brains/claude-haiku-4-5-20251001-default.yaml +4 -0
- package/presets/fleet/brains/claude-opus-5-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-high.yaml +1 -1
- package/presets/fleet/brains/claude-opus-5-low.yaml +1 -1
- package/presets/fleet/brains/claude-opus-5-max.yaml +1 -1
- package/presets/fleet/brains/claude-opus-5-medium.yaml +1 -1
- package/presets/fleet/brains/claude-opus-5-xhigh.yaml +1 -1
- package/presets/fleet/brains/claude-sonnet-5-high.yaml +1 -1
- package/presets/fleet/brains/claude-sonnet-5-low.yaml +1 -1
- package/presets/fleet/brains/claude-sonnet-5-max.yaml +1 -1
- package/presets/fleet/brains/claude-sonnet-5-medium.yaml +1 -1
- package/presets/fleet/brains/claude-sonnet-5-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-mini-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-mini-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-mini-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-mini-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-4-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-5-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-5-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-5-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-5-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-luna-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-luna-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-luna-max.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-luna-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-luna-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-max.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-ultra.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-sol-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-max.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-ultra.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-5-6-terra-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-1-sol-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-1-sol-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-1-sol-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-1-sol-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-1-sol-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-1-sol-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-high.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-astra-low.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-astra-max.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-astra-medium.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-astra-ultra.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-astra-xhigh.yaml +1 -1
- package/presets/fleet/brains/codex-gpt-6-luna-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-luna-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-luna-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-luna-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-luna-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-sol-xhigh.yaml +5 -0
- package/presets/fleet/roles/Coordinator.yaml +22 -0
- package/presets/fleet/roles/Engineer.yaml +40 -0
- package/presets/fleet/room_layouts/engineering.yaml +25 -0
- package/presets/fleet/room_layouts/pair.yaml +24 -0
- package/presets/fleet/room_layouts/single.yaml +17 -0
- package/presets/fleet/room_layouts/team.yaml +30 -0
- package/presets/fleet/room_templates/engineering.yaml +19 -0
- package/presets/manifest.json +2 -2
- package/dist/web-app/assets/index-59GF-gsZ.css +0 -1
- package/dist/web-app/assets/index-C0h3ALvs.js +0 -10
- package/dist/web-app/icons/ours-fleet-maskable.svg +0 -4
- package/dist/web-app/icons/ours-fleet.svg +0 -4
- package/dist/web-app/index.html +0 -17
- package/dist/web-app/manifest.webmanifest +0 -15
- package/dist/web-app/offline.html +0 -18
- package/dist/web-app/sw.js +0 -51
package/dist/runner.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { createLayoutControl } from './rooms-tasks/layout-control.js';
|
|
2
|
+
import { TemporaryChatIdle } from './temp-idle.js';
|
|
3
|
+
import { prepareManagedAgent, releaseManagedAgent } from './agent-ours/service.js';
|
|
4
|
+
import { prepareManagedHarness } from './agent-ours/harness.js';
|
|
1
5
|
import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, realpathSync } from 'node:fs';
|
|
2
6
|
import { join } from 'node:path';
|
|
7
|
+
import { fileURLToPath } from 'node:url';
|
|
3
8
|
import { randomUUID } from 'node:crypto';
|
|
4
9
|
import { parse } from 'yaml';
|
|
5
10
|
import { agentDir, stateRoot } from './paths.js';
|
|
@@ -7,7 +12,6 @@ import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolv
|
|
|
7
12
|
import { getAdapter } from './harness/registry.js';
|
|
8
13
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
14
|
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
-
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
11
15
|
import { realExec } from './exec.js';
|
|
12
16
|
import { resolveIsolation } from './isolation/policy.js';
|
|
13
17
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
@@ -36,6 +40,8 @@ export class SupervisorRecycleRequiredError extends Error {
|
|
|
36
40
|
}
|
|
37
41
|
}
|
|
38
42
|
const defaultDeps = () => ({
|
|
43
|
+
prepareAgentOurs: prepareManagedAgent,
|
|
44
|
+
releaseAgentOurs: releaseManagedAgent,
|
|
39
45
|
exec: realExec,
|
|
40
46
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
41
47
|
isAlive: pid => { try {
|
|
@@ -71,12 +77,16 @@ const defaultDeps = () => ({
|
|
|
71
77
|
});
|
|
72
78
|
const MONITOR_OWNER_FILE = '.monitor-owner';
|
|
73
79
|
const OBSOLETE_OURS_AUTOSTART_ENV = 'OURS_AUTOSTART';
|
|
74
|
-
function localFleetAuditor(stateDir, caller, log) {
|
|
80
|
+
export function localFleetAuditor(stateDir, caller, log, session) {
|
|
81
|
+
const bindings = new Map();
|
|
75
82
|
const store = new FleetCommandAuditStore(join(stateDir, '.fleet-command-audit.json'));
|
|
76
83
|
return {
|
|
77
84
|
async begin(requestId, argv) {
|
|
78
85
|
let attempt = store.begin(requestId, caller, argv);
|
|
79
86
|
if (attempt.invocation === 'sending') {
|
|
87
|
+
const binding = session.taskNoticeBinding?.();
|
|
88
|
+
if (binding)
|
|
89
|
+
bindings.set(attempt.correlationId, binding);
|
|
80
90
|
log(`[${caller}] fleet proxy command ${attempt.correlationId} `
|
|
81
91
|
+ `route=${attempt.classification.route} decision=${attempt.classification.decision}`);
|
|
82
92
|
attempt = store.invocation(attempt.correlationId, caller, 'delivered');
|
|
@@ -90,6 +100,19 @@ function localFleetAuditor(stateDir, caller, log) {
|
|
|
90
100
|
...(input.resourceIds ? { resourceIds: input.resourceIds } : {}),
|
|
91
101
|
...(input.presentations ? { presentations: input.presentations } : {}),
|
|
92
102
|
});
|
|
103
|
+
const binding = bindings.get(attempt.correlationId);
|
|
104
|
+
if (binding && attempt.outcome?.class === 'success' && attempt.outcome.effect === 'completed') {
|
|
105
|
+
for (const p of attempt.outcome.presentations ?? [])
|
|
106
|
+
if (p.kind === 'task' && p.operation === 'create') {
|
|
107
|
+
try {
|
|
108
|
+
session.recordTaskCreated?.(binding, { operationId: p.eventId, taskId: p.id, title: p.title ?? p.id, state: p.newState });
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
log(`[${caller}] task-created conversation notice could not be persisted`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
bindings.delete(attempt.correlationId);
|
|
115
|
+
}
|
|
93
116
|
if (attempt.outcome?.delivery === 'sending')
|
|
94
117
|
attempt = store.outcome(attempt.correlationId, caller, 'delivered');
|
|
95
118
|
return attempt;
|
|
@@ -423,7 +446,7 @@ export function loadTempRole(name) {
|
|
|
423
446
|
* with "no such role". An empty/missing marker means "use the default", same as
|
|
424
447
|
* no -c was ever given.
|
|
425
448
|
*/
|
|
426
|
-
function resolveConfigPath(dir, explicit) {
|
|
449
|
+
export function resolveConfigPath(dir, explicit) {
|
|
427
450
|
if (explicit)
|
|
428
451
|
return explicit;
|
|
429
452
|
const marker = join(dir, '.config-path');
|
|
@@ -497,11 +520,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
497
520
|
// already ran, so rewriting cannot change the fresh/resume decision.
|
|
498
521
|
writeFileSync(bootedFile, `${new Date(deps.now()).toISOString()} ${mode}\n`);
|
|
499
522
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
523
|
+
role = { ...role, monitor: { ...role.monitor, mode: 'fleet' } };
|
|
500
524
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
501
525
|
const sessionBackend = role.session ?? 'acp';
|
|
502
526
|
const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
|
|
503
527
|
let launch = adapter.agentSession.prepareLaunch(role, prep);
|
|
504
|
-
//
|
|
528
|
+
// Preserve the role's existing isolation policy.
|
|
505
529
|
let wrappedArgv = launch.argv;
|
|
506
530
|
if (role.isolation) {
|
|
507
531
|
// Start with the same durable context that config validation and doctor judged,
|
|
@@ -511,7 +535,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
511
535
|
launch = { ...launch, argv: runtime.argv };
|
|
512
536
|
const ctx = {
|
|
513
537
|
...isolationContextFor(role), stateDir: dir, runCwd,
|
|
514
|
-
runtimeReadPaths: runtime.readPaths,
|
|
538
|
+
runtimeReadPaths: [...runtime.readPaths, ...resolveLaunchRuntime([process.execPath, fileURLToPath(new URL('./agent-ours/bridge.js', import.meta.url))]).readPaths],
|
|
515
539
|
};
|
|
516
540
|
const policy = resolveIsolation(role.isolation, ctx);
|
|
517
541
|
const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
|
|
@@ -533,511 +557,556 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
533
557
|
if (rprefix.length)
|
|
534
558
|
wrappedArgv = [...rprefix, ...wrappedArgv];
|
|
535
559
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
await
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
554
|
-
// treated as native (monitor may be undefined on an old role.yaml).
|
|
555
|
-
let sessionHandle;
|
|
556
|
-
let modelRecovery;
|
|
557
|
-
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
558
|
-
resolvedMonitorDeps.onFailureEvidence = evidence => {
|
|
559
|
-
if (!role.model_chain)
|
|
560
|
-
return false;
|
|
561
|
-
const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
|
|
562
|
-
if (action.kind === 'advance') {
|
|
563
|
-
modelRecovery = 'advance';
|
|
564
|
-
deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
|
|
565
|
-
void sessionHandle?.close();
|
|
566
|
-
return true;
|
|
567
|
-
}
|
|
568
|
-
else if (action.kind === 'hold') {
|
|
569
|
-
modelRecovery = 'hold';
|
|
570
|
-
deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
|
|
571
|
-
void sessionHandle?.close();
|
|
572
|
-
return true;
|
|
560
|
+
const managedService = await deps.prepareAgentOurs(role, dir, opts.identityLifetime ? opts.identityLifetime === 'temporary' : temp);
|
|
561
|
+
let layoutRetiring = false;
|
|
562
|
+
try {
|
|
563
|
+
const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, harnessChildEnv(role, launch.env, dir));
|
|
564
|
+
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
565
|
+
// agent launch across the whole host, so a burst of boots (systemd starts every
|
|
566
|
+
// user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
|
|
567
|
+
// hit the harness/API rate limit at once. Time-based via a shared launch gate, so
|
|
568
|
+
// a lone start or a solo crash-restart waits zero. Applied right before the harness
|
|
569
|
+
// agent-session start; the cheap monitor prime still runs immediately after.
|
|
570
|
+
if (staggerMs > 0) {
|
|
571
|
+
const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
|
|
572
|
+
const wait = slot - deps.now();
|
|
573
|
+
if (wait > 0) {
|
|
574
|
+
deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
|
|
575
|
+
await deps.sleep(wait);
|
|
576
|
+
}
|
|
573
577
|
}
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
let unsubscribeRecovery;
|
|
592
|
-
let monitorLoop;
|
|
593
|
-
let sessionStartupComplete = false;
|
|
594
|
-
let sessionClosed = false;
|
|
595
|
-
let ownerChannel;
|
|
596
|
-
let ownerBinder;
|
|
597
|
-
let loopManager;
|
|
598
|
-
let arbiter;
|
|
599
|
-
let recoveryController;
|
|
600
|
-
let reloadLoopConfig;
|
|
601
|
-
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
602
|
-
loop.name, loop.definitionHash, loop.promptHash,
|
|
603
|
-
]));
|
|
604
|
-
{
|
|
605
|
-
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
606
|
-
// Say once, at startup, that this role will decide permission requests by
|
|
607
|
-
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
608
|
-
// that quietly did less than it was asked to.
|
|
609
|
-
if (perms.unattended === 'deny')
|
|
610
|
-
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
611
|
-
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
612
|
-
agentSession = await deps.startAgentSession(adapter.agentSession, {
|
|
613
|
-
role, prep,
|
|
614
|
-
launch: { ...launch, argv: wrappedArgv, env: harnessChildEnv(role, launch.env, dir) },
|
|
615
|
-
cwd: runCwd, stateDir: dir, mode, permissions: perms,
|
|
616
|
-
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
617
|
-
});
|
|
618
|
-
pid = agentSession.pid;
|
|
619
|
-
arbiter = new RoleTurnArbiter(agentSession);
|
|
620
|
-
sessionHandle = arbiter;
|
|
621
|
-
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
622
|
-
if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
|
|
623
|
-
return;
|
|
624
|
-
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
625
|
-
if (evidence)
|
|
626
|
-
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
627
|
-
});
|
|
628
|
-
if (role.owner_channel) {
|
|
629
|
-
try {
|
|
630
|
-
ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
|
|
578
|
+
// Prime the supervisor mail monitor's notification cursor at the
|
|
579
|
+
// stream tip BEFORE the session launches so no arrival is missed during boot
|
|
580
|
+
// (backlog before the tip is the SessionStart hook's job). Native-mode roles
|
|
581
|
+
// leave wake ownership to the harness. Temp snapshots predating `monitor:` are
|
|
582
|
+
// treated as native (monitor may be undefined on an old role.yaml).
|
|
583
|
+
let sessionHandle;
|
|
584
|
+
let modelRecovery;
|
|
585
|
+
const resolvedMonitorDeps = monitorDeps(deps, role.env);
|
|
586
|
+
resolvedMonitorDeps.onFailureEvidence = evidence => {
|
|
587
|
+
if (!role.model_chain)
|
|
588
|
+
return false;
|
|
589
|
+
const action = recordModelFailure(dir, role, { ...evidence, model: evidence.model ?? role.model }, role.monitor.turn_fail_threshold ?? 3);
|
|
590
|
+
if (action.kind === 'advance') {
|
|
591
|
+
modelRecovery = 'advance';
|
|
592
|
+
deps.log(`[${name}] MODEL DOWN-SHIFT ${action.from} -> ${action.to}; restarting with resume`);
|
|
593
|
+
void sessionHandle?.close();
|
|
594
|
+
return true;
|
|
631
595
|
}
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
}
|
|
638
|
-
catch (notifyError) {
|
|
639
|
-
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
640
|
-
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
await agentSession.close();
|
|
644
|
-
unsubscribeRecovery?.();
|
|
645
|
-
throw new Error(`[${name}] owner channel failed to start: `
|
|
646
|
-
+ `${error?.message ?? String(error)}`);
|
|
596
|
+
else if (action.kind === 'hold') {
|
|
597
|
+
modelRecovery = 'hold';
|
|
598
|
+
deps.log(`[${name}] MODEL CHAIN EXHAUSTED at ${action.model}; held down`);
|
|
599
|
+
void sessionHandle?.close();
|
|
600
|
+
return true;
|
|
647
601
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
602
|
+
return false;
|
|
603
|
+
};
|
|
604
|
+
const monitorOwner = role.monitor?.mode === 'fleet' ? 'fleet' : 'native';
|
|
605
|
+
const resetMonitorCursor = recordMonitorOwner(dir, monitorOwner);
|
|
606
|
+
const monitor = monitorOwner === 'fleet' ? deps.createMonitor({
|
|
607
|
+
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
608
|
+
deps: resolvedMonitorDeps,
|
|
609
|
+
}) : null;
|
|
610
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
611
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
612
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
613
|
+
if (monitor)
|
|
614
|
+
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
615
|
+
rmSync(exitFile, { force: true });
|
|
616
|
+
let pid;
|
|
617
|
+
let agentSession;
|
|
618
|
+
let control;
|
|
619
|
+
let unsubscribeRecovery;
|
|
620
|
+
let monitorLoop;
|
|
621
|
+
let sessionStartupComplete = false;
|
|
622
|
+
let sessionClosed = false;
|
|
623
|
+
let ownerChannel;
|
|
624
|
+
let ownerBinder;
|
|
625
|
+
let loopManager;
|
|
626
|
+
let arbiter;
|
|
627
|
+
let recoveryController;
|
|
628
|
+
let reloadLoopConfig;
|
|
629
|
+
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
630
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
631
|
+
]));
|
|
632
|
+
{
|
|
633
|
+
const perms = role.permissions ?? resolvePermissions(undefined, undefined);
|
|
634
|
+
// Say once, at startup, that this role will decide permission requests by
|
|
635
|
+
// itself. Without it the only trace of an auto-denied tool call is a turn
|
|
636
|
+
// that quietly did less than it was asked to.
|
|
637
|
+
if (perms.unattended === 'deny')
|
|
638
|
+
deps.log(`[${name}] permission policy: unattended=deny — with no console attached, ` +
|
|
639
|
+
`permission requests are automatically denied once each (reject_once) and the turn continues`);
|
|
640
|
+
agentSession = await managedService.runtime.startHarness(() => deps.startAgentSession(adapter.agentSession, {
|
|
641
|
+
managedOurs: managedHarness.ours,
|
|
642
|
+
role, prep,
|
|
643
|
+
launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
|
|
644
|
+
cwd: runCwd, stateDir: dir, mode, permissions: perms,
|
|
645
|
+
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
646
|
+
}));
|
|
647
|
+
pid = agentSession.pid;
|
|
648
|
+
arbiter = new RoleTurnArbiter(agentSession);
|
|
649
|
+
sessionHandle = arbiter;
|
|
650
|
+
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
651
|
+
if (event.kind !== 'error' || !event.text || event.origin?.kind === 'stall-watchdog')
|
|
652
|
+
return;
|
|
653
|
+
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
654
|
+
if (evidence)
|
|
655
|
+
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
656
|
+
});
|
|
657
|
+
if (role.owner_channel) {
|
|
658
|
+
try {
|
|
659
|
+
ownerBinder = await deps.acquireOwnerBinder(dir, name, role.owner_channel.identity);
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
if (error instanceof OwnerBinderHandoffTimeoutError) {
|
|
663
|
+
try {
|
|
664
|
+
const status = await deps.reportOwnerStartupFailure(dir);
|
|
665
|
+
deps.log(`[${name}] owner channel startup recovery notice ${status} by authenticated predecessor`);
|
|
666
|
+
}
|
|
667
|
+
catch (notifyError) {
|
|
668
|
+
deps.log(`[${name}] owner channel startup recovery notice unavailable: `
|
|
669
|
+
+ `${notifyError?.message ?? String(notifyError)}`);
|
|
670
|
+
}
|
|
689
671
|
}
|
|
672
|
+
await agentSession.close();
|
|
673
|
+
unsubscribeRecovery?.();
|
|
674
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
675
|
+
+ `${error?.message ?? String(error)}`);
|
|
690
676
|
}
|
|
691
|
-
return event;
|
|
692
|
-
});
|
|
693
|
-
resolvedMonitorDeps.delivery = {
|
|
694
|
-
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
695
|
-
// refusal or a cancellation reached the agent and was not acted on, so
|
|
696
|
-
// the monitor must keep its cursor and try again.
|
|
697
|
-
submit: async (text, options) => {
|
|
698
|
-
// Cancelling the runner-owned startup prompt makes startup look failed
|
|
699
|
-
// and closes the session before the wake turn can run. During startup,
|
|
700
|
-
// steer into the live turn instead; after it completes, honor the
|
|
701
|
-
// configured interrupt policy normally.
|
|
702
|
-
const policy = options?.interrupt;
|
|
703
|
-
const interrupt = policy === true && sessionStartupComplete;
|
|
704
|
-
const promptOptions = {
|
|
705
|
-
interrupt, steer: true,
|
|
706
|
-
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
707
|
-
origin: { kind: 'fleet-monitor' },
|
|
708
|
-
};
|
|
709
|
-
// Startup is already a protected boundary: as with immediate mode,
|
|
710
|
-
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
711
|
-
const result = policy === 'after_tool' && sessionStartupComplete
|
|
712
|
-
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
713
|
-
: await arbiter.submitPrompt(text, promptOptions);
|
|
714
|
-
const steered = result.accepted
|
|
715
|
-
&& (result.detail === 'injected' || result.detail === 'startedNewTurn');
|
|
716
|
-
const boundary = result.safeBoundary;
|
|
717
|
-
const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
|
|
718
|
-
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
719
|
-
? boundary.state === 'timeout'
|
|
720
|
-
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
721
|
-
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
722
|
-
: boundary
|
|
723
|
-
? boundary.state === 'timeout'
|
|
724
|
-
? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
|
|
725
|
-
: boundary.state === 'unsupported'
|
|
726
|
-
? 'after_tool unsupported; used non-cancelling queued delivery'
|
|
727
|
-
: `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
|
|
728
|
-
: result.detail;
|
|
729
|
-
return {
|
|
730
|
-
succeeded: result.succeeded || steered,
|
|
731
|
-
outcome: steered ? result.detail : result.outcome,
|
|
732
|
-
detail: result.succeeded || steered || !boundary
|
|
733
|
-
? boundaryDetail
|
|
734
|
-
: [result.detail, boundaryDetail].filter(Boolean).join('; '),
|
|
735
|
-
...(boundary ? { safeBoundary: boundary.state } : {}),
|
|
736
|
-
};
|
|
737
|
-
},
|
|
738
|
-
};
|
|
739
|
-
const firstPrompt = mode === 'fresh'
|
|
740
|
-
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
741
|
-
: adapter.vocabulary.restartPrompt(role.identity, join(dir, 'WORKLOG.md'), role);
|
|
742
|
-
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
743
|
-
// startup prompt and then refuses it has not started; logging the role as
|
|
744
|
-
// up would hide a role that never read its briefing.
|
|
745
|
-
const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
|
|
746
|
-
// Monitoring starts immediately. The delivery adapter above downgrades
|
|
747
|
-
// interruption to steering until this startup turn reaches a terminal
|
|
748
|
-
// success, so there is neither a deaf gap nor a boot-cancellation loop.
|
|
749
|
-
monitorLoop = monitor?.run(pid);
|
|
750
|
-
const started = await starting;
|
|
751
|
-
// A temporary role's first turn can be the active turn when an ours wake
|
|
752
|
-
// needs immediate attention. A typed console/monitor cancellation ends
|
|
753
|
-
// only that turn: the already-live agent session and any queued wake remain
|
|
754
|
-
// valid. Keep every unproven cancellation, refusal, shutdown, and genuine
|
|
755
|
-
// failure terminal so a role that never accepted its briefing is not
|
|
756
|
-
// silently reported as healthy.
|
|
757
|
-
const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
|
|
758
|
-
|| (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
|
|
759
|
-
if (!started.succeeded && !interruptedForWake) {
|
|
760
|
-
monitor?.stop();
|
|
761
|
-
await control.close();
|
|
762
|
-
ownerBinder?.release();
|
|
763
|
-
await agentSession.close();
|
|
764
|
-
unsubscribeRecovery?.();
|
|
765
|
-
if (modelRecovery) {
|
|
766
|
-
if (monitorLoop)
|
|
767
|
-
await monitorLoop;
|
|
768
|
-
return {
|
|
769
|
-
elapsedSecs: 0,
|
|
770
|
-
exit: {
|
|
771
|
-
version: 1,
|
|
772
|
-
class: 'program-exit',
|
|
773
|
-
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
774
|
-
},
|
|
775
|
-
rotated: false,
|
|
776
|
-
mode,
|
|
777
|
-
modelRecovery,
|
|
778
|
-
};
|
|
779
677
|
}
|
|
780
|
-
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
781
|
-
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
782
|
-
}
|
|
783
|
-
if (started.cancellationSource === 'stall-watchdog')
|
|
784
|
-
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
785
|
-
else if (interruptedForWake)
|
|
786
|
-
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
787
|
-
+ 'keeping temporary supervisor alive');
|
|
788
|
-
sessionStartupComplete = true;
|
|
789
|
-
if (ownerChannel) {
|
|
790
678
|
try {
|
|
791
|
-
|
|
679
|
+
if (role.owner_channel)
|
|
680
|
+
ownerChannel = deps.createOwnerChannel({
|
|
681
|
+
role: name,
|
|
682
|
+
harness: role.harness,
|
|
683
|
+
config: role.owner_channel,
|
|
684
|
+
session: arbiter,
|
|
685
|
+
startupPending: () => !sessionStartupComplete,
|
|
686
|
+
stateDir: dir,
|
|
687
|
+
env: role.env,
|
|
688
|
+
log: deps.log,
|
|
689
|
+
...(ownerBinder ? { binderLease: ownerBinder } : {}),
|
|
690
|
+
...(configPath ? { configPath } : {}),
|
|
691
|
+
});
|
|
692
|
+
control = deps.createControlServer(dir, arbiter, deps.log);
|
|
693
|
+
control.setCorrespondence?.({ contacts: () => managedService.runtime.readContacts(), history: query => managedService.runtime.readHistory(query) });
|
|
694
|
+
control.setFleetAuditor(ownerChannel ? {
|
|
695
|
+
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
696
|
+
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
697
|
+
present: presentations => ownerChannel.notifyFleetLifecycle(presentations),
|
|
698
|
+
} : localFleetAuditor(dir, name, deps.log, agentSession));
|
|
699
|
+
control.setLayoutControl?.(createLayoutControl({
|
|
700
|
+
agent: name, temporary: temp, standalone: !role.roomMemberStartup,
|
|
701
|
+
runtime: managedService.runtime, session: arbiter,
|
|
702
|
+
stopping: () => Boolean(deps.shouldStop?.()),
|
|
703
|
+
retire: async () => { layoutRetiring = true; await arbiter.close(); },
|
|
704
|
+
}));
|
|
705
|
+
await control.start();
|
|
792
706
|
}
|
|
793
707
|
catch (error) {
|
|
794
|
-
|
|
795
|
-
if (monitorLoop)
|
|
796
|
-
await monitorLoop;
|
|
797
|
-
await ownerChannel.close().catch(() => undefined);
|
|
798
|
-
await control.close();
|
|
708
|
+
await ownerChannel?.close().catch(() => undefined);
|
|
799
709
|
ownerBinder?.release();
|
|
800
710
|
await agentSession.close();
|
|
801
711
|
unsubscribeRecovery?.();
|
|
802
|
-
throw
|
|
803
|
-
+ `${error?.message ?? String(error)}`);
|
|
804
|
-
}
|
|
805
|
-
}
|
|
806
|
-
if (ownerChannel) {
|
|
807
|
-
control.setOwnerChannel(ownerChannel);
|
|
808
|
-
}
|
|
809
|
-
reloadLoopConfig = async () => {
|
|
810
|
-
const nextRole = findRole(loadConfig(configPath), name);
|
|
811
|
-
const definitions = nextRole.loops ?? [];
|
|
812
|
-
const generation = JSON.stringify(definitions.map(loop => [
|
|
813
|
-
loop.name, loop.definitionHash, loop.promptHash,
|
|
814
|
-
]));
|
|
815
|
-
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
816
|
-
return { changed: false, loops: definitions.length };
|
|
817
|
-
if (!loopManager && definitions.length) {
|
|
818
|
-
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
819
|
-
now: deps.now,
|
|
820
|
-
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
821
|
-
clearTimer: timer => clearTimeout(timer),
|
|
822
|
-
log: deps.log,
|
|
823
|
-
});
|
|
824
|
-
control.setLoopManager(loopManager);
|
|
825
|
-
loopManager.start();
|
|
712
|
+
throw error;
|
|
826
713
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
clearTimer: timer => clearTimeout(timer),
|
|
843
|
-
log: deps.log,
|
|
714
|
+
if (!role.roomMemberStartup)
|
|
715
|
+
control.setFleetSpawner(async (requested) => {
|
|
716
|
+
const event = await executeManagedSpawn(role, configPath, requested, deps.log);
|
|
717
|
+
// Room-member launches are collected with their Task/Room transaction so
|
|
718
|
+
// causal delivery is Task → Room → Agents → Room active → Task active.
|
|
719
|
+
if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
|
|
720
|
+
try {
|
|
721
|
+
await ownerChannel.notifyFleetSpawn(event);
|
|
722
|
+
}
|
|
723
|
+
catch (error) {
|
|
724
|
+
deps.log(`[${name}] Agent lifecycle notice delivery failed: `
|
|
725
|
+
+ `${error?.message ?? String(error)}`);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
return event;
|
|
844
729
|
});
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
730
|
+
resolvedMonitorDeps.delivery = {
|
|
731
|
+
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
732
|
+
// refusal or a cancellation reached the agent and was not acted on, so
|
|
733
|
+
// the monitor must keep its cursor and try again.
|
|
734
|
+
submit: async (text, options) => {
|
|
735
|
+
// Cancelling the runner-owned startup prompt makes startup look failed
|
|
736
|
+
// and closes the session before the wake turn can run. During startup,
|
|
737
|
+
// steer into the live turn instead; after it completes, honor the
|
|
738
|
+
// configured interrupt policy normally.
|
|
739
|
+
const policy = options?.interrupt;
|
|
740
|
+
const interrupt = policy === true && sessionStartupComplete;
|
|
741
|
+
const promptOptions = {
|
|
742
|
+
interrupt, steer: true,
|
|
743
|
+
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
744
|
+
origin: { kind: 'fleet-monitor' },
|
|
745
|
+
};
|
|
746
|
+
// Startup is already a protected boundary: as with immediate mode,
|
|
747
|
+
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
748
|
+
const result = policy === 'after_tool' && sessionStartupComplete
|
|
749
|
+
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
750
|
+
: await arbiter.submitPrompt(text, promptOptions);
|
|
751
|
+
const steered = result.accepted
|
|
752
|
+
&& (result.detail === 'injected' || result.detail === 'startedNewTurn');
|
|
753
|
+
const boundary = result.safeBoundary;
|
|
754
|
+
const queuedAfterSteeringFailure = result.detail?.startsWith('steering rejected;') === true;
|
|
755
|
+
const boundaryDetail = boundary && queuedAfterSteeringFailure
|
|
756
|
+
? boundary.state === 'timeout'
|
|
757
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
758
|
+
: `after_tool ${boundary.state} boundary after ${boundary.waitedMs}ms; steering rejected, queued without cancellation`
|
|
759
|
+
: boundary
|
|
760
|
+
? boundary.state === 'timeout'
|
|
761
|
+
? `after_tool timed out after ${boundary.waitedMs}ms; steered without cancellation`
|
|
762
|
+
: boundary.state === 'unsupported'
|
|
763
|
+
? 'after_tool unsupported; used non-cancelling queued delivery'
|
|
764
|
+
: `after_tool ${boundary.state} delivery after ${boundary.waitedMs}ms`
|
|
765
|
+
: result.detail;
|
|
766
|
+
return {
|
|
767
|
+
succeeded: result.succeeded || steered,
|
|
768
|
+
outcome: steered ? result.detail : result.outcome,
|
|
769
|
+
detail: result.succeeded || steered || !boundary
|
|
770
|
+
? boundaryDetail
|
|
771
|
+
: [result.detail, boundaryDetail].filter(Boolean).join('; '),
|
|
772
|
+
...(boundary ? { safeBoundary: boundary.state } : {}),
|
|
773
|
+
};
|
|
774
|
+
},
|
|
775
|
+
};
|
|
776
|
+
const firstPrompt = mode === 'fresh'
|
|
777
|
+
? `Read and follow ${join(dir, 'briefing.md')} now.`
|
|
778
|
+
: `Your supervisor has verified your assigned identity and room readiness. Read ${join(dir, 'WORKLOG.md')} and ${join(dir, 'briefing.md')}, then continue using the available ours tools.`;
|
|
779
|
+
// Wait for the first turn's TERMINAL result. An agent that accepts the
|
|
780
|
+
// startup prompt and then refuses it has not started; logging the role as
|
|
781
|
+
// up would hide a role that never read its briefing.
|
|
782
|
+
const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
|
|
783
|
+
// Monitoring starts immediately. The delivery adapter above downgrades
|
|
784
|
+
// interruption to steering until this startup turn reaches a terminal
|
|
785
|
+
// success, so there is neither a deaf gap nor a boot-cancellation loop.
|
|
786
|
+
monitorLoop = monitor?.run(pid);
|
|
787
|
+
// Owner traffic and agent replies must be observed during a long first
|
|
788
|
+
// turn. Ordinary owner input steers until startup has proved successful.
|
|
789
|
+
if (ownerChannel) {
|
|
790
|
+
try {
|
|
791
|
+
await ownerChannel.start();
|
|
792
|
+
control.setOwnerChannel(ownerChannel);
|
|
793
|
+
}
|
|
794
|
+
catch (error) {
|
|
850
795
|
monitor?.stop();
|
|
851
796
|
if (monitorLoop)
|
|
852
797
|
await monitorLoop;
|
|
853
798
|
await control.close();
|
|
799
|
+
await ownerChannel.close().catch(() => undefined);
|
|
854
800
|
ownerBinder?.release();
|
|
855
801
|
await agentSession.close();
|
|
856
802
|
unsubscribeRecovery?.();
|
|
857
|
-
throw new Error(`[${name}]
|
|
803
|
+
throw new Error(`[${name}] owner channel failed to start: `
|
|
858
804
|
+ `${error?.message ?? String(error)}`);
|
|
859
805
|
}
|
|
860
|
-
|
|
806
|
+
}
|
|
807
|
+
const started = await starting;
|
|
808
|
+
// A temporary role's first turn can be the active turn when an ours wake
|
|
809
|
+
// needs immediate attention. A typed console/monitor cancellation ends
|
|
810
|
+
// only that turn: the already-live agent session and any queued wake remain
|
|
811
|
+
// valid. Keep every unproven cancellation, refusal, shutdown, and genuine
|
|
812
|
+
// failure terminal so a role that never accepted its briefing is not
|
|
813
|
+
// silently reported as healthy.
|
|
814
|
+
const interruptedForWake = isRecoverableTempStartupCancellation(temp, started)
|
|
815
|
+
|| (started.outcome === 'cancelled' && started.cancellationSource === 'stall-watchdog');
|
|
816
|
+
if (!started.succeeded && !interruptedForWake) {
|
|
817
|
+
monitor?.stop();
|
|
818
|
+
await control.close();
|
|
819
|
+
await ownerChannel?.close().catch(() => undefined);
|
|
820
|
+
ownerBinder?.release();
|
|
821
|
+
await agentSession.close();
|
|
822
|
+
unsubscribeRecovery?.();
|
|
823
|
+
if (modelRecovery) {
|
|
824
|
+
if (monitorLoop)
|
|
825
|
+
await monitorLoop;
|
|
826
|
+
return {
|
|
827
|
+
elapsedSecs: 0,
|
|
828
|
+
exit: {
|
|
829
|
+
version: 1,
|
|
830
|
+
class: 'program-exit',
|
|
831
|
+
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
832
|
+
},
|
|
833
|
+
rotated: false,
|
|
834
|
+
mode,
|
|
835
|
+
modelRecovery,
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
839
|
+
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
840
|
+
}
|
|
841
|
+
if (started.cancellationSource === 'stall-watchdog')
|
|
842
|
+
deps.log(`[${name}] ${sessionLabel} startup diagnostic recovery requires operator attention; keeping supervisor alive`);
|
|
843
|
+
else if (interruptedForWake)
|
|
844
|
+
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
845
|
+
+ 'keeping temporary supervisor alive');
|
|
846
|
+
sessionStartupComplete = true;
|
|
847
|
+
reloadLoopConfig = async () => {
|
|
848
|
+
const nextRole = findRole(loadConfig(configPath), name);
|
|
849
|
+
const definitions = nextRole.loops ?? [];
|
|
850
|
+
const generation = JSON.stringify(definitions.map(loop => [
|
|
851
|
+
loop.name, loop.definitionHash, loop.promptHash,
|
|
852
|
+
]));
|
|
853
|
+
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
854
|
+
return { changed: false, loops: definitions.length };
|
|
855
|
+
if (!loopManager && definitions.length) {
|
|
856
|
+
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
857
|
+
now: deps.now,
|
|
858
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
859
|
+
clearTimer: timer => clearTimeout(timer),
|
|
860
|
+
log: deps.log,
|
|
861
|
+
});
|
|
862
|
+
control.setLoopManager(loopManager);
|
|
863
|
+
loopManager.start();
|
|
864
|
+
}
|
|
865
|
+
else {
|
|
866
|
+
loopManager?.reconcile(definitions);
|
|
867
|
+
}
|
|
868
|
+
loopGeneration = generation;
|
|
869
|
+
deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
|
|
870
|
+
return { changed: true, loops: definitions.length };
|
|
871
|
+
};
|
|
872
|
+
// Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
|
|
873
|
+
if (!temp)
|
|
874
|
+
control.setConfigReloader(reloadLoopConfig);
|
|
875
|
+
if (role.loops?.length) {
|
|
876
|
+
try {
|
|
877
|
+
loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
|
|
878
|
+
now: deps.now,
|
|
879
|
+
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
880
|
+
clearTimer: timer => clearTimeout(timer),
|
|
881
|
+
log: deps.log,
|
|
882
|
+
});
|
|
883
|
+
control.setLoopManager(loopManager);
|
|
884
|
+
loopManager.start();
|
|
885
|
+
}
|
|
886
|
+
catch (error) {
|
|
887
|
+
if (temp) {
|
|
888
|
+
monitor?.stop();
|
|
889
|
+
if (monitorLoop)
|
|
890
|
+
await monitorLoop;
|
|
891
|
+
await control.close();
|
|
892
|
+
ownerBinder?.release();
|
|
893
|
+
await agentSession.close();
|
|
894
|
+
unsubscribeRecovery?.();
|
|
895
|
+
throw new Error(`[${name}] configured temporary loop manager failed to start: `
|
|
896
|
+
+ `${error?.message ?? String(error)}`);
|
|
897
|
+
}
|
|
898
|
+
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
899
|
+
}
|
|
861
900
|
}
|
|
862
901
|
}
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
902
|
+
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
903
|
+
// The monitor loop lives exactly as long as the session: it starts once the
|
|
904
|
+
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
905
|
+
monitorLoop ??= monitor?.run(pid);
|
|
906
|
+
recoveryController = new RoleRecoveryController({
|
|
907
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
908
|
+
log: deps.log,
|
|
909
|
+
recoverAgent: async () => {
|
|
910
|
+
try {
|
|
911
|
+
const release = await managedService.runtime.admit();
|
|
912
|
+
release();
|
|
913
|
+
return { ok: true };
|
|
914
|
+
}
|
|
915
|
+
catch {
|
|
916
|
+
return { ok: false, reason: 'SUPERVISOR_IDENTITY_NOT_READY' };
|
|
917
|
+
}
|
|
918
|
+
},
|
|
919
|
+
recoverOwner: async (epoch) => {
|
|
920
|
+
if (!ownerChannel?.recover)
|
|
921
|
+
return { ok: true };
|
|
922
|
+
try {
|
|
923
|
+
await ownerChannel.recover(epoch);
|
|
924
|
+
return { ok: true };
|
|
925
|
+
}
|
|
926
|
+
catch (error) {
|
|
927
|
+
return { ok: false, reason: error instanceof Error
|
|
928
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
929
|
+
}
|
|
930
|
+
},
|
|
931
|
+
});
|
|
932
|
+
let chatIdle;
|
|
933
|
+
try {
|
|
934
|
+
if (temp && !role.roomMemberStartup)
|
|
935
|
+
chatIdle = new TemporaryChatIdle(dir, deps.now());
|
|
936
|
+
}
|
|
937
|
+
catch {
|
|
938
|
+
deps.log(`[${name}] automatic idle closure disabled: activity state is unreadable`);
|
|
939
|
+
}
|
|
940
|
+
const start = deps.now();
|
|
941
|
+
let nextLoopReloadAt = deps.now() + 30_000;
|
|
942
|
+
let nextIdentityPollAt = deps.now();
|
|
943
|
+
let nextDaemonProbeAt = deps.now();
|
|
944
|
+
let lastReloadError = '';
|
|
945
|
+
let identityObserved = false;
|
|
946
|
+
let identityAbsentSince;
|
|
947
|
+
let retirementReason;
|
|
948
|
+
let supervisorRecycleRequired = false;
|
|
949
|
+
while (sessionHandle.isAlive()) {
|
|
950
|
+
await deps.sleep(temp ? 500 : 2000);
|
|
951
|
+
const now = deps.now();
|
|
952
|
+
if (now >= nextDaemonProbeAt) {
|
|
953
|
+
nextDaemonProbeAt = now + 2_000;
|
|
954
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
955
|
+
if (observation.kind === 'lost')
|
|
956
|
+
recoveryController.noteLoss(observation.reason);
|
|
957
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
958
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
959
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
960
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
let idleExpired = false;
|
|
878
964
|
try {
|
|
879
|
-
|
|
880
|
-
return { ok: true };
|
|
965
|
+
idleExpired = chatIdle?.observe(sessionHandle.snapshot(), sessionHandle.conversationSnapshot?.().queueDepth ?? -1, now) ?? false;
|
|
881
966
|
}
|
|
882
|
-
catch
|
|
883
|
-
|
|
884
|
-
|
|
967
|
+
catch {
|
|
968
|
+
chatIdle = undefined;
|
|
969
|
+
deps.log(`[${name}] automatic idle closure disabled: activity state could not be saved`);
|
|
885
970
|
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
let nextIdentityPollAt = deps.now();
|
|
891
|
-
let nextDaemonProbeAt = deps.now();
|
|
892
|
-
let lastReloadError = '';
|
|
893
|
-
let identityObserved = false;
|
|
894
|
-
let identityAbsentSince;
|
|
895
|
-
let retirementReason;
|
|
896
|
-
let supervisorRecycleRequired = false;
|
|
897
|
-
while (sessionHandle.isAlive()) {
|
|
898
|
-
await deps.sleep(temp ? 500 : 2000);
|
|
899
|
-
const now = deps.now();
|
|
900
|
-
if (now >= nextDaemonProbeAt) {
|
|
901
|
-
nextDaemonProbeAt = now + 2_000;
|
|
902
|
-
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
903
|
-
if (observation.kind === 'lost')
|
|
904
|
-
recoveryController.noteLoss(observation.reason);
|
|
905
|
-
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
906
|
-
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
907
|
-
sessionStartedWithoutDaemonBaseline = false;
|
|
908
|
-
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
971
|
+
if (idleExpired && await arbiter.retireIfIdle((snapshot, queue) => chatIdle.observe(snapshot, queue, deps.now()))) {
|
|
972
|
+
retirementReason = 'idle-timeout';
|
|
973
|
+
sessionClosed = true;
|
|
974
|
+
break;
|
|
909
975
|
}
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
976
|
+
if (deps.shouldStop?.()) {
|
|
977
|
+
if (temp) {
|
|
978
|
+
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
979
|
+
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
980
|
+
}
|
|
981
|
+
await sessionHandle.close();
|
|
982
|
+
sessionClosed = true;
|
|
983
|
+
break;
|
|
915
984
|
}
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
985
|
+
if (temp && now >= nextIdentityPollAt) {
|
|
986
|
+
nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
|
|
987
|
+
const presence = await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env, deps.identityProbeDeps);
|
|
988
|
+
if (presence.state === 'present') {
|
|
989
|
+
identityObserved = true;
|
|
990
|
+
identityAbsentSince = undefined;
|
|
991
|
+
}
|
|
992
|
+
else if (presence.state === 'absent' && identityObserved) {
|
|
993
|
+
identityAbsentSince ??= now;
|
|
994
|
+
// Require a continuous, time-bounded run of authoritative absence.
|
|
995
|
+
// The first positive observation is the readiness gate: cold harness
|
|
996
|
+
// starts may spend minutes loading the harness and briefing before the
|
|
997
|
+
// agent creates/binds its identity, and absence before then is not a
|
|
998
|
+
// close event. After readiness, debounce a real disappearance.
|
|
999
|
+
if (now - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
1000
|
+
retirementReason = 'identity-closed';
|
|
1001
|
+
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
1002
|
+
await sessionHandle.close();
|
|
1003
|
+
sessionClosed = true;
|
|
1004
|
+
break;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
else if (presence.state === 'unknown')
|
|
1008
|
+
identityAbsentSince = undefined;
|
|
926
1009
|
}
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
sessionClosed = true;
|
|
939
|
-
break;
|
|
1010
|
+
if (reloadLoopConfig && now >= nextLoopReloadAt) {
|
|
1011
|
+
nextLoopReloadAt = now + 30_000;
|
|
1012
|
+
try {
|
|
1013
|
+
await reloadLoopConfig();
|
|
1014
|
+
lastReloadError = '';
|
|
1015
|
+
}
|
|
1016
|
+
catch (error) {
|
|
1017
|
+
const message = error?.message ?? String(error);
|
|
1018
|
+
if (message !== lastReloadError)
|
|
1019
|
+
deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
|
|
1020
|
+
lastReloadError = message;
|
|
940
1021
|
}
|
|
941
1022
|
}
|
|
942
|
-
else if (presence.state === 'unknown')
|
|
943
|
-
identityAbsentSince = undefined;
|
|
944
1023
|
}
|
|
945
|
-
if (
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
1024
|
+
if (loopManager) {
|
|
1025
|
+
control?.setLoopManager(undefined);
|
|
1026
|
+
await loopManager.stop();
|
|
1027
|
+
}
|
|
1028
|
+
// Let already-settled path operations publish their aggregate result before
|
|
1029
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
1030
|
+
await Promise.resolve();
|
|
1031
|
+
await Promise.resolve();
|
|
1032
|
+
recoveryController?.cancel();
|
|
1033
|
+
control?.setConfigReloader(undefined);
|
|
1034
|
+
// Close the authenticated control route before releasing the binder lease;
|
|
1035
|
+
// otherwise the predecessor can unlink the replacement's new socket.
|
|
1036
|
+
if (control) {
|
|
1037
|
+
control.setOwnerChannel(undefined);
|
|
1038
|
+
await control.close();
|
|
1039
|
+
control = undefined;
|
|
1040
|
+
}
|
|
1041
|
+
if (ownerChannel)
|
|
1042
|
+
await ownerChannel.close();
|
|
1043
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
1044
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
1045
|
+
supervisorRecycleRequired = true;
|
|
957
1046
|
}
|
|
958
|
-
}
|
|
959
|
-
if (loopManager) {
|
|
960
|
-
control?.setLoopManager(undefined);
|
|
961
|
-
await loopManager.stop();
|
|
962
|
-
}
|
|
963
|
-
// Let already-settled path operations publish their aggregate result before
|
|
964
|
-
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
965
|
-
await Promise.resolve();
|
|
966
|
-
await Promise.resolve();
|
|
967
|
-
recoveryController?.cancel();
|
|
968
|
-
control?.setConfigReloader(undefined);
|
|
969
|
-
// Close the authenticated control route before releasing the binder lease;
|
|
970
|
-
// otherwise the predecessor can unlink the replacement's new socket.
|
|
971
|
-
if (control) {
|
|
972
|
-
control.setOwnerChannel(undefined);
|
|
973
|
-
await control.close();
|
|
974
|
-
control = undefined;
|
|
975
|
-
}
|
|
976
|
-
if (ownerChannel)
|
|
977
|
-
await ownerChannel.close();
|
|
978
|
-
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
979
|
-
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
980
|
-
supervisorRecycleRequired = true;
|
|
981
|
-
}
|
|
982
|
-
else
|
|
983
|
-
ownerBinder?.release();
|
|
984
|
-
if (monitor) {
|
|
985
|
-
monitor.stop();
|
|
986
|
-
await monitorLoop;
|
|
987
|
-
}
|
|
988
|
-
unsubscribeRecovery?.();
|
|
989
|
-
if (agentSession && !sessionClosed)
|
|
990
|
-
await agentSession.close();
|
|
991
|
-
const elapsed = (deps.now() - start) / 1000;
|
|
992
|
-
// Establish what actually happened before deciding anything. Absence of a
|
|
993
|
-
// record is `unknown` — except when the console itself is gone, which is a
|
|
994
|
-
// different event with a different consequence.
|
|
995
|
-
const exitRecord = agentSession.exitResult()
|
|
996
|
-
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
997
|
-
writeFileSync(exitFile, JSON.stringify({
|
|
998
|
-
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
999
|
-
}) + '\n');
|
|
1000
|
-
let rotated = false;
|
|
1001
|
-
const rotate = (why) => {
|
|
1002
|
-
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1003
|
-
rmSync(bootedFile, { force: true });
|
|
1004
|
-
rotated = true;
|
|
1005
|
-
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1006
|
-
};
|
|
1007
|
-
if (!temp && deps.shouldStop?.())
|
|
1008
|
-
deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
|
|
1009
|
-
else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1010
|
-
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1011
|
-
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1012
|
-
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1013
|
-
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1014
|
-
// opens its circuit after the configured number of consecutive failures.
|
|
1015
|
-
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1016
|
-
`-> next start RESUMES context`);
|
|
1017
|
-
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1018
|
-
rotate(`clean exit (code 0)`);
|
|
1019
|
-
else if (exitRecord.class === 'session-destroyed')
|
|
1020
|
-
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1021
|
-
// would discard a live conversation for an operator action.
|
|
1022
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1023
|
-
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1024
|
-
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1025
|
-
// on every attempt would discard the conversation again and again while the
|
|
1026
|
-
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1027
|
-
if (opts.allowResumeRotation === false)
|
|
1028
|
-
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1029
|
-
`-> resume state was already discarded once; keeping it`);
|
|
1030
1047
|
else
|
|
1031
|
-
|
|
1048
|
+
ownerBinder?.release();
|
|
1049
|
+
if (monitor) {
|
|
1050
|
+
monitor.stop();
|
|
1051
|
+
await monitorLoop;
|
|
1052
|
+
}
|
|
1053
|
+
unsubscribeRecovery?.();
|
|
1054
|
+
if (agentSession && !sessionClosed)
|
|
1055
|
+
await agentSession.close();
|
|
1056
|
+
const elapsed = (deps.now() - start) / 1000;
|
|
1057
|
+
// Establish what actually happened before deciding anything. Absence of a
|
|
1058
|
+
// record is `unknown` — except when the console itself is gone, which is a
|
|
1059
|
+
// different event with a different consequence.
|
|
1060
|
+
const exitRecord = agentSession.exitResult()
|
|
1061
|
+
?? { version: 1, class: 'unknown', detail: 'the agent session stopped without reporting an exit' };
|
|
1062
|
+
writeFileSync(exitFile, JSON.stringify({
|
|
1063
|
+
...exitRecord, at: new Date(deps.now()).toISOString(), elapsedSecs: Number(elapsed.toFixed(1)),
|
|
1064
|
+
}) + '\n');
|
|
1065
|
+
let rotated = false;
|
|
1066
|
+
const rotate = (why) => {
|
|
1067
|
+
writeFileSync(sidFile, randomUUID() + '\n');
|
|
1068
|
+
rmSync(bootedFile, { force: true });
|
|
1069
|
+
rotated = true;
|
|
1070
|
+
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
1071
|
+
};
|
|
1072
|
+
if (!temp && deps.shouldStop?.())
|
|
1073
|
+
deps.log(`[${name}] supervisor stop requested -> next start RESUMES context`);
|
|
1074
|
+
else if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
1075
|
+
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
1076
|
+
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
1077
|
+
// is poisoned. Preserve the context even when the resumed generation hits
|
|
1078
|
+
// the same bound immediately; runSupervised still counts the fast exit and
|
|
1079
|
+
// opens its circuit after the configured number of consecutive failures.
|
|
1080
|
+
deps.log(`[${name}] forced cancellation recovery (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1081
|
+
`-> next start RESUMES context`);
|
|
1082
|
+
else if (exitRecord.class === 'clean' && adapter.exitPolicy.cleanExitIsFresh)
|
|
1083
|
+
rotate(`clean exit (code 0)`);
|
|
1084
|
+
else if (exitRecord.class === 'session-destroyed')
|
|
1085
|
+
// Someone tore the console down; the agent did not fail. Rotating here
|
|
1086
|
+
// would discard a live conversation for an operator action.
|
|
1087
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1088
|
+
else if (mode === 'resume' && elapsed < adapter.exitPolicy.fastFailSecs) {
|
|
1089
|
+
// Self-heal a poisoned resume — but only once per failure sequence. Rotating
|
|
1090
|
+
// on every attempt would discard the conversation again and again while the
|
|
1091
|
+
// real cause (a broken command, a missing binary) went unaddressed.
|
|
1092
|
+
if (opts.allowResumeRotation === false)
|
|
1093
|
+
deps.log(`[${name}] resume failed fast again (${elapsed.toFixed(0)}s, ${exitRecord.detail}) ` +
|
|
1094
|
+
`-> resume state was already discarded once; keeping it`);
|
|
1095
|
+
else
|
|
1096
|
+
rotate(`resume failed fast (${elapsed.toFixed(0)}s, ${exitRecord.detail})`);
|
|
1097
|
+
}
|
|
1098
|
+
else
|
|
1099
|
+
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1100
|
+
if (supervisorRecycleRequired)
|
|
1101
|
+
throw new SupervisorRecycleRequiredError();
|
|
1102
|
+
return {
|
|
1103
|
+
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1104
|
+
...(layoutRetiring ? { retirementReason: 'operator-stop' } : retirementReason ? { retirementReason } : {}),
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
finally {
|
|
1108
|
+
await managedService.close(layoutRetiring);
|
|
1032
1109
|
}
|
|
1033
|
-
else
|
|
1034
|
-
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
1035
|
-
if (supervisorRecycleRequired)
|
|
1036
|
-
throw new SupervisorRecycleRequiredError();
|
|
1037
|
-
return {
|
|
1038
|
-
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1039
|
-
...(retirementReason ? { retirementReason } : {}),
|
|
1040
|
-
};
|
|
1041
1110
|
}
|
|
1042
1111
|
/**
|
|
1043
1112
|
* The persistent supervisor for one permanent role: run child sessions in a
|
|
@@ -1205,6 +1274,8 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1205
1274
|
process.off('SIGINT', requestStop);
|
|
1206
1275
|
// An orderly shutdown clears the marker; an unhandled signal or OOM-kill
|
|
1207
1276
|
// leaves it so the successor can identify an abrupt termination.
|
|
1277
|
+
if (shouldStop())
|
|
1278
|
+
await deps.releaseAgentOurs(findRole(loadConfig(opts.configPath), name));
|
|
1208
1279
|
releaseSupervisorRun(dir);
|
|
1209
1280
|
}
|
|
1210
1281
|
}
|
|
@@ -1234,10 +1305,15 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1234
1305
|
let result;
|
|
1235
1306
|
let failure;
|
|
1236
1307
|
try {
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1308
|
+
for (let recoveryAttempt = 0;; recoveryAttempt++) {
|
|
1309
|
+
result = await attempt(name, { temp: true }, {
|
|
1310
|
+
...deps,
|
|
1311
|
+
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1312
|
+
});
|
|
1313
|
+
if (result.exit.class === 'clean' || result.retirementReason || signal || deps.shouldStop?.() || recoveryAttempt >= 2)
|
|
1314
|
+
break;
|
|
1315
|
+
await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(Math.min(5000, 1000 * (recoveryAttempt + 1)));
|
|
1316
|
+
}
|
|
1241
1317
|
}
|
|
1242
1318
|
catch (error) {
|
|
1243
1319
|
failure = error;
|
|
@@ -1245,23 +1321,27 @@ export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
|
1245
1321
|
finally {
|
|
1246
1322
|
process.off('SIGTERM', onTerm);
|
|
1247
1323
|
process.off('SIGINT', onInt);
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
?
|
|
1261
|
-
:
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1324
|
+
// A recycle replaces the supervisor process, not the logical temporary agent.
|
|
1325
|
+
if (!(failure instanceof SupervisorRecycleRequiredError) || signal || deps.shouldStop?.()) {
|
|
1326
|
+
const requested = requestedTempStopReason(dir);
|
|
1327
|
+
const reason = requested
|
|
1328
|
+
?? result?.retirementReason
|
|
1329
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1330
|
+
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1331
|
+
// A service-manager stop can make the child connection close before the
|
|
1332
|
+
// runner reaches its normal loop. That is still a successful requested
|
|
1333
|
+
// retirement, not a startup failure.
|
|
1334
|
+
const outcome = failure && !requested && !signal ? 'failed' : 'retired';
|
|
1335
|
+
const detail = failure
|
|
1336
|
+
? (failure instanceof Error ? failure.message : String(failure))
|
|
1337
|
+
: result
|
|
1338
|
+
? `${result.exit.detail}; elapsed=${result.elapsedSecs.toFixed(1)}s`
|
|
1339
|
+
: 'temporary supervisor ended without an attempt result';
|
|
1340
|
+
await (deps.releaseAgentOurs ?? releaseManagedAgent)(loadTempRole(name));
|
|
1341
|
+
const archived = archiveTempState(name, reason, outcome, detail);
|
|
1342
|
+
deps.log?.(`[${name}] temporary lifecycle ${outcome}: ${reason}`
|
|
1343
|
+
+ `${archived ? `; evidence archived at ${archived}` : '; state already archived'}`);
|
|
1344
|
+
}
|
|
1265
1345
|
}
|
|
1266
1346
|
if (failure)
|
|
1267
1347
|
throw failure;
|