@ours.network/fleet 1.1.0-nightly.2 → 1.1.0-nightly.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -55
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/capabilities.js +18 -6
- package/dist/application/fleet-query-service.js +20 -5
- package/dist/application/role-command-service.js +9 -0
- package/dist/application/role-creation-service.d.ts +6 -25
- package/dist/application/role-creation-service.js +34 -72
- package/dist/application/role-repository.d.ts +4 -2
- package/dist/application/role-repository.js +6 -3
- package/dist/application/session-control.d.ts +3 -1
- package/dist/application/session-control.js +4 -2
- package/dist/application/task-room-service.d.ts +58 -6
- package/dist/application/task-room-service.js +435 -79
- package/dist/briefing.js +74 -24
- package/dist/build-info.json +4 -4
- package/dist/canonical-json.d.ts +2 -0
- package/dist/canonical-json.js +10 -0
- package/dist/cli.js +241 -76
- package/dist/config-yaml.d.ts +1 -1
- package/dist/config.d.ts +75 -3
- package/dist/config.js +274 -34
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +3 -2
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +192 -66
- package/dist/doctor.d.ts +2 -0
- package/dist/doctor.js +54 -4
- package/dist/fleet-command-audit.d.ts +165 -0
- package/dist/fleet-command-audit.js +555 -0
- package/dist/fleet-proxy.d.ts +7 -2
- package/dist/fleet-proxy.js +30 -8
- package/dist/harness/agent-session.d.ts +5 -0
- package/dist/harness/claude-code-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +6 -1
- package/dist/harness/claude-code.js +2 -0
- package/dist/harness/codex-app-server-proxy.d.ts +40 -0
- package/dist/harness/codex-app-server-proxy.js +348 -14
- package/dist/harness/codex-session.d.ts +11 -2
- package/dist/harness/codex-session.js +22 -2
- package/dist/harness/codex.d.ts +5 -1
- package/dist/harness/codex.js +110 -10
- package/dist/harness/types.d.ts +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +4 -3
- package/dist/init-guidance.d.ts +1 -1
- package/dist/init-guidance.js +4 -3
- package/dist/init-wizard.d.ts +98 -0
- package/dist/init-wizard.js +629 -0
- package/dist/lifecycle-summary.d.ts +134 -0
- package/dist/lifecycle-summary.js +179 -0
- package/dist/loops/config.d.ts +10 -1
- package/dist/loops/config.js +38 -5
- package/dist/model-env.d.ts +2 -2
- package/dist/model-env.js +1 -1
- package/dist/model-recovery.d.ts +1 -1
- package/dist/ops.js +2 -17
- package/dist/owner-channel/channel.d.ts +58 -0
- package/dist/owner-channel/channel.js +321 -42
- package/dist/owner-channel/commands.d.ts +6 -1
- package/dist/owner-channel/commands.js +34 -19
- package/dist/owner-channel/state.d.ts +2 -0
- package/dist/owner-channel/state.js +8 -2
- package/dist/permissions.js +1 -1
- package/dist/preset-bootstrap.d.ts +10 -0
- package/dist/preset-bootstrap.js +85 -0
- package/dist/preset-migration.d.ts +42 -0
- package/dist/preset-migration.js +414 -0
- package/dist/reports/artifact.d.ts +10 -0
- package/dist/reports/artifact.js +32 -0
- package/dist/reports/index.d.ts +5 -0
- package/dist/reports/index.js +5 -0
- package/dist/reports/render.d.ts +6 -0
- package/dist/reports/render.js +240 -0
- package/dist/reports/service.d.ts +42 -0
- package/dist/reports/service.js +57 -0
- package/dist/reports/tasks.d.ts +52 -0
- package/dist/reports/tasks.js +177 -0
- package/dist/reports/types.d.ts +315 -0
- package/dist/reports/types.js +6 -0
- package/dist/resolved-plan.js +13 -1
- package/dist/rooms-tasks/cli.d.ts +2 -0
- package/dist/rooms-tasks/cli.js +514 -75
- package/dist/rooms-tasks/close.d.ts +9 -1
- package/dist/rooms-tasks/close.js +20 -4
- package/dist/rooms-tasks/config.js +16 -14
- package/dist/rooms-tasks/cowork-adapter.d.ts +1 -0
- package/dist/rooms-tasks/cowork-adapter.js +6 -0
- package/dist/rooms-tasks/deletion.d.ts +51 -0
- package/dist/rooms-tasks/deletion.js +217 -0
- package/dist/rooms-tasks/external-worker.d.ts +1 -0
- package/dist/rooms-tasks/external-worker.js +14 -9
- package/dist/rooms-tasks/launch-snapshot.d.ts +19 -0
- package/dist/rooms-tasks/launch-snapshot.js +181 -0
- package/dist/rooms-tasks/member-overrides.d.ts +30 -0
- package/dist/rooms-tasks/member-overrides.js +204 -0
- package/dist/rooms-tasks/provision.js +184 -49
- package/dist/rooms-tasks/room-state.d.ts +1 -0
- package/dist/rooms-tasks/room-state.js +12 -1
- package/dist/rooms-tasks/task-state.d.ts +115 -3
- package/dist/rooms-tasks/task-state.js +361 -11
- package/dist/rooms-tasks/templates.d.ts +4 -2
- package/dist/rooms-tasks/templates.js +42 -78
- package/dist/rooms-tasks/terminal.d.ts +3 -0
- package/dist/rooms-tasks/terminal.js +7 -3
- package/dist/rooms-tasks/types.d.ts +96 -17
- package/dist/rooms-tasks/types.js +14 -4
- package/dist/runner.d.ts +12 -2
- package/dist/runner.js +124 -43
- package/dist/session/acp.d.ts +9 -1
- package/dist/session/acp.js +35 -9
- package/dist/session/arbiter.d.ts +2 -1
- package/dist/session/arbiter.js +2 -0
- package/dist/session/codex-app-server-transport.d.ts +53 -0
- package/dist/session/codex-app-server-transport.js +184 -0
- package/dist/session/codex-app-server.d.ts +122 -0
- package/dist/session/codex-app-server.js +1031 -0
- package/dist/session/control.d.ts +14 -1
- package/dist/session/control.js +38 -0
- package/dist/session/conversation-types.d.ts +4 -4
- package/dist/session/types.d.ts +24 -6
- package/dist/session/types.js +28 -0
- package/dist/spawn.d.ts +15 -31
- package/dist/spawn.js +117 -237
- package/dist/temp-lifecycle.d.ts +1 -1
- package/dist/watchdog/config.d.ts +10 -7
- package/dist/watchdog/config.js +29 -19
- package/dist/watchdog/run.js +4 -13
- package/dist/web/fleet-config-service.d.ts +1 -0
- package/dist/web/fleet-config-service.js +27 -8
- package/dist/web/runtime.js +11 -6
- package/dist/web/server.js +28 -1
- package/dist/web/topology-promote.js +8 -1
- package/dist/web-app/assets/index-C0h3ALvs.js +10 -0
- package/dist/web-app/index.html +1 -1
- package/package.json +3 -2
- package/presets/brain-catalog.json +16 -0
- package/presets/fleet/agent_templates/Critic.yaml +4 -0
- package/presets/fleet/agent_templates/Developer.yaml +4 -0
- package/presets/fleet/agent_templates/LocalCoordinator.yaml +4 -0
- package/presets/fleet/agents/FleetCoordinator.yaml +3 -0
- package/presets/fleet/brains/claude-default.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-xhigh.yaml +5 -0
- package/presets/fleet/roles/Coordinator.yaml +30 -0
- package/presets/fleet/roles/Critic.yaml +19 -0
- package/presets/fleet/roles/Developer.yaml +19 -0
- package/presets/fleet/roles/LocalCoordinator.yaml +25 -0
- package/presets/fleet/room_templates/pair.yaml +10 -0
- package/presets/fleet/room_templates/single.yaml +7 -0
- package/presets/fleet/room_templates/team.yaml +12 -0
- package/presets/fleet.yaml +7 -0
- package/presets/manifest.json +6 -0
- package/dist/web-app/assets/index-BbE9EX6n.js +0 -10
|
@@ -2,10 +2,11 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { withFileLock } from '../atomic-file.js';
|
|
3
3
|
import { stateRoot } from '../paths.js';
|
|
4
4
|
import { deleteManagedRoom } from './close.js';
|
|
5
|
-
import { beginTaskTerminalIntent, finishTaskTerminalIntent, getTask, setTaskTerminalIntentError, } from './task-state.js';
|
|
5
|
+
import { assertNoPendingDeletion, beginTaskTerminalIntent, finishTaskTerminalIntent, getTask, setTaskTerminalIntentError, } from './task-state.js';
|
|
6
6
|
import { getRoomRecord } from './room-state.js';
|
|
7
|
-
const TASK_OPERATION_LOCK_STALE_MS = 10 * 60_000;
|
|
8
|
-
|
|
7
|
+
export const TASK_OPERATION_LOCK_STALE_MS = 10 * 60_000;
|
|
8
|
+
/** Common per-task operation lock serializing terminal, deletion, and recovery acceptance. */
|
|
9
|
+
export function taskOperationLockPath(taskId) {
|
|
9
10
|
return join(stateRoot(), 'locks', 'task-terminal', encodeURIComponent(taskId));
|
|
10
11
|
}
|
|
11
12
|
function errorText(error) {
|
|
@@ -24,6 +25,9 @@ export function acceptTaskTerminalIntent(input) {
|
|
|
24
25
|
export function settleTaskTerminalIntent(input) {
|
|
25
26
|
return withFileLock(taskOperationLockPath(input.taskId), async () => {
|
|
26
27
|
const current = getTask(input.taskId);
|
|
28
|
+
// Deletion supersedes a pending terminal intent: fail boundedly before any
|
|
29
|
+
// room side effect so the deletion worker owns all remaining cleanup.
|
|
30
|
+
assertNoPendingDeletion(current);
|
|
27
31
|
const intent = current.terminal_intent;
|
|
28
32
|
if (!intent)
|
|
29
33
|
throw new Error(`task ${input.taskId} has no accepted terminal intent`);
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* membership, and archive. Messenger Server owns owner presentation.
|
|
6
6
|
* Fleet stores only IDs, CIDs, orchestration state, and saga cursors.
|
|
7
7
|
*/
|
|
8
|
+
import type { AgentLaunchConfiguration } from '../lifecycle-summary.js';
|
|
8
9
|
export type TaskState = 'backlog' | 'provisioning' | 'active' | 'review' | 'done' | 'cancelled' | 'failed';
|
|
9
10
|
export declare const TASK_TERMINAL_STATES: readonly TaskState[];
|
|
10
11
|
export declare const TASK_CANCELLABLE_STATES: readonly TaskState[];
|
|
@@ -44,6 +45,47 @@ export interface TaskTerminalIntent {
|
|
|
44
45
|
first_failure?: string;
|
|
45
46
|
first_recovery_hint?: string;
|
|
46
47
|
}
|
|
48
|
+
/** Origin evidence for an accepted deletion; retained until the record is unlinked. */
|
|
49
|
+
export type TaskDeletionActor = {
|
|
50
|
+
kind: 'local_control';
|
|
51
|
+
surface: 'cli' | 'web';
|
|
52
|
+
} | {
|
|
53
|
+
kind: 'authenticated_owner';
|
|
54
|
+
surface: 'messenger';
|
|
55
|
+
cid: string;
|
|
56
|
+
};
|
|
57
|
+
export type TaskDeletionMemberPhase = 'pending' | 'stop_requested' | 'liveness_absent' | 'archive_secured' | 'identity_absent';
|
|
58
|
+
/**
|
|
59
|
+
* Durable retirement cursor for a managed member when no room orchestration
|
|
60
|
+
* record exists to carry the equivalent seat cursor.
|
|
61
|
+
*/
|
|
62
|
+
export interface TaskDeletionMemberCursor {
|
|
63
|
+
name: string;
|
|
64
|
+
identity_cid: string;
|
|
65
|
+
phase: TaskDeletionMemberPhase;
|
|
66
|
+
launch_id?: string;
|
|
67
|
+
archive_path?: string;
|
|
68
|
+
updated_at: string;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* First-wins durable deletion intent. There is no settled status: settlement
|
|
72
|
+
* ends by unlinking the task record, so physical absence is the completion
|
|
73
|
+
* evidence. While pending, the task is hidden from normal operation and every
|
|
74
|
+
* lifecycle mutation or room publication is rejected.
|
|
75
|
+
*/
|
|
76
|
+
export interface TaskDeletionIntent {
|
|
77
|
+
status: 'pending';
|
|
78
|
+
accepted_at: string;
|
|
79
|
+
actor: TaskDeletionActor;
|
|
80
|
+
room_id?: string;
|
|
81
|
+
/** Snapshot of managed members at acceptance; the missing-room retirement evidence. */
|
|
82
|
+
members: TaskDeletionMemberCursor[];
|
|
83
|
+
error?: string;
|
|
84
|
+
error_at?: string;
|
|
85
|
+
recovery_hint?: string;
|
|
86
|
+
first_failure?: string;
|
|
87
|
+
first_recovery_hint?: string;
|
|
88
|
+
}
|
|
47
89
|
export interface TaskRecord {
|
|
48
90
|
task_id: string;
|
|
49
91
|
/** Stable organizational list identifier. Missing legacy values mean `default`. */
|
|
@@ -56,6 +98,14 @@ export interface TaskRecord {
|
|
|
56
98
|
state: TaskState;
|
|
57
99
|
blocked?: TaskBlocked;
|
|
58
100
|
template?: TaskTemplateRef;
|
|
101
|
+
execution_plan?: {
|
|
102
|
+
schema_version: 1;
|
|
103
|
+
snapshot: TemplateSnapshot;
|
|
104
|
+
room_policy?: RoomLaunchPolicy;
|
|
105
|
+
overrides: Record<string, unknown>;
|
|
106
|
+
overrides_hash: string;
|
|
107
|
+
plan_hash: string;
|
|
108
|
+
};
|
|
59
109
|
no_room?: boolean;
|
|
60
110
|
room_id?: string;
|
|
61
111
|
room_identity_cid?: string;
|
|
@@ -67,6 +117,7 @@ export interface TaskRecord {
|
|
|
67
117
|
ended_at?: string;
|
|
68
118
|
outcome?: TaskOutcome;
|
|
69
119
|
terminal_intent?: TaskTerminalIntent;
|
|
120
|
+
deletion?: TaskDeletionIntent;
|
|
70
121
|
}
|
|
71
122
|
export interface TaskListRecord {
|
|
72
123
|
list_id: string;
|
|
@@ -100,6 +151,16 @@ export interface RoomMemberLaunchState {
|
|
|
100
151
|
/** Expected authenticated proxy caller while adopting a post-spawn crash. */
|
|
101
152
|
caller_role?: string;
|
|
102
153
|
mission_sha256?: string;
|
|
154
|
+
/** Redacted, deterministic effective Agent launch definition retained for retry inspection. */
|
|
155
|
+
agent_definition?: Record<string, unknown>;
|
|
156
|
+
agent_fingerprint?: string;
|
|
157
|
+
agent_template?: string;
|
|
158
|
+
agent_template_hash?: string;
|
|
159
|
+
/**
|
|
160
|
+
* Operator-facing launch configuration captured from the exact resolved
|
|
161
|
+
* launch state; the single source for later Task/Room lifecycle reports.
|
|
162
|
+
*/
|
|
163
|
+
presentation?: AgentLaunchConfiguration;
|
|
103
164
|
launch_id?: string;
|
|
104
165
|
updated_at: string;
|
|
105
166
|
error?: string;
|
|
@@ -172,6 +233,8 @@ export interface RoomOrchestrationRecord {
|
|
|
172
233
|
goal?: string;
|
|
173
234
|
task_id?: string;
|
|
174
235
|
template_snapshot?: TemplateSnapshot;
|
|
236
|
+
/** Resolved once at launch; legacy absence means non-anonymous. */
|
|
237
|
+
room_policy?: RoomLaunchPolicy;
|
|
175
238
|
saga: SagaCursor;
|
|
176
239
|
provisioning_detail?: ProvisioningDetail;
|
|
177
240
|
owner_seat_cid?: string;
|
|
@@ -189,35 +252,50 @@ export interface TemplateMemberSlot {
|
|
|
189
252
|
slot: string;
|
|
190
253
|
role: string;
|
|
191
254
|
count: number;
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
}
|
|
195
|
-
export interface TemplateRoleOverrides {
|
|
196
|
-
harness?: string;
|
|
197
|
-
model?: string;
|
|
198
|
-
model_chain?: string[];
|
|
199
|
-
permissions?: Record<string, unknown>;
|
|
200
|
-
isolation?: Record<string, unknown>;
|
|
201
|
-
cwd?: string;
|
|
202
|
-
env?: Record<string, string>;
|
|
203
|
-
persona?: string;
|
|
204
|
-
mission?: string;
|
|
255
|
+
/** Stable reference to an inert Agent Template, never a persistent Agent. */
|
|
256
|
+
agent_template: string;
|
|
205
257
|
}
|
|
206
258
|
export interface TemplateRoomConfig {
|
|
207
259
|
quiet_membership?: boolean;
|
|
208
260
|
anonymous?: boolean;
|
|
209
261
|
}
|
|
262
|
+
export interface RoomLaunchPolicy {
|
|
263
|
+
anonymous: boolean;
|
|
264
|
+
}
|
|
265
|
+
export declare const LEGACY_ROOM_LAUNCH_POLICY: Readonly<RoomLaunchPolicy>;
|
|
266
|
+
/** Validate durable policy before it can influence room creation or recovery. */
|
|
267
|
+
export declare function storedRoomLaunchPolicy(value: RoomLaunchPolicy | undefined): RoomLaunchPolicy;
|
|
210
268
|
export interface TemplateDefinition {
|
|
211
269
|
name: string;
|
|
212
270
|
version: number;
|
|
213
271
|
description: string;
|
|
214
|
-
|
|
272
|
+
/** Authoring provenance only; excluded from snapshots and semantic hashes. */
|
|
273
|
+
sourceFile?: string;
|
|
215
274
|
room?: TemplateRoomConfig;
|
|
216
275
|
contract?: string;
|
|
217
276
|
members: TemplateMemberSlot[];
|
|
218
277
|
}
|
|
219
|
-
export interface
|
|
278
|
+
export interface TemplateSnapshotMember extends TemplateMemberSlot {
|
|
279
|
+
/** Source of the effective temporary-loop policy, independent of its sealed value. */
|
|
280
|
+
loop_source?: 'agent-template' | 'cli' | 'omitted';
|
|
281
|
+
/** Secret-safe public projection; launch material lives in the sealed snapshot. */
|
|
282
|
+
agent_projection?: Record<string, unknown>;
|
|
283
|
+
agent_template_hash?: string;
|
|
284
|
+
/** Private sealed-snapshot lookup key; harmless provenance, not an identity. */
|
|
285
|
+
launch_definition_id?: string;
|
|
286
|
+
role_preset?: {
|
|
287
|
+
id: string;
|
|
288
|
+
hash: string;
|
|
289
|
+
};
|
|
290
|
+
brain_preset?: {
|
|
291
|
+
id: string;
|
|
292
|
+
hash: string;
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
export interface TemplateSnapshot extends Omit<TemplateDefinition, 'members'> {
|
|
296
|
+
members: TemplateSnapshotMember[];
|
|
220
297
|
content_hash: string;
|
|
298
|
+
launch_snapshot_hash?: string;
|
|
221
299
|
}
|
|
222
300
|
export interface RoomsOwnerConfig {
|
|
223
301
|
provider: string;
|
|
@@ -252,5 +330,6 @@ export declare const ROOMS_COWORK_KEYS: readonly ["config"];
|
|
|
252
330
|
export declare const ROOMS_DEFAULTS_KEYS: readonly ["template", "attach_owner", "close_when_task_done"];
|
|
253
331
|
export declare const TASKS_KEYS: readonly ["default_room_template", "create_mode", "close_room_on_done", "retain_completed_for"];
|
|
254
332
|
export declare const TEMPLATE_KEYS: readonly ["version", "description", "room", "contract", "members", "override_builtin"];
|
|
255
|
-
export declare const
|
|
256
|
-
|
|
333
|
+
export declare const TEMPLATE_ROOM_KEYS: readonly ["quiet_membership", "anonymous"];
|
|
334
|
+
/** `agent` is accepted only so validation can emit its actionable migration error. */
|
|
335
|
+
export declare const TEMPLATE_MEMBER_KEYS: readonly ["slot", "role", "count", "agent_template", "agent"];
|
|
@@ -7,6 +7,17 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export const TASK_TERMINAL_STATES = ['done', 'cancelled', 'failed'];
|
|
9
9
|
export const TASK_CANCELLABLE_STATES = ['backlog', 'provisioning', 'active', 'review'];
|
|
10
|
+
export const LEGACY_ROOM_LAUNCH_POLICY = Object.freeze({
|
|
11
|
+
anonymous: false,
|
|
12
|
+
});
|
|
13
|
+
/** Validate durable policy before it can influence room creation or recovery. */
|
|
14
|
+
export function storedRoomLaunchPolicy(value) {
|
|
15
|
+
if (value === undefined)
|
|
16
|
+
return { ...LEGACY_ROOM_LAUNCH_POLICY };
|
|
17
|
+
if (typeof value.anonymous !== 'boolean')
|
|
18
|
+
throw new Error('invalid durable Room launch policy; anonymous must be a boolean');
|
|
19
|
+
return { ...value };
|
|
20
|
+
}
|
|
10
21
|
// ── Validation keys ─────────────────────────────────────────────────────
|
|
11
22
|
export const ROOMS_KEYS = ['cowork', 'owner', 'defaults'];
|
|
12
23
|
export const ROOMS_OWNER_KEYS = ['provider', 'public_invite', 'public_invite_file', 'expected_cid', 'role'];
|
|
@@ -14,7 +25,6 @@ export const ROOMS_COWORK_KEYS = ['config'];
|
|
|
14
25
|
export const ROOMS_DEFAULTS_KEYS = ['template', 'attach_owner', 'close_when_task_done'];
|
|
15
26
|
export const TASKS_KEYS = ['default_room_template', 'create_mode', 'close_room_on_done', 'retain_completed_for'];
|
|
16
27
|
export const TEMPLATE_KEYS = ['version', 'description', 'room', 'contract', 'members', 'override_builtin'];
|
|
17
|
-
export const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
];
|
|
28
|
+
export const TEMPLATE_ROOM_KEYS = ['quiet_membership', 'anonymous'];
|
|
29
|
+
/** `agent` is accepted only so validation can emit its actionable migration error. */
|
|
30
|
+
export const TEMPLATE_MEMBER_KEYS = ['slot', 'role', 'count', 'agent_template', 'agent'];
|
package/dist/runner.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { type ResolvedRole } from './config.js';
|
|
2
2
|
import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
|
|
3
|
+
import { type DaemonGenerationProbe } from './daemon-recovery.js';
|
|
3
4
|
import { type Exec } from './exec.js';
|
|
4
5
|
import { RoleControlServer } from './session/control.js';
|
|
5
6
|
import type { AgentSession, ExitRecord, TurnResult } from './session/types.js';
|
|
6
7
|
import type { AgentSessionAdapter, AgentSessionStartOptions } from './harness/agent-session.js';
|
|
7
8
|
import { type OwnerChannelHandle, type OwnerChannelOptions } from './owner-channel/channel.js';
|
|
8
9
|
import { type OwnerBinderLease } from './owner-channel/binder.js';
|
|
10
|
+
import { ScheduledLoopManager, type ScheduledLoopManagerHandle } from './loops/manager.js';
|
|
9
11
|
export interface RunnerDeps {
|
|
10
12
|
exec: Exec;
|
|
11
13
|
cpuDelegated(): boolean;
|
|
@@ -15,6 +17,7 @@ export interface RunnerDeps {
|
|
|
15
17
|
log(line: string): void;
|
|
16
18
|
/** HTTP transport for the monitor's daemon long-poll (injectable for tests). */
|
|
17
19
|
fetch: FetchLike;
|
|
20
|
+
probeGeneration(env: NodeJS.ProcessEnv): Promise<DaemonGenerationProbe>;
|
|
18
21
|
/** Construct the supervisor mail monitor (injectable so tests stub it out). */
|
|
19
22
|
createMonitor(opts: MonitorOpts): MonitorHandle;
|
|
20
23
|
/** Construct trusted owner ingress (injectable for lifecycle tests). */
|
|
@@ -23,7 +26,9 @@ export interface RunnerDeps {
|
|
|
23
26
|
/** Neutral construction seam for deterministic runner lifecycle tests. */
|
|
24
27
|
startAgentSession(adapter: AgentSessionAdapter, options: AgentSessionStartOptions): Promise<AgentSession>;
|
|
25
28
|
/** Construct the authenticated role control route (injectable where sockets are unavailable). */
|
|
26
|
-
createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
|
|
29
|
+
createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setFleetAuditor' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
|
|
30
|
+
/** Construct scheduled-loop execution (injectable for fail-closed startup tests). */
|
|
31
|
+
createLoopManager(...args: ConstructorParameters<typeof ScheduledLoopManager>): ScheduledLoopManagerHandle;
|
|
27
32
|
/** Acquire the cross-process owner-channel binder lease before replacing the control socket. */
|
|
28
33
|
acquireOwnerBinder(stateDir: string, role: string, identity: string): Promise<OwnerBinderLease>;
|
|
29
34
|
/** Ask the still-authenticated predecessor to emit the fixed recovery notice. */
|
|
@@ -31,6 +36,11 @@ export interface RunnerDeps {
|
|
|
31
36
|
/** Lets a test (or a shutdown path) end the supervised restart loop. */
|
|
32
37
|
shouldStop?(): boolean;
|
|
33
38
|
}
|
|
39
|
+
export declare const SUPERVISOR_RECYCLE_REQUIRED = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
40
|
+
export declare class SupervisorRecycleRequiredError extends Error {
|
|
41
|
+
readonly code = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
42
|
+
constructor();
|
|
43
|
+
}
|
|
34
44
|
/** Environment injected only into the managed harness process. */
|
|
35
45
|
export declare function managedFleetProxyEnv(role: ResolvedRole, stateDir: string): Record<string, string>;
|
|
36
46
|
/**
|
|
@@ -176,4 +186,4 @@ export declare function runSupervised(name: string, opts?: {
|
|
|
176
186
|
allowResumeRotation?: boolean;
|
|
177
187
|
}, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
|
|
178
188
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
179
|
-
export declare function runTemp(name: string, deps?: Partial<RunnerDeps
|
|
189
|
+
export declare function runTemp(name: string, deps?: Partial<RunnerDeps>, attempt?: typeof runOnce): Promise<void>;
|
package/dist/runner.js
CHANGED
|
@@ -6,13 +6,15 @@ import { agentDir, stateRoot } from './paths.js';
|
|
|
6
6
|
import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolvePermissions, } from './config.js';
|
|
7
7
|
import { getAdapter } from './harness/registry.js';
|
|
8
8
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
|
+
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
+
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
9
11
|
import { realExec } from './exec.js';
|
|
10
12
|
import { resolveIsolation } from './isolation/policy.js';
|
|
11
13
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
12
14
|
import { resourceArgs, cpuControllerDelegated } from './isolation/resources.js';
|
|
13
15
|
import { resolveLaunchRuntime } from './isolation/runtime.js';
|
|
14
16
|
import { controlRequest, RoleControlServer } from './session/control.js';
|
|
15
|
-
import { ACP_CANCEL_DEADLINE_EXCEEDED, classifyShellStatus } from './session/types.js';
|
|
17
|
+
import { ACP_CANCEL_DEADLINE_EXCEEDED, CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED, classifyShellStatus, } from './session/types.js';
|
|
16
18
|
import { effectiveModelForRole, modelRecoveryHeld, reconcileModelRecovery, recordModelFailure, classifyFailureText, } from './model-recovery.js';
|
|
17
19
|
import { rotateWorklog } from './worklog.js';
|
|
18
20
|
import { reconcilePermanentRoleIdentities } from './creation.js';
|
|
@@ -24,6 +26,14 @@ import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, } from './fleet-prox
|
|
|
24
26
|
import { effectivePermissionMode } from './permissions.js';
|
|
25
27
|
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
26
28
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
29
|
+
export const SUPERVISOR_RECYCLE_REQUIRED = 'OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED';
|
|
30
|
+
export class SupervisorRecycleRequiredError extends Error {
|
|
31
|
+
code = SUPERVISOR_RECYCLE_REQUIRED;
|
|
32
|
+
constructor() {
|
|
33
|
+
super('owner channel requires a fresh supervisor process after unproven client quiescence');
|
|
34
|
+
this.name = 'SupervisorRecycleRequiredError';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
27
37
|
const defaultDeps = () => ({
|
|
28
38
|
exec: realExec,
|
|
29
39
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
@@ -38,10 +48,12 @@ const defaultDeps = () => ({
|
|
|
38
48
|
now: () => Date.now(),
|
|
39
49
|
log: line => process.stderr.write(line + '\n'),
|
|
40
50
|
fetch: (url, init) => globalThis.fetch(url, init),
|
|
51
|
+
probeGeneration: env => probeDaemonGeneration((url, init) => globalThis.fetch(url, init), env),
|
|
41
52
|
createMonitor: opts => createMonitor(opts),
|
|
42
53
|
createOwnerChannel: opts => new OwnerChannel(opts),
|
|
43
54
|
startAgentSession: (adapter, options) => adapter.start(options),
|
|
44
55
|
createControlServer: (stateDir, session, log) => new RoleControlServer(stateDir, session, log),
|
|
56
|
+
createLoopManager: (...args) => new ScheduledLoopManager(...args),
|
|
45
57
|
acquireOwnerBinder: (stateDir, role, identity) => acquireOwnerBinderLease(stateDir, role, identity),
|
|
46
58
|
reportOwnerStartupFailure: async (stateDir) => {
|
|
47
59
|
const response = await controlRequest(stateDir, {
|
|
@@ -453,6 +465,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
453
465
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
454
466
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
455
467
|
const sessionBackend = role.session ?? 'acp';
|
|
468
|
+
const sessionLabel = sessionBackend === 'acp' ? 'ACP' : 'Codex app-server';
|
|
456
469
|
let launch = adapter.agentSession.prepareLaunch(role, prep);
|
|
457
470
|
// Isolation is additive: only roles that declare `isolation:` are wrapped.
|
|
458
471
|
let wrappedArgv = launch.argv;
|
|
@@ -532,6 +545,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
532
545
|
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
533
546
|
deps: resolvedMonitorDeps,
|
|
534
547
|
}) : null;
|
|
548
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
549
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
550
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
535
551
|
if (monitor)
|
|
536
552
|
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
537
553
|
rmSync(exitFile, { force: true });
|
|
@@ -540,13 +556,13 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
540
556
|
let control;
|
|
541
557
|
let unsubscribeRecovery;
|
|
542
558
|
let monitorLoop;
|
|
543
|
-
let
|
|
559
|
+
let sessionStartupComplete = false;
|
|
544
560
|
let sessionClosed = false;
|
|
545
561
|
let ownerChannel;
|
|
546
562
|
let ownerBinder;
|
|
547
|
-
const pendingFleetSpawnNotices = [];
|
|
548
563
|
let loopManager;
|
|
549
564
|
let arbiter;
|
|
565
|
+
let recoveryController;
|
|
550
566
|
let reloadLoopConfig;
|
|
551
567
|
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
552
568
|
loop.name, loop.definitionHash, loop.promptHash,
|
|
@@ -571,7 +587,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
571
587
|
unsubscribeRecovery = agentSession.subscribe(event => {
|
|
572
588
|
if (event.kind !== 'error' || !event.text)
|
|
573
589
|
return;
|
|
574
|
-
const evidence = classifyFailureText(event.text,
|
|
590
|
+
const evidence = classifyFailureText(event.text, sessionBackend, new Date(deps.now()).toISOString());
|
|
575
591
|
if (evidence)
|
|
576
592
|
resolvedMonitorDeps.onFailureEvidence?.(evidence);
|
|
577
593
|
});
|
|
@@ -606,23 +622,22 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
606
622
|
unsubscribeRecovery?.();
|
|
607
623
|
throw error;
|
|
608
624
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
625
|
+
if (!role.roomMemberStartup)
|
|
626
|
+
control.setFleetSpawner(async (requested) => {
|
|
627
|
+
const event = await executeManagedSpawn(role, configPath, requested, deps.log);
|
|
628
|
+
// Room-member launches are collected with their Task/Room transaction so
|
|
629
|
+
// causal delivery is Task → Room → Agents → Room active → Task active.
|
|
630
|
+
if (!requested.roomMemberStartup && role.owner_channel && ownerChannel?.notifyFleetSpawn) {
|
|
631
|
+
try {
|
|
632
|
+
await ownerChannel.notifyFleetSpawn(event);
|
|
633
|
+
}
|
|
634
|
+
catch (error) {
|
|
635
|
+
deps.log(`[${name}] Agent lifecycle notice delivery failed: `
|
|
636
|
+
+ `${error?.message ?? String(error)}`);
|
|
637
|
+
}
|
|
620
638
|
}
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
pendingFleetSpawnNotices.push(event);
|
|
624
|
-
return event;
|
|
625
|
-
});
|
|
639
|
+
return event;
|
|
640
|
+
});
|
|
626
641
|
resolvedMonitorDeps.delivery = {
|
|
627
642
|
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
628
643
|
// refusal or a cancellation reached the agent and was not acted on, so
|
|
@@ -633,7 +648,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
633
648
|
// steer into the live turn instead; after it completes, honor the
|
|
634
649
|
// configured interrupt policy normally.
|
|
635
650
|
const policy = options?.interrupt;
|
|
636
|
-
const interrupt = policy === true &&
|
|
651
|
+
const interrupt = policy === true && sessionStartupComplete;
|
|
637
652
|
const promptOptions = {
|
|
638
653
|
interrupt, steer: true,
|
|
639
654
|
...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
|
|
@@ -641,7 +656,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
641
656
|
};
|
|
642
657
|
// Startup is already a protected boundary: as with immediate mode,
|
|
643
658
|
// steer rather than waiting on/cancelling the runner-owned first turn.
|
|
644
|
-
const result = policy === 'after_tool' &&
|
|
659
|
+
const result = policy === 'after_tool' && sessionStartupComplete
|
|
645
660
|
? await arbiter.submitPromptAfterTool(text, promptOptions)
|
|
646
661
|
: await arbiter.submitPrompt(text, promptOptions);
|
|
647
662
|
const steered = result.accepted
|
|
@@ -702,20 +717,20 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
702
717
|
exit: {
|
|
703
718
|
version: 1,
|
|
704
719
|
class: 'program-exit',
|
|
705
|
-
detail:
|
|
720
|
+
detail: `${sessionLabel} startup triggered model recovery (${modelRecovery})`,
|
|
706
721
|
},
|
|
707
722
|
rotated: false,
|
|
708
723
|
mode,
|
|
709
724
|
modelRecovery,
|
|
710
725
|
};
|
|
711
726
|
}
|
|
712
|
-
throw new Error(`[${name}]
|
|
727
|
+
throw new Error(`[${name}] ${sessionLabel} startup prompt ${started.outcome}` +
|
|
713
728
|
`${started.detail ? `: ${started.detail}` : ''}`);
|
|
714
729
|
}
|
|
715
730
|
if (interruptedForWake)
|
|
716
|
-
deps.log(`[${name}]
|
|
731
|
+
deps.log(`[${name}] ${sessionLabel} startup prompt cancelled by ${started.cancellationSource}; `
|
|
717
732
|
+ 'keeping temporary supervisor alive');
|
|
718
|
-
|
|
733
|
+
sessionStartupComplete = true;
|
|
719
734
|
if (role.owner_channel) {
|
|
720
735
|
ownerChannel = deps.createOwnerChannel({
|
|
721
736
|
role: name,
|
|
@@ -743,18 +758,14 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
743
758
|
throw new Error(`[${name}] owner channel failed to start: `
|
|
744
759
|
+ `${error?.message ?? String(error)}`);
|
|
745
760
|
}
|
|
746
|
-
for (const event of pendingFleetSpawnNotices.splice(0)) {
|
|
747
|
-
try {
|
|
748
|
-
await ownerChannel.notifyFleetSpawn?.(event);
|
|
749
|
-
}
|
|
750
|
-
catch (error) {
|
|
751
|
-
deps.log(`[${name}] deferred spawned-agent owner notice failed: `
|
|
752
|
-
+ `${error?.message ?? String(error)}`);
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
761
|
}
|
|
756
|
-
if (ownerChannel)
|
|
762
|
+
if (ownerChannel) {
|
|
757
763
|
control.setOwnerChannel(ownerChannel);
|
|
764
|
+
control.setFleetAuditor({
|
|
765
|
+
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
766
|
+
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
767
|
+
});
|
|
768
|
+
}
|
|
758
769
|
reloadLoopConfig = async () => {
|
|
759
770
|
const nextRole = findRole(loadConfig(configPath), name);
|
|
760
771
|
const definitions = nextRole.loops ?? [];
|
|
@@ -764,7 +775,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
764
775
|
if (generation === loopGeneration && (loopManager || !definitions.length))
|
|
765
776
|
return { changed: false, loops: definitions.length };
|
|
766
777
|
if (!loopManager && definitions.length) {
|
|
767
|
-
loopManager =
|
|
778
|
+
loopManager = deps.createLoopManager(name, definitions, dir, arbiter, {
|
|
768
779
|
now: deps.now,
|
|
769
780
|
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
770
781
|
clearTimer: timer => clearTimeout(timer),
|
|
@@ -780,10 +791,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
780
791
|
deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
|
|
781
792
|
return { changed: true, loops: definitions.length };
|
|
782
793
|
};
|
|
783
|
-
|
|
794
|
+
// Temporary agents are immutable launch snapshots: never re-resolve mutable Fleet YAML.
|
|
795
|
+
if (!temp)
|
|
796
|
+
control.setConfigReloader(reloadLoopConfig);
|
|
784
797
|
if (role.loops?.length) {
|
|
785
798
|
try {
|
|
786
|
-
loopManager =
|
|
799
|
+
loopManager = deps.createLoopManager(name, role.loops, dir, arbiter, {
|
|
787
800
|
now: deps.now,
|
|
788
801
|
setTimer: (callback, ms) => setTimeout(callback, ms),
|
|
789
802
|
clearTimer: timer => clearTimeout(timer),
|
|
@@ -793,6 +806,17 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
793
806
|
loopManager.start();
|
|
794
807
|
}
|
|
795
808
|
catch (error) {
|
|
809
|
+
if (temp) {
|
|
810
|
+
monitor?.stop();
|
|
811
|
+
if (monitorLoop)
|
|
812
|
+
await monitorLoop;
|
|
813
|
+
await control.close();
|
|
814
|
+
ownerBinder?.release();
|
|
815
|
+
await agentSession.close();
|
|
816
|
+
unsubscribeRecovery?.();
|
|
817
|
+
throw new Error(`[${name}] configured temporary loop manager failed to start: `
|
|
818
|
+
+ `${error?.message ?? String(error)}`);
|
|
819
|
+
}
|
|
796
820
|
deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
|
|
797
821
|
}
|
|
798
822
|
}
|
|
@@ -801,16 +825,49 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
801
825
|
// The monitor loop lives exactly as long as the session: it starts once the
|
|
802
826
|
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
803
827
|
monitorLoop ??= monitor?.run(pid);
|
|
828
|
+
recoveryController = new RoleRecoveryController({
|
|
829
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
830
|
+
log: deps.log,
|
|
831
|
+
recoverAgent: async () => {
|
|
832
|
+
const evidence = await recoverAgentIdentity(arbiter, role.identity);
|
|
833
|
+
return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
|
|
834
|
+
},
|
|
835
|
+
recoverOwner: async (epoch) => {
|
|
836
|
+
if (!ownerChannel?.recover)
|
|
837
|
+
return { ok: true };
|
|
838
|
+
try {
|
|
839
|
+
await ownerChannel.recover(epoch);
|
|
840
|
+
return { ok: true };
|
|
841
|
+
}
|
|
842
|
+
catch (error) {
|
|
843
|
+
return { ok: false, reason: error instanceof Error
|
|
844
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
845
|
+
}
|
|
846
|
+
},
|
|
847
|
+
});
|
|
804
848
|
const start = deps.now();
|
|
805
849
|
let nextLoopReloadAt = deps.now() + 30_000;
|
|
806
850
|
let nextIdentityPollAt = deps.now();
|
|
851
|
+
let nextDaemonProbeAt = deps.now();
|
|
807
852
|
let lastReloadError = '';
|
|
808
853
|
let identityObserved = false;
|
|
809
854
|
let identityAbsentSince;
|
|
810
855
|
let retirementReason;
|
|
856
|
+
let supervisorRecycleRequired = false;
|
|
811
857
|
while (sessionHandle.isAlive()) {
|
|
812
858
|
await deps.sleep(temp ? 500 : 2000);
|
|
813
859
|
const now = deps.now();
|
|
860
|
+
if (now >= nextDaemonProbeAt) {
|
|
861
|
+
nextDaemonProbeAt = now + 2_000;
|
|
862
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
863
|
+
if (observation.kind === 'lost')
|
|
864
|
+
recoveryController.noteLoss(observation.reason);
|
|
865
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
866
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
867
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
868
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
869
|
+
}
|
|
870
|
+
}
|
|
814
871
|
if (temp && deps.shouldStop?.()) {
|
|
815
872
|
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
816
873
|
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
@@ -861,6 +918,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
861
918
|
control?.setLoopManager(undefined);
|
|
862
919
|
await loopManager.stop();
|
|
863
920
|
}
|
|
921
|
+
// Let already-settled path operations publish their aggregate result before
|
|
922
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
923
|
+
await Promise.resolve();
|
|
924
|
+
await Promise.resolve();
|
|
925
|
+
recoveryController?.cancel();
|
|
864
926
|
control?.setConfigReloader(undefined);
|
|
865
927
|
// Close the authenticated control route before releasing the binder lease;
|
|
866
928
|
// otherwise the predecessor can unlink the replacement's new socket.
|
|
@@ -871,7 +933,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
871
933
|
}
|
|
872
934
|
if (ownerChannel)
|
|
873
935
|
await ownerChannel.close();
|
|
874
|
-
|
|
936
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
937
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
938
|
+
supervisorRecycleRequired = true;
|
|
939
|
+
}
|
|
940
|
+
else
|
|
941
|
+
ownerBinder?.release();
|
|
875
942
|
if (monitor) {
|
|
876
943
|
monitor.stop();
|
|
877
944
|
await monitorLoop;
|
|
@@ -895,7 +962,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
895
962
|
rotated = true;
|
|
896
963
|
deps.log(`[${name}] ${why} -> rotated session-id; next start is FRESH`);
|
|
897
964
|
};
|
|
898
|
-
if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
965
|
+
if (exitRecord.detail.includes(ACP_CANCEL_DEADLINE_EXCEEDED)
|
|
966
|
+
|| exitRecord.detail.includes(CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED))
|
|
899
967
|
// This is a deliberate adapter reclamation, not evidence that resume state
|
|
900
968
|
// is poisoned. Preserve the context even when the resumed generation hits
|
|
901
969
|
// the same bound immediately; runSupervised still counts the fast exit and
|
|
@@ -920,6 +988,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
920
988
|
}
|
|
921
989
|
else
|
|
922
990
|
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
991
|
+
if (supervisorRecycleRequired)
|
|
992
|
+
throw new SupervisorRecycleRequiredError();
|
|
923
993
|
return {
|
|
924
994
|
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
925
995
|
...(retirementReason ? { retirementReason } : {}),
|
|
@@ -996,6 +1066,16 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
996
1066
|
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
997
1067
|
}
|
|
998
1068
|
catch (e) {
|
|
1069
|
+
if (e instanceof SupervisorRecycleRequiredError) {
|
|
1070
|
+
writeRestartLedger(dir, {
|
|
1071
|
+
...ledger,
|
|
1072
|
+
lastReason: SUPERVISOR_RECYCLE_REQUIRED,
|
|
1073
|
+
nextDelayMs: 0,
|
|
1074
|
+
updatedAt: stamp(),
|
|
1075
|
+
});
|
|
1076
|
+
deps.log(`[${name}] supervisor recycle required: ${SUPERVISOR_RECYCLE_REQUIRED}`);
|
|
1077
|
+
throw e;
|
|
1078
|
+
}
|
|
999
1079
|
// A session that could not even start is an immediate failure like any
|
|
1000
1080
|
// other; it must count, or an unstartable role loops forever.
|
|
1001
1081
|
result = {
|
|
@@ -1089,7 +1169,7 @@ function fastFailSecsFor(name, configPath) {
|
|
|
1089
1169
|
}
|
|
1090
1170
|
}
|
|
1091
1171
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
1092
|
-
export async function runTemp(name, deps = {}) {
|
|
1172
|
+
export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
1093
1173
|
const dir = agentDir(name, true);
|
|
1094
1174
|
await markTempSupervisorActive(dir);
|
|
1095
1175
|
let signal;
|
|
@@ -1100,7 +1180,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1100
1180
|
let result;
|
|
1101
1181
|
let failure;
|
|
1102
1182
|
try {
|
|
1103
|
-
result = await
|
|
1183
|
+
result = await attempt(name, { temp: true }, {
|
|
1104
1184
|
...deps,
|
|
1105
1185
|
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1106
1186
|
});
|
|
@@ -1114,6 +1194,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1114
1194
|
const requested = requestedTempStopReason(dir);
|
|
1115
1195
|
const reason = requested
|
|
1116
1196
|
?? result?.retirementReason
|
|
1197
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1117
1198
|
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1118
1199
|
// A service-manager stop can make the child connection close before the
|
|
1119
1200
|
// runner reaches its normal loop. That is still a successful requested
|