@ours.network/fleet 1.1.0-nightly.5 → 1.1.0-nightly.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { readFileSync } from 'node:fs';
2
+ import { createHash } from 'node:crypto';
2
3
  import { loadConfig, ConfigError } from '../config.js';
3
4
  import { provisionMembers, getBinPath } from './provision.js';
4
5
  import { launchFleetWorker } from './external-worker.js';
@@ -9,6 +10,7 @@ import { createCoworkAdapter, CoworkProtocolError } from './cowork-adapter.js';
9
10
  import { markdownCode, markdownProse, renderMarkdownFailure, renderMarkdownList, renderMarkdownResult, roomStatus, taskStatus, } from './markdown.js';
10
11
  import { TaskRoomApplicationError, TaskRoomApplicationService } from '../application/task-room-service.js';
11
12
  import { TaskListError } from './task-lists.js';
13
+ import { recordFleetAuditFailure, recordFleetAuditPresentation, recordFleetAuditResource, } from '../fleet-command-audit.js';
12
14
  class TaskRoomPublicError extends Error {
13
15
  code;
14
16
  fields;
@@ -89,8 +91,29 @@ function die(e) {
89
91
  const msg = e instanceof TaskRoomPublicError
90
92
  ? taskRoomPublicFailure(e).legacy : e instanceof Error ? e.message : String(e);
91
93
  process.stderr.write(`error: ${msg}\n`);
94
+ recordFleetAuditFailure(classifyTaskRoomFailure(e));
92
95
  process.exit(1);
93
96
  }
97
+ function classifyTaskRoomFailure(e) {
98
+ if (e instanceof TaskRoomApplicationError)
99
+ e = taskRoomPublicError(e.code, e.fields);
100
+ if (e instanceof TaskRoomPublicError || e instanceof TaskListError || e instanceof ConfigError)
101
+ return { class: 'validation', effect: 'not_started' };
102
+ if (e instanceof TaskStateError) {
103
+ const known = /^task not found: [A-Za-z0-9_-]{1,128}$/u.test(e.message)
104
+ || e.message === 'invalid task ID: expected the canonical 17-character lowercase ID'
105
+ || isKnownTaskStateMessage(e.message);
106
+ return known ? { class: 'validation', effect: 'not_started' }
107
+ : { class: 'runtime', effect: 'unknown' };
108
+ }
109
+ if (e instanceof RoomStateError) {
110
+ const known = /^room not found: [A-Za-z0-9_-]{1,128}$/u.test(e.message)
111
+ || isKnownRoomStateMessage(e.message);
112
+ return known ? { class: 'validation', effect: 'not_started' }
113
+ : { class: 'runtime', effect: 'unknown' };
114
+ }
115
+ return { class: 'runtime', effect: 'unknown' };
116
+ }
94
117
  const TASK_STATE_WORD = '(?:backlog|provisioning|active|review|done|cancelled|failed)';
95
118
  const SAFE_ID_WORD = '[A-Za-z0-9_-]{1,128}';
96
119
  function isKnownTaskStateMessage(message) {
@@ -127,6 +150,7 @@ function dieTaskRoom(e) {
127
150
  action: failure.action,
128
151
  });
129
152
  process.stderr.write(`${output}\n`);
153
+ recordFleetAuditFailure({ class: 'validation', effect: 'not_started' });
130
154
  process.exit(1);
131
155
  }
132
156
  if (e instanceof TaskListError) {
@@ -142,6 +166,7 @@ function dieTaskRoom(e) {
142
166
  : 'Use a valid NFC-normalized name of at most 64 Unicode code points.',
143
167
  });
144
168
  process.stderr.write(`${output}\n`);
169
+ recordFleetAuditFailure({ class: 'validation', effect: 'not_started' });
145
170
  process.exit(1);
146
171
  }
147
172
  const taskMissing = e instanceof TaskStateError
@@ -167,6 +192,7 @@ function dieTaskRoom(e) {
167
192
  : 'Retry once; if it repeats, run ours-fleet doctor and inspect the role logs.',
168
193
  });
169
194
  process.stderr.write(`${output}\n`);
195
+ recordFleetAuditFailure(classifyTaskRoomFailure(e));
170
196
  process.exit(1);
171
197
  }
172
198
  const taskListMarkdown = (tasks, title = 'Tasks') => renderMarkdownList({
@@ -175,23 +201,61 @@ const taskListMarkdown = (tasks, title = 'Tasks') => renderMarkdownList({
175
201
  + ` — List ${markdownCode(t.list_name ?? 'default')}`
176
202
  + (t.blocked ? ` — 🚧 Blocked: ${markdownProse(t.blocked.reason)}` : '')),
177
203
  });
178
- const taskActionMarkdown = (title, task, fields = []) => renderMarkdownResult({
179
- icon: '📋', title,
180
- fields: [
181
- { label: 'ID', value: task.task_id, kind: 'code' },
182
- { label: 'Status', value: taskStatus(task.state), kind: 'markdown' },
183
- { label: 'List', value: task.list_name ?? 'default', kind: 'code' },
184
- ...fields,
185
- ],
186
- });
187
- const roomActionMarkdown = (title, room, fields = []) => renderMarkdownResult({
188
- icon: '🏠', title,
189
- fields: [
190
- { label: 'ID', value: room.room_id, kind: 'code' },
191
- { label: 'Status', value: roomStatus(room.state), kind: 'markdown' },
192
- ...fields,
193
- ],
194
- });
204
+ const taskActionMarkdown = (title, task, fields = []) => {
205
+ recordFleetAuditResource('task', task.task_id);
206
+ if (task.room_id)
207
+ recordFleetAuditResource('room', task.room_id);
208
+ return renderMarkdownResult({
209
+ icon: '📋', title,
210
+ fields: [
211
+ { label: 'ID', value: task.task_id, kind: 'code' },
212
+ { label: 'Status', value: taskStatus(task.state), kind: 'markdown' },
213
+ { label: 'List', value: task.list_name ?? 'default', kind: 'code' },
214
+ ...fields,
215
+ ],
216
+ });
217
+ };
218
+ function safeBrainSummary(definition) {
219
+ const brain = definition?.brain;
220
+ if (!brain || typeof brain !== 'object' || Array.isArray(brain))
221
+ return 'unresolved';
222
+ const selection = brain;
223
+ if (typeof selection.ref === 'string')
224
+ return `ref:${selection.ref}`;
225
+ if (!Object.hasOwn(selection, 'inline'))
226
+ return 'unresolved';
227
+ const fingerprint = createHash('sha256').update(JSON.stringify(selection.inline)).digest('hex').slice(0, 16);
228
+ return `inline:sha256:${fingerprint}`;
229
+ }
230
+ function auditTask(operation, task, previousState, newState = task.state) {
231
+ recordFleetAuditResource('task', task.task_id);
232
+ if (task.room_id)
233
+ recordFleetAuditResource('room', task.room_id);
234
+ const room = task.room_id ? getRoomRecord(task.room_id) : undefined;
235
+ const brains = new Map(room?.member_seats.map(seat => [seat.role_name, safeBrainSummary(seat.launch?.agent_definition)]) ?? []);
236
+ recordFleetAuditPresentation({ kind: 'task', operation, id: task.task_id, title: task.title,
237
+ previousState, newState,
238
+ template: task.template ? `${task.template.name}@${task.template.version}` : undefined,
239
+ roomId: task.room_id, agents: task.member_roles.map(member => ({ name: member.name,
240
+ brain: brains.get(member.name) ?? 'unresolved', role: member.cowork_role })) });
241
+ }
242
+ function auditRoom(operation, room, previousState, newState = room.state) {
243
+ recordFleetAuditResource('room', room.room_id);
244
+ recordFleetAuditPresentation({ kind: 'room', operation, id: room.room_id, previousState, newState,
245
+ template: room.template_snapshot ? `${room.template_snapshot.name}@${room.template_snapshot.version}` : undefined,
246
+ participants: room.member_seats.map(member => ({ name: member.role_name, role: member.cowork_role })) });
247
+ }
248
+ const roomActionMarkdown = (title, room, fields = []) => {
249
+ recordFleetAuditResource('room', room.room_id);
250
+ return renderMarkdownResult({
251
+ icon: '🏠', title,
252
+ fields: [
253
+ { label: 'ID', value: room.room_id, kind: 'code' },
254
+ { label: 'Status', value: roomStatus(room.state), kind: 'markdown' },
255
+ ...fields,
256
+ ],
257
+ });
258
+ };
195
259
  const PUBLIC_SETTLE_WAIT_MS = 60_000;
196
260
  const PUBLIC_SETTLE_POLL_MS = 100;
197
261
  const sleep = (ms) => new Promise(resolve => { setTimeout(resolve, ms); });
@@ -465,6 +529,7 @@ export function registerTaskCommands(parent, cOpt) {
465
529
  idempotencyKey: opts.idempotencyKey, origin: { type: 'cli' },
466
530
  list: opts.list,
467
531
  });
532
+ auditTask('create', record, 'none');
468
533
  if (opts.json) {
469
534
  console.log(JSON.stringify({ schema_version: 1, task: record }, null, 2));
470
535
  return;
@@ -651,9 +716,11 @@ export function registerTaskCommands(parent, cOpt) {
651
716
  .option('--json', 'JSON output')
652
717
  .action(async (id, opts) => {
653
718
  try {
719
+ const previous = getTask(id).state;
654
720
  const t = await taskRoomService(opts.configuration).startTask({
655
721
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
656
722
  });
723
+ auditTask('start', t, previous);
657
724
  if (opts.json) {
658
725
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
659
726
  return;
@@ -677,9 +744,11 @@ export function registerTaskCommands(parent, cOpt) {
677
744
  .option('--json', 'JSON output')
678
745
  .action((id, opts) => {
679
746
  try {
747
+ const previous = getTask(id).state;
680
748
  const t = taskRoomService().blockTask({
681
749
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id, reason: opts.reason,
682
750
  });
751
+ auditTask('block', t, previous);
683
752
  if (opts.json) {
684
753
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
685
754
  return;
@@ -697,9 +766,11 @@ export function registerTaskCommands(parent, cOpt) {
697
766
  .option('--json', 'JSON output')
698
767
  .action((id, opts) => {
699
768
  try {
769
+ const previous = getTask(id).state;
700
770
  const t = taskRoomService().unblockTask({
701
771
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
702
772
  });
773
+ auditTask('unblock', t, previous);
703
774
  if (opts.json) {
704
775
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
705
776
  return;
@@ -717,9 +788,11 @@ export function registerTaskCommands(parent, cOpt) {
717
788
  .option('--json', 'JSON output')
718
789
  .action((id, opts) => {
719
790
  try {
791
+ const previous = getTask(id).state;
720
792
  const t = taskRoomService().reviewTask({
721
793
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
722
794
  });
795
+ auditTask('review', t, previous);
723
796
  if (opts.json) {
724
797
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
725
798
  return;
@@ -739,6 +812,7 @@ export function registerTaskCommands(parent, cOpt) {
739
812
  .option('--json', 'JSON output')
740
813
  .action(async (id, opts) => {
741
814
  try {
815
+ const previous = getTask(id).state;
742
816
  let summary = opts.summary;
743
817
  if (opts.summaryFile)
744
818
  summary = readFileSync(opts.summaryFile, 'utf8');
@@ -750,6 +824,7 @@ export function registerTaskCommands(parent, cOpt) {
750
824
  ? await launchTaskSettleWorker(id, opts.configuration)
751
825
  : { task: plan.task, timedOut: false };
752
826
  const t = settled.task;
827
+ auditTask('done', t, previous);
753
828
  if (opts.json) {
754
829
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
755
830
  return;
@@ -777,6 +852,7 @@ export function registerTaskCommands(parent, cOpt) {
777
852
  try {
778
853
  if (id !== confirmId)
779
854
  throw taskRoomPublicError('task_confirmation_mismatch');
855
+ const previous = getTask(id).state;
780
856
  const plan = await taskRoomService(opts.configuration).cancelTask({
781
857
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
782
858
  });
@@ -784,6 +860,7 @@ export function registerTaskCommands(parent, cOpt) {
784
860
  ? await launchTaskSettleWorker(id, opts.configuration)
785
861
  : { task: plan.task, timedOut: false };
786
862
  const t = settled.task;
863
+ auditTask('cancel', t, previous);
787
864
  if (opts.json) {
788
865
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
789
866
  return;
@@ -811,9 +888,16 @@ export function registerTaskCommands(parent, cOpt) {
811
888
  try {
812
889
  if (id !== confirmId)
813
890
  throw taskRoomPublicError('task_confirmation_mismatch');
891
+ let prior;
892
+ try {
893
+ prior = getTask(id);
894
+ }
895
+ catch { /* idempotent already-absent delete */ }
814
896
  const deleted = taskRoomService().deleteTask({
815
897
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
816
898
  });
899
+ if (prior)
900
+ auditTask('delete', prior, prior.state, 'deleted');
817
901
  if (opts.json) {
818
902
  console.log(JSON.stringify({ schema_version: 1, task_id: id, deleted }, null, 2));
819
903
  return;
@@ -834,6 +918,7 @@ export function registerTaskCommands(parent, cOpt) {
834
918
  .option('--json', 'JSON output')
835
919
  .action(async (id, opts) => {
836
920
  try {
921
+ const previous = getTask(id).state;
837
922
  const app = taskRoomService(opts.configuration);
838
923
  const actor = { kind: 'local_control', surface: 'cli' };
839
924
  const begin = await app.beginTaskRecovery({ actor, taskId: id });
@@ -844,6 +929,7 @@ export function registerTaskCommands(parent, cOpt) {
844
929
  })()
845
930
  : begin.result;
846
931
  const t = recovered.task;
932
+ auditTask('recover', t, previous);
847
933
  const room = recovered.room;
848
934
  const recoveryActions = recovered.issues.map(issue => {
849
935
  if (issue.code === 'terminal_pending')
@@ -951,10 +1037,12 @@ export function registerTaskCommands(parent, cOpt) {
951
1037
  .option('--json', 'JSON output')
952
1038
  .action(async (id, opts) => {
953
1039
  try {
1040
+ const previous = getTask(id).state;
954
1041
  const result = await taskRoomService(opts.configuration).ensureTaskWork({
955
1042
  actor: { kind: 'local_control', surface: 'cli' }, taskId: id, template: opts.template,
956
1043
  });
957
1044
  const t = result.task;
1045
+ auditTask('work', t, previous);
958
1046
  if (result.status === 'already_active') {
959
1047
  if (opts.json) {
960
1048
  console.log(JSON.stringify({ schema_version: 1, task: t, status: 'already_active' }, null, 2));
@@ -993,6 +1081,7 @@ export function registerTaskCommands(parent, cOpt) {
993
1081
  .option('--json', 'JSON output')
994
1082
  .action(async (id, opts) => {
995
1083
  try {
1084
+ const previous = getTask(id).state;
996
1085
  let summary = opts.summary;
997
1086
  if (opts.summaryFile)
998
1087
  summary = readFileSync(opts.summaryFile, 'utf8');
@@ -1004,6 +1093,7 @@ export function registerTaskCommands(parent, cOpt) {
1004
1093
  ? await launchTaskSettleWorker(id, opts.configuration)
1005
1094
  : { task: plan.task, timedOut: false };
1006
1095
  const t = settled.task;
1096
+ auditTask('finish', t, previous);
1007
1097
  if (opts.json) {
1008
1098
  console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
1009
1099
  return;
@@ -1043,6 +1133,7 @@ export function registerRoomCommands(parent, cOpt) {
1043
1133
  actor: { kind: 'local_control', surface: 'cli' }, name: opts.name,
1044
1134
  template: opts.template, goal: opts.goal, brief: opts.brief, briefFile: opts.briefFile,
1045
1135
  });
1136
+ auditRoom('create', record, 'none');
1046
1137
  if (opts.json) {
1047
1138
  console.log(JSON.stringify({ schema_version: 1, room: record }, null, 2));
1048
1139
  return;
@@ -1127,7 +1218,9 @@ export function registerRoomCommands(parent, cOpt) {
1127
1218
  .option('--json', 'JSON output')
1128
1219
  .action(async (id, opts) => {
1129
1220
  try {
1130
- const { room } = await taskRoomService(opts.configuration).getRoomDetail(id);
1221
+ const { room, orchestration } = await taskRoomService(opts.configuration).getRoomDetail(id);
1222
+ if (orchestration)
1223
+ auditRoom('open', orchestration, orchestration.state);
1131
1224
  const url = `http://localhost:4460/room/${id}`;
1132
1225
  if (opts.json) {
1133
1226
  console.log(JSON.stringify({ schema_version: 1, room_id: id, url, room_name: room.room_name }, null, 2));
@@ -1154,6 +1247,8 @@ export function registerRoomCommands(parent, cOpt) {
1154
1247
  .action(async (id, opts) => {
1155
1248
  try {
1156
1249
  const { orchestration: r, members } = await taskRoomService(opts.configuration).getRoomMembers(id);
1250
+ if (r)
1251
+ auditRoom('members', r, r.state);
1157
1252
  if (opts.json) {
1158
1253
  console.log(JSON.stringify({
1159
1254
  schema_version: 1,
@@ -1187,10 +1282,13 @@ export function registerRoomCommands(parent, cOpt) {
1187
1282
  try {
1188
1283
  if (id !== confirmId)
1189
1284
  throw taskRoomPublicError('room_confirmation_mismatch');
1285
+ const previous = getRoomRecord(id);
1190
1286
  await taskRoomService(opts.configuration).requestRoomDeletion({
1191
1287
  actor: { kind: 'local_control', surface: 'cli' }, roomId: id,
1192
1288
  });
1193
1289
  const settled = await launchRoomDeleteWorker(id, opts.configuration);
1290
+ if (previous)
1291
+ auditRoom('delete', previous, previous.state, settled.deleted ? 'deleted' : 'closing');
1194
1292
  if (opts.json) {
1195
1293
  console.log(JSON.stringify({ schema_version: 1, room_id: id, deleted: settled.deleted }, null, 2));
1196
1294
  return;
@@ -1227,11 +1325,14 @@ export function registerRoomCommands(parent, cOpt) {
1227
1325
  .option('--json', 'JSON output')
1228
1326
  .action(async (id, opts) => {
1229
1327
  try {
1328
+ const previous = getRoomRecord(id);
1230
1329
  const recovered = await taskRoomService(opts.configuration).recoverRoom({
1231
1330
  actor: { kind: 'local_control', surface: 'cli' }, roomId: id,
1232
1331
  });
1233
1332
  if (recovered.kind === 'deletion_worker_required') {
1234
1333
  const settled = await launchRoomDeleteWorker(id, opts.configuration);
1334
+ if (previous)
1335
+ auditRoom('recover', previous, previous.state, settled.deleted ? 'deleted' : 'closing');
1235
1336
  if (opts.json) {
1236
1337
  console.log(JSON.stringify({
1237
1338
  schema_version: 1, room: null, orchestration: null,
@@ -1252,6 +1353,8 @@ export function registerRoomCommands(parent, cOpt) {
1252
1353
  return;
1253
1354
  }
1254
1355
  const { room: cowork, orchestration: r, issues: actions } = recovered;
1356
+ if (r)
1357
+ auditRoom('recover', r, previous?.state ?? 'unresolved', r.state);
1255
1358
  if (opts.json) {
1256
1359
  console.log(JSON.stringify({ schema_version: 1, room: cowork, orchestration: r ?? null, recovery_actions: actions }, null, 2));
1257
1360
  return;
@@ -1,2 +1,3 @@
1
+ export declare function fleetWorkerEnv(source?: NodeJS.ProcessEnv): Record<string, string>;
1
2
  /** Launch a Fleet CLI operation outside the caller's supervisor/session lifecycle. */
2
3
  export declare function launchFleetWorker(args: string[], operation: string, configPath?: string): Promise<void>;
@@ -1,14 +1,23 @@
1
1
  import { execFile, spawn } from 'node:child_process';
2
2
  import { randomUUID } from 'node:crypto';
3
- /** Launch a Fleet CLI operation outside the caller's supervisor/session lifecycle. */
4
- export function launchFleetWorker(args, operation, configPath) {
5
- const cli = [process.argv[1], ...args, ...(configPath ? ['-c', configPath] : [])];
3
+ import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV } from '../fleet-proxy.js';
4
+ export function fleetWorkerEnv(source = process.env) {
6
5
  const inheritedKeys = [
7
6
  'HOME', 'PATH', 'XDG_RUNTIME_DIR', 'OURS_FLEET_HOME', 'OURS_CONFIG',
8
7
  'OURS_PORT', 'OURS_STATE_DIR', 'OURS_API_TOKEN', 'OURS_COWORK_CONFIG',
9
8
  ];
10
- const inherited = inheritedKeys
11
- .flatMap(key => process.env[key] === undefined ? [] : [`${key}=${process.env[key]}`]);
9
+ const env = Object.fromEntries(inheritedKeys.flatMap(key => source[key] === undefined ? [] : [[key, source[key]]]));
10
+ // Explicit defense: trusted settle/recover/delete/close workers are effects
11
+ // nested inside the outer audited command, never fresh agent attempts.
12
+ delete env[FLEET_PROXY_STATE_DIR_ENV];
13
+ delete env[FLEET_PROXY_CALLER_ENV];
14
+ return env;
15
+ }
16
+ /** Launch a Fleet CLI operation outside the caller's supervisor/session lifecycle. */
17
+ export function launchFleetWorker(args, operation, configPath) {
18
+ const cli = [process.argv[1], ...args, ...(configPath ? ['-c', configPath] : [])];
19
+ const env = fleetWorkerEnv();
20
+ const inherited = Object.entries(env).map(([key, value]) => `${key}=${value}`);
12
21
  const suffix = randomUUID().slice(0, 8);
13
22
  const safeOperation = operation.replace(/[^A-Za-z0-9_.-]/g, '-');
14
23
  return new Promise((resolve, reject) => {
@@ -29,10 +38,6 @@ export function launchFleetWorker(args, operation, configPath) {
29
38
  ], { timeout: 15_000 }, error => error ? reject(error) : resolve());
30
39
  return;
31
40
  }
32
- const env = Object.fromEntries(inherited.map(value => {
33
- const split = value.indexOf('=');
34
- return [value.slice(0, split), value.slice(split + 1)];
35
- }));
36
41
  const child = spawn(process.execPath, cli, { detached: true, stdio: 'ignore', env });
37
42
  child.once('error', reject);
38
43
  child.once('spawn', () => { child.unref(); resolve(); });
package/dist/runner.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { type ResolvedRole } from './config.js';
2
2
  import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
3
+ import { type DaemonGenerationProbe } from './daemon-recovery.js';
3
4
  import { type Exec } from './exec.js';
4
5
  import { RoleControlServer } from './session/control.js';
5
6
  import type { AgentSession, ExitRecord, TurnResult } from './session/types.js';
@@ -15,6 +16,7 @@ export interface RunnerDeps {
15
16
  log(line: string): void;
16
17
  /** HTTP transport for the monitor's daemon long-poll (injectable for tests). */
17
18
  fetch: FetchLike;
19
+ probeGeneration(env: NodeJS.ProcessEnv): Promise<DaemonGenerationProbe>;
18
20
  /** Construct the supervisor mail monitor (injectable so tests stub it out). */
19
21
  createMonitor(opts: MonitorOpts): MonitorHandle;
20
22
  /** Construct trusted owner ingress (injectable for lifecycle tests). */
@@ -23,7 +25,7 @@ export interface RunnerDeps {
23
25
  /** Neutral construction seam for deterministic runner lifecycle tests. */
24
26
  startAgentSession(adapter: AgentSessionAdapter, options: AgentSessionStartOptions): Promise<AgentSession>;
25
27
  /** Construct the authenticated role control route (injectable where sockets are unavailable). */
26
- createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
28
+ createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setFleetAuditor' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
27
29
  /** Acquire the cross-process owner-channel binder lease before replacing the control socket. */
28
30
  acquireOwnerBinder(stateDir: string, role: string, identity: string): Promise<OwnerBinderLease>;
29
31
  /** Ask the still-authenticated predecessor to emit the fixed recovery notice. */
@@ -31,6 +33,11 @@ export interface RunnerDeps {
31
33
  /** Lets a test (or a shutdown path) end the supervised restart loop. */
32
34
  shouldStop?(): boolean;
33
35
  }
36
+ export declare const SUPERVISOR_RECYCLE_REQUIRED = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
37
+ export declare class SupervisorRecycleRequiredError extends Error {
38
+ readonly code = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
39
+ constructor();
40
+ }
34
41
  /** Environment injected only into the managed harness process. */
35
42
  export declare function managedFleetProxyEnv(role: ResolvedRole, stateDir: string): Record<string, string>;
36
43
  /**
@@ -176,4 +183,4 @@ export declare function runSupervised(name: string, opts?: {
176
183
  allowResumeRotation?: boolean;
177
184
  }, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
178
185
  /** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
179
- export declare function runTemp(name: string, deps?: Partial<RunnerDeps>): Promise<void>;
186
+ export declare function runTemp(name: string, deps?: Partial<RunnerDeps>, attempt?: typeof runOnce): Promise<void>;
package/dist/runner.js CHANGED
@@ -6,6 +6,8 @@ import { agentDir, stateRoot } from './paths.js';
6
6
  import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolvePermissions, } from './config.js';
7
7
  import { getAdapter } from './harness/registry.js';
8
8
  import { createMonitor, probeIdentityPresence, } from './monitor.js';
9
+ import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
10
+ import { recoverAgentIdentity } from './agent-recovery-gate.js';
9
11
  import { realExec } from './exec.js';
10
12
  import { resolveIsolation } from './isolation/policy.js';
11
13
  import { selectIsolationBackend } from './isolation/registry.js';
@@ -24,6 +26,14 @@ import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, } from './fleet-prox
24
26
  import { effectivePermissionMode } from './permissions.js';
25
27
  import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
26
28
  import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
29
+ export const SUPERVISOR_RECYCLE_REQUIRED = 'OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED';
30
+ export class SupervisorRecycleRequiredError extends Error {
31
+ code = SUPERVISOR_RECYCLE_REQUIRED;
32
+ constructor() {
33
+ super('owner channel requires a fresh supervisor process after unproven client quiescence');
34
+ this.name = 'SupervisorRecycleRequiredError';
35
+ }
36
+ }
27
37
  const defaultDeps = () => ({
28
38
  exec: realExec,
29
39
  cpuDelegated: () => cpuControllerDelegated(),
@@ -38,6 +48,7 @@ const defaultDeps = () => ({
38
48
  now: () => Date.now(),
39
49
  log: line => process.stderr.write(line + '\n'),
40
50
  fetch: (url, init) => globalThis.fetch(url, init),
51
+ probeGeneration: env => probeDaemonGeneration((url, init) => globalThis.fetch(url, init), env),
41
52
  createMonitor: opts => createMonitor(opts),
42
53
  createOwnerChannel: opts => new OwnerChannel(opts),
43
54
  startAgentSession: (adapter, options) => adapter.start(options),
@@ -532,6 +543,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
532
543
  name, identity: role.identity, agentDir: dir, cfg: role.monitor,
533
544
  deps: resolvedMonitorDeps,
534
545
  }) : null;
546
+ const daemonObserver = new DaemonGenerationObserver();
547
+ const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
548
+ let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
535
549
  if (monitor)
536
550
  await monitor.prime({ resetCursor: resetMonitorCursor });
537
551
  rmSync(exitFile, { force: true });
@@ -544,9 +558,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
544
558
  let sessionClosed = false;
545
559
  let ownerChannel;
546
560
  let ownerBinder;
547
- const pendingFleetSpawnNotices = [];
548
561
  let loopManager;
549
562
  let arbiter;
563
+ let recoveryController;
550
564
  let reloadLoopConfig;
551
565
  let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
552
566
  loop.name, loop.definitionHash, loop.promptHash,
@@ -607,21 +621,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
607
621
  throw error;
608
622
  }
609
623
  control.setFleetSpawner(async (requested) => {
610
- const event = await executeManagedSpawn(role, configPath, requested, deps.log);
611
- if (!role.owner_channel)
612
- return event;
613
- if (ownerChannel?.notifyFleetSpawn) {
614
- try {
615
- await ownerChannel.notifyFleetSpawn(event);
616
- }
617
- catch (error) {
618
- deps.log(`[${name}] spawned-agent owner notice failed: `
619
- + `${error?.message ?? String(error)}`);
620
- }
621
- }
622
- else
623
- pendingFleetSpawnNotices.push(event);
624
- return event;
624
+ return executeManagedSpawn(role, configPath, requested, deps.log);
625
625
  });
626
626
  resolvedMonitorDeps.delivery = {
627
627
  // A wake is only delivered when its turn TERMINATES successfully. A
@@ -743,18 +743,14 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
743
743
  throw new Error(`[${name}] owner channel failed to start: `
744
744
  + `${error?.message ?? String(error)}`);
745
745
  }
746
- for (const event of pendingFleetSpawnNotices.splice(0)) {
747
- try {
748
- await ownerChannel.notifyFleetSpawn?.(event);
749
- }
750
- catch (error) {
751
- deps.log(`[${name}] deferred spawned-agent owner notice failed: `
752
- + `${error?.message ?? String(error)}`);
753
- }
754
- }
755
746
  }
756
- if (ownerChannel)
747
+ if (ownerChannel) {
757
748
  control.setOwnerChannel(ownerChannel);
749
+ control.setFleetAuditor({
750
+ begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
751
+ finish: input => ownerChannel.finishFleetCommandAudit(input),
752
+ });
753
+ }
758
754
  reloadLoopConfig = async () => {
759
755
  const nextRole = findRole(loadConfig(configPath), name);
760
756
  const definitions = nextRole.loops ?? [];
@@ -801,16 +797,49 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
801
797
  // The monitor loop lives exactly as long as the session: it starts once the
802
798
  // pane pid is known and is stopped when that pid dies (task dies with runner).
803
799
  monitorLoop ??= monitor?.run(pid);
800
+ recoveryController = new RoleRecoveryController({
801
+ role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
802
+ log: deps.log,
803
+ recoverAgent: async () => {
804
+ const evidence = await recoverAgentIdentity(arbiter, role.identity);
805
+ return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
806
+ },
807
+ recoverOwner: async (epoch) => {
808
+ if (!ownerChannel?.recover)
809
+ return { ok: true };
810
+ try {
811
+ await ownerChannel.recover(epoch);
812
+ return { ok: true };
813
+ }
814
+ catch (error) {
815
+ return { ok: false, reason: error instanceof Error
816
+ ? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
817
+ }
818
+ },
819
+ });
804
820
  const start = deps.now();
805
821
  let nextLoopReloadAt = deps.now() + 30_000;
806
822
  let nextIdentityPollAt = deps.now();
823
+ let nextDaemonProbeAt = deps.now();
807
824
  let lastReloadError = '';
808
825
  let identityObserved = false;
809
826
  let identityAbsentSince;
810
827
  let retirementReason;
828
+ let supervisorRecycleRequired = false;
811
829
  while (sessionHandle.isAlive()) {
812
830
  await deps.sleep(temp ? 500 : 2000);
813
831
  const now = deps.now();
832
+ if (now >= nextDaemonProbeAt) {
833
+ nextDaemonProbeAt = now + 2_000;
834
+ const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
835
+ if (observation.kind === 'lost')
836
+ recoveryController.noteLoss(observation.reason);
837
+ if (observation.kind === 'changed' || observation.kind === 'available'
838
+ || (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
839
+ sessionStartedWithoutDaemonBaseline = false;
840
+ void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
841
+ }
842
+ }
814
843
  if (temp && deps.shouldStop?.()) {
815
844
  retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
816
845
  deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
@@ -861,6 +890,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
861
890
  control?.setLoopManager(undefined);
862
891
  await loopManager.stop();
863
892
  }
893
+ // Let already-settled path operations publish their aggregate result before
894
+ // the shutdown fence invalidates the epoch. This never waits on I/O.
895
+ await Promise.resolve();
896
+ await Promise.resolve();
897
+ recoveryController?.cancel();
864
898
  control?.setConfigReloader(undefined);
865
899
  // Close the authenticated control route before releasing the binder lease;
866
900
  // otherwise the predecessor can unlink the replacement's new socket.
@@ -871,7 +905,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
871
905
  }
872
906
  if (ownerChannel)
873
907
  await ownerChannel.close();
874
- ownerBinder?.release();
908
+ if (ownerChannel?.binderReleaseSafe?.() === false) {
909
+ deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
910
+ supervisorRecycleRequired = true;
911
+ }
912
+ else
913
+ ownerBinder?.release();
875
914
  if (monitor) {
876
915
  monitor.stop();
877
916
  await monitorLoop;
@@ -920,6 +959,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
920
959
  }
921
960
  else
922
961
  deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
962
+ if (supervisorRecycleRequired)
963
+ throw new SupervisorRecycleRequiredError();
923
964
  return {
924
965
  elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
925
966
  ...(retirementReason ? { retirementReason } : {}),
@@ -996,6 +1037,16 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
996
1037
  result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
997
1038
  }
998
1039
  catch (e) {
1040
+ if (e instanceof SupervisorRecycleRequiredError) {
1041
+ writeRestartLedger(dir, {
1042
+ ...ledger,
1043
+ lastReason: SUPERVISOR_RECYCLE_REQUIRED,
1044
+ nextDelayMs: 0,
1045
+ updatedAt: stamp(),
1046
+ });
1047
+ deps.log(`[${name}] supervisor recycle required: ${SUPERVISOR_RECYCLE_REQUIRED}`);
1048
+ throw e;
1049
+ }
999
1050
  // A session that could not even start is an immediate failure like any
1000
1051
  // other; it must count, or an unstartable role loops forever.
1001
1052
  result = {
@@ -1089,7 +1140,7 @@ function fastFailSecsFor(name, configPath) {
1089
1140
  }
1090
1141
  }
1091
1142
  /** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
1092
- export async function runTemp(name, deps = {}) {
1143
+ export async function runTemp(name, deps = {}, attempt = runOnce) {
1093
1144
  const dir = agentDir(name, true);
1094
1145
  await markTempSupervisorActive(dir);
1095
1146
  let signal;
@@ -1100,7 +1151,7 @@ export async function runTemp(name, deps = {}) {
1100
1151
  let result;
1101
1152
  let failure;
1102
1153
  try {
1103
- result = await runOnce(name, { temp: true }, {
1154
+ result = await attempt(name, { temp: true }, {
1104
1155
  ...deps,
1105
1156
  shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
1106
1157
  });
@@ -1114,6 +1165,7 @@ export async function runTemp(name, deps = {}) {
1114
1165
  const requested = requestedTempStopReason(dir);
1115
1166
  const reason = requested
1116
1167
  ?? result?.retirementReason
1168
+ ?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
1117
1169
  ?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
1118
1170
  // A service-manager stop can make the child connection close before the
1119
1171
  // runner reaches its normal loop. That is still a successful requested