@ours.network/fleet 1.1.0-nightly.5 → 1.1.0-nightly.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/fleet-query-service.js +13 -1
- package/dist/application/role-creation-service.js +11 -0
- package/dist/briefing.js +6 -2
- package/dist/build-info.json +4 -4
- package/dist/cli.js +95 -8
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +15 -5
- package/dist/doctor.js +14 -0
- package/dist/fleet-command-audit.d.ts +131 -0
- package/dist/fleet-command-audit.js +375 -0
- package/dist/fleet-proxy.d.ts +2 -0
- package/dist/owner-channel/channel.d.ts +56 -0
- package/dist/owner-channel/channel.js +251 -17
- package/dist/rooms-tasks/cli.js +121 -18
- package/dist/rooms-tasks/external-worker.d.ts +1 -0
- package/dist/rooms-tasks/external-worker.js +14 -9
- package/dist/runner.d.ts +9 -2
- package/dist/runner.js +81 -29
- package/dist/session/control.d.ts +14 -1
- package/dist/session/control.js +35 -0
- package/dist/temp-lifecycle.d.ts +1 -1
- package/package.json +1 -1
package/dist/rooms-tasks/cli.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import { loadConfig, ConfigError } from '../config.js';
|
|
3
4
|
import { provisionMembers, getBinPath } from './provision.js';
|
|
4
5
|
import { launchFleetWorker } from './external-worker.js';
|
|
@@ -9,6 +10,7 @@ import { createCoworkAdapter, CoworkProtocolError } from './cowork-adapter.js';
|
|
|
9
10
|
import { markdownCode, markdownProse, renderMarkdownFailure, renderMarkdownList, renderMarkdownResult, roomStatus, taskStatus, } from './markdown.js';
|
|
10
11
|
import { TaskRoomApplicationError, TaskRoomApplicationService } from '../application/task-room-service.js';
|
|
11
12
|
import { TaskListError } from './task-lists.js';
|
|
13
|
+
import { recordFleetAuditFailure, recordFleetAuditPresentation, recordFleetAuditResource, } from '../fleet-command-audit.js';
|
|
12
14
|
class TaskRoomPublicError extends Error {
|
|
13
15
|
code;
|
|
14
16
|
fields;
|
|
@@ -89,8 +91,29 @@ function die(e) {
|
|
|
89
91
|
const msg = e instanceof TaskRoomPublicError
|
|
90
92
|
? taskRoomPublicFailure(e).legacy : e instanceof Error ? e.message : String(e);
|
|
91
93
|
process.stderr.write(`error: ${msg}\n`);
|
|
94
|
+
recordFleetAuditFailure(classifyTaskRoomFailure(e));
|
|
92
95
|
process.exit(1);
|
|
93
96
|
}
|
|
97
|
+
function classifyTaskRoomFailure(e) {
|
|
98
|
+
if (e instanceof TaskRoomApplicationError)
|
|
99
|
+
e = taskRoomPublicError(e.code, e.fields);
|
|
100
|
+
if (e instanceof TaskRoomPublicError || e instanceof TaskListError || e instanceof ConfigError)
|
|
101
|
+
return { class: 'validation', effect: 'not_started' };
|
|
102
|
+
if (e instanceof TaskStateError) {
|
|
103
|
+
const known = /^task not found: [A-Za-z0-9_-]{1,128}$/u.test(e.message)
|
|
104
|
+
|| e.message === 'invalid task ID: expected the canonical 17-character lowercase ID'
|
|
105
|
+
|| isKnownTaskStateMessage(e.message);
|
|
106
|
+
return known ? { class: 'validation', effect: 'not_started' }
|
|
107
|
+
: { class: 'runtime', effect: 'unknown' };
|
|
108
|
+
}
|
|
109
|
+
if (e instanceof RoomStateError) {
|
|
110
|
+
const known = /^room not found: [A-Za-z0-9_-]{1,128}$/u.test(e.message)
|
|
111
|
+
|| isKnownRoomStateMessage(e.message);
|
|
112
|
+
return known ? { class: 'validation', effect: 'not_started' }
|
|
113
|
+
: { class: 'runtime', effect: 'unknown' };
|
|
114
|
+
}
|
|
115
|
+
return { class: 'runtime', effect: 'unknown' };
|
|
116
|
+
}
|
|
94
117
|
const TASK_STATE_WORD = '(?:backlog|provisioning|active|review|done|cancelled|failed)';
|
|
95
118
|
const SAFE_ID_WORD = '[A-Za-z0-9_-]{1,128}';
|
|
96
119
|
function isKnownTaskStateMessage(message) {
|
|
@@ -127,6 +150,7 @@ function dieTaskRoom(e) {
|
|
|
127
150
|
action: failure.action,
|
|
128
151
|
});
|
|
129
152
|
process.stderr.write(`${output}\n`);
|
|
153
|
+
recordFleetAuditFailure({ class: 'validation', effect: 'not_started' });
|
|
130
154
|
process.exit(1);
|
|
131
155
|
}
|
|
132
156
|
if (e instanceof TaskListError) {
|
|
@@ -142,6 +166,7 @@ function dieTaskRoom(e) {
|
|
|
142
166
|
: 'Use a valid NFC-normalized name of at most 64 Unicode code points.',
|
|
143
167
|
});
|
|
144
168
|
process.stderr.write(`${output}\n`);
|
|
169
|
+
recordFleetAuditFailure({ class: 'validation', effect: 'not_started' });
|
|
145
170
|
process.exit(1);
|
|
146
171
|
}
|
|
147
172
|
const taskMissing = e instanceof TaskStateError
|
|
@@ -167,6 +192,7 @@ function dieTaskRoom(e) {
|
|
|
167
192
|
: 'Retry once; if it repeats, run ours-fleet doctor and inspect the role logs.',
|
|
168
193
|
});
|
|
169
194
|
process.stderr.write(`${output}\n`);
|
|
195
|
+
recordFleetAuditFailure(classifyTaskRoomFailure(e));
|
|
170
196
|
process.exit(1);
|
|
171
197
|
}
|
|
172
198
|
const taskListMarkdown = (tasks, title = 'Tasks') => renderMarkdownList({
|
|
@@ -175,23 +201,61 @@ const taskListMarkdown = (tasks, title = 'Tasks') => renderMarkdownList({
|
|
|
175
201
|
+ ` — List ${markdownCode(t.list_name ?? 'default')}`
|
|
176
202
|
+ (t.blocked ? ` — 🚧 Blocked: ${markdownProse(t.blocked.reason)}` : '')),
|
|
177
203
|
});
|
|
178
|
-
const taskActionMarkdown = (title, task, fields = []) =>
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
204
|
+
const taskActionMarkdown = (title, task, fields = []) => {
|
|
205
|
+
recordFleetAuditResource('task', task.task_id);
|
|
206
|
+
if (task.room_id)
|
|
207
|
+
recordFleetAuditResource('room', task.room_id);
|
|
208
|
+
return renderMarkdownResult({
|
|
209
|
+
icon: '📋', title,
|
|
210
|
+
fields: [
|
|
211
|
+
{ label: 'ID', value: task.task_id, kind: 'code' },
|
|
212
|
+
{ label: 'Status', value: taskStatus(task.state), kind: 'markdown' },
|
|
213
|
+
{ label: 'List', value: task.list_name ?? 'default', kind: 'code' },
|
|
214
|
+
...fields,
|
|
215
|
+
],
|
|
216
|
+
});
|
|
217
|
+
};
|
|
218
|
+
function safeBrainSummary(definition) {
|
|
219
|
+
const brain = definition?.brain;
|
|
220
|
+
if (!brain || typeof brain !== 'object' || Array.isArray(brain))
|
|
221
|
+
return 'unresolved';
|
|
222
|
+
const selection = brain;
|
|
223
|
+
if (typeof selection.ref === 'string')
|
|
224
|
+
return `ref:${selection.ref}`;
|
|
225
|
+
if (!Object.hasOwn(selection, 'inline'))
|
|
226
|
+
return 'unresolved';
|
|
227
|
+
const fingerprint = createHash('sha256').update(JSON.stringify(selection.inline)).digest('hex').slice(0, 16);
|
|
228
|
+
return `inline:sha256:${fingerprint}`;
|
|
229
|
+
}
|
|
230
|
+
function auditTask(operation, task, previousState, newState = task.state) {
|
|
231
|
+
recordFleetAuditResource('task', task.task_id);
|
|
232
|
+
if (task.room_id)
|
|
233
|
+
recordFleetAuditResource('room', task.room_id);
|
|
234
|
+
const room = task.room_id ? getRoomRecord(task.room_id) : undefined;
|
|
235
|
+
const brains = new Map(room?.member_seats.map(seat => [seat.role_name, safeBrainSummary(seat.launch?.agent_definition)]) ?? []);
|
|
236
|
+
recordFleetAuditPresentation({ kind: 'task', operation, id: task.task_id, title: task.title,
|
|
237
|
+
previousState, newState,
|
|
238
|
+
template: task.template ? `${task.template.name}@${task.template.version}` : undefined,
|
|
239
|
+
roomId: task.room_id, agents: task.member_roles.map(member => ({ name: member.name,
|
|
240
|
+
brain: brains.get(member.name) ?? 'unresolved', role: member.cowork_role })) });
|
|
241
|
+
}
|
|
242
|
+
function auditRoom(operation, room, previousState, newState = room.state) {
|
|
243
|
+
recordFleetAuditResource('room', room.room_id);
|
|
244
|
+
recordFleetAuditPresentation({ kind: 'room', operation, id: room.room_id, previousState, newState,
|
|
245
|
+
template: room.template_snapshot ? `${room.template_snapshot.name}@${room.template_snapshot.version}` : undefined,
|
|
246
|
+
participants: room.member_seats.map(member => ({ name: member.role_name, role: member.cowork_role })) });
|
|
247
|
+
}
|
|
248
|
+
const roomActionMarkdown = (title, room, fields = []) => {
|
|
249
|
+
recordFleetAuditResource('room', room.room_id);
|
|
250
|
+
return renderMarkdownResult({
|
|
251
|
+
icon: '🏠', title,
|
|
252
|
+
fields: [
|
|
253
|
+
{ label: 'ID', value: room.room_id, kind: 'code' },
|
|
254
|
+
{ label: 'Status', value: roomStatus(room.state), kind: 'markdown' },
|
|
255
|
+
...fields,
|
|
256
|
+
],
|
|
257
|
+
});
|
|
258
|
+
};
|
|
195
259
|
const PUBLIC_SETTLE_WAIT_MS = 60_000;
|
|
196
260
|
const PUBLIC_SETTLE_POLL_MS = 100;
|
|
197
261
|
const sleep = (ms) => new Promise(resolve => { setTimeout(resolve, ms); });
|
|
@@ -465,6 +529,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
465
529
|
idempotencyKey: opts.idempotencyKey, origin: { type: 'cli' },
|
|
466
530
|
list: opts.list,
|
|
467
531
|
});
|
|
532
|
+
auditTask('create', record, 'none');
|
|
468
533
|
if (opts.json) {
|
|
469
534
|
console.log(JSON.stringify({ schema_version: 1, task: record }, null, 2));
|
|
470
535
|
return;
|
|
@@ -651,9 +716,11 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
651
716
|
.option('--json', 'JSON output')
|
|
652
717
|
.action(async (id, opts) => {
|
|
653
718
|
try {
|
|
719
|
+
const previous = getTask(id).state;
|
|
654
720
|
const t = await taskRoomService(opts.configuration).startTask({
|
|
655
721
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
|
|
656
722
|
});
|
|
723
|
+
auditTask('start', t, previous);
|
|
657
724
|
if (opts.json) {
|
|
658
725
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
659
726
|
return;
|
|
@@ -677,9 +744,11 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
677
744
|
.option('--json', 'JSON output')
|
|
678
745
|
.action((id, opts) => {
|
|
679
746
|
try {
|
|
747
|
+
const previous = getTask(id).state;
|
|
680
748
|
const t = taskRoomService().blockTask({
|
|
681
749
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id, reason: opts.reason,
|
|
682
750
|
});
|
|
751
|
+
auditTask('block', t, previous);
|
|
683
752
|
if (opts.json) {
|
|
684
753
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
685
754
|
return;
|
|
@@ -697,9 +766,11 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
697
766
|
.option('--json', 'JSON output')
|
|
698
767
|
.action((id, opts) => {
|
|
699
768
|
try {
|
|
769
|
+
const previous = getTask(id).state;
|
|
700
770
|
const t = taskRoomService().unblockTask({
|
|
701
771
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
|
|
702
772
|
});
|
|
773
|
+
auditTask('unblock', t, previous);
|
|
703
774
|
if (opts.json) {
|
|
704
775
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
705
776
|
return;
|
|
@@ -717,9 +788,11 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
717
788
|
.option('--json', 'JSON output')
|
|
718
789
|
.action((id, opts) => {
|
|
719
790
|
try {
|
|
791
|
+
const previous = getTask(id).state;
|
|
720
792
|
const t = taskRoomService().reviewTask({
|
|
721
793
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
|
|
722
794
|
});
|
|
795
|
+
auditTask('review', t, previous);
|
|
723
796
|
if (opts.json) {
|
|
724
797
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
725
798
|
return;
|
|
@@ -739,6 +812,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
739
812
|
.option('--json', 'JSON output')
|
|
740
813
|
.action(async (id, opts) => {
|
|
741
814
|
try {
|
|
815
|
+
const previous = getTask(id).state;
|
|
742
816
|
let summary = opts.summary;
|
|
743
817
|
if (opts.summaryFile)
|
|
744
818
|
summary = readFileSync(opts.summaryFile, 'utf8');
|
|
@@ -750,6 +824,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
750
824
|
? await launchTaskSettleWorker(id, opts.configuration)
|
|
751
825
|
: { task: plan.task, timedOut: false };
|
|
752
826
|
const t = settled.task;
|
|
827
|
+
auditTask('done', t, previous);
|
|
753
828
|
if (opts.json) {
|
|
754
829
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
755
830
|
return;
|
|
@@ -777,6 +852,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
777
852
|
try {
|
|
778
853
|
if (id !== confirmId)
|
|
779
854
|
throw taskRoomPublicError('task_confirmation_mismatch');
|
|
855
|
+
const previous = getTask(id).state;
|
|
780
856
|
const plan = await taskRoomService(opts.configuration).cancelTask({
|
|
781
857
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
|
|
782
858
|
});
|
|
@@ -784,6 +860,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
784
860
|
? await launchTaskSettleWorker(id, opts.configuration)
|
|
785
861
|
: { task: plan.task, timedOut: false };
|
|
786
862
|
const t = settled.task;
|
|
863
|
+
auditTask('cancel', t, previous);
|
|
787
864
|
if (opts.json) {
|
|
788
865
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
789
866
|
return;
|
|
@@ -811,9 +888,16 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
811
888
|
try {
|
|
812
889
|
if (id !== confirmId)
|
|
813
890
|
throw taskRoomPublicError('task_confirmation_mismatch');
|
|
891
|
+
let prior;
|
|
892
|
+
try {
|
|
893
|
+
prior = getTask(id);
|
|
894
|
+
}
|
|
895
|
+
catch { /* idempotent already-absent delete */ }
|
|
814
896
|
const deleted = taskRoomService().deleteTask({
|
|
815
897
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id,
|
|
816
898
|
});
|
|
899
|
+
if (prior)
|
|
900
|
+
auditTask('delete', prior, prior.state, 'deleted');
|
|
817
901
|
if (opts.json) {
|
|
818
902
|
console.log(JSON.stringify({ schema_version: 1, task_id: id, deleted }, null, 2));
|
|
819
903
|
return;
|
|
@@ -834,6 +918,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
834
918
|
.option('--json', 'JSON output')
|
|
835
919
|
.action(async (id, opts) => {
|
|
836
920
|
try {
|
|
921
|
+
const previous = getTask(id).state;
|
|
837
922
|
const app = taskRoomService(opts.configuration);
|
|
838
923
|
const actor = { kind: 'local_control', surface: 'cli' };
|
|
839
924
|
const begin = await app.beginTaskRecovery({ actor, taskId: id });
|
|
@@ -844,6 +929,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
844
929
|
})()
|
|
845
930
|
: begin.result;
|
|
846
931
|
const t = recovered.task;
|
|
932
|
+
auditTask('recover', t, previous);
|
|
847
933
|
const room = recovered.room;
|
|
848
934
|
const recoveryActions = recovered.issues.map(issue => {
|
|
849
935
|
if (issue.code === 'terminal_pending')
|
|
@@ -951,10 +1037,12 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
951
1037
|
.option('--json', 'JSON output')
|
|
952
1038
|
.action(async (id, opts) => {
|
|
953
1039
|
try {
|
|
1040
|
+
const previous = getTask(id).state;
|
|
954
1041
|
const result = await taskRoomService(opts.configuration).ensureTaskWork({
|
|
955
1042
|
actor: { kind: 'local_control', surface: 'cli' }, taskId: id, template: opts.template,
|
|
956
1043
|
});
|
|
957
1044
|
const t = result.task;
|
|
1045
|
+
auditTask('work', t, previous);
|
|
958
1046
|
if (result.status === 'already_active') {
|
|
959
1047
|
if (opts.json) {
|
|
960
1048
|
console.log(JSON.stringify({ schema_version: 1, task: t, status: 'already_active' }, null, 2));
|
|
@@ -993,6 +1081,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
993
1081
|
.option('--json', 'JSON output')
|
|
994
1082
|
.action(async (id, opts) => {
|
|
995
1083
|
try {
|
|
1084
|
+
const previous = getTask(id).state;
|
|
996
1085
|
let summary = opts.summary;
|
|
997
1086
|
if (opts.summaryFile)
|
|
998
1087
|
summary = readFileSync(opts.summaryFile, 'utf8');
|
|
@@ -1004,6 +1093,7 @@ export function registerTaskCommands(parent, cOpt) {
|
|
|
1004
1093
|
? await launchTaskSettleWorker(id, opts.configuration)
|
|
1005
1094
|
: { task: plan.task, timedOut: false };
|
|
1006
1095
|
const t = settled.task;
|
|
1096
|
+
auditTask('finish', t, previous);
|
|
1007
1097
|
if (opts.json) {
|
|
1008
1098
|
console.log(JSON.stringify({ schema_version: 1, task: t }, null, 2));
|
|
1009
1099
|
return;
|
|
@@ -1043,6 +1133,7 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1043
1133
|
actor: { kind: 'local_control', surface: 'cli' }, name: opts.name,
|
|
1044
1134
|
template: opts.template, goal: opts.goal, brief: opts.brief, briefFile: opts.briefFile,
|
|
1045
1135
|
});
|
|
1136
|
+
auditRoom('create', record, 'none');
|
|
1046
1137
|
if (opts.json) {
|
|
1047
1138
|
console.log(JSON.stringify({ schema_version: 1, room: record }, null, 2));
|
|
1048
1139
|
return;
|
|
@@ -1127,7 +1218,9 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1127
1218
|
.option('--json', 'JSON output')
|
|
1128
1219
|
.action(async (id, opts) => {
|
|
1129
1220
|
try {
|
|
1130
|
-
const { room } = await taskRoomService(opts.configuration).getRoomDetail(id);
|
|
1221
|
+
const { room, orchestration } = await taskRoomService(opts.configuration).getRoomDetail(id);
|
|
1222
|
+
if (orchestration)
|
|
1223
|
+
auditRoom('open', orchestration, orchestration.state);
|
|
1131
1224
|
const url = `http://localhost:4460/room/${id}`;
|
|
1132
1225
|
if (opts.json) {
|
|
1133
1226
|
console.log(JSON.stringify({ schema_version: 1, room_id: id, url, room_name: room.room_name }, null, 2));
|
|
@@ -1154,6 +1247,8 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1154
1247
|
.action(async (id, opts) => {
|
|
1155
1248
|
try {
|
|
1156
1249
|
const { orchestration: r, members } = await taskRoomService(opts.configuration).getRoomMembers(id);
|
|
1250
|
+
if (r)
|
|
1251
|
+
auditRoom('members', r, r.state);
|
|
1157
1252
|
if (opts.json) {
|
|
1158
1253
|
console.log(JSON.stringify({
|
|
1159
1254
|
schema_version: 1,
|
|
@@ -1187,10 +1282,13 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1187
1282
|
try {
|
|
1188
1283
|
if (id !== confirmId)
|
|
1189
1284
|
throw taskRoomPublicError('room_confirmation_mismatch');
|
|
1285
|
+
const previous = getRoomRecord(id);
|
|
1190
1286
|
await taskRoomService(opts.configuration).requestRoomDeletion({
|
|
1191
1287
|
actor: { kind: 'local_control', surface: 'cli' }, roomId: id,
|
|
1192
1288
|
});
|
|
1193
1289
|
const settled = await launchRoomDeleteWorker(id, opts.configuration);
|
|
1290
|
+
if (previous)
|
|
1291
|
+
auditRoom('delete', previous, previous.state, settled.deleted ? 'deleted' : 'closing');
|
|
1194
1292
|
if (opts.json) {
|
|
1195
1293
|
console.log(JSON.stringify({ schema_version: 1, room_id: id, deleted: settled.deleted }, null, 2));
|
|
1196
1294
|
return;
|
|
@@ -1227,11 +1325,14 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1227
1325
|
.option('--json', 'JSON output')
|
|
1228
1326
|
.action(async (id, opts) => {
|
|
1229
1327
|
try {
|
|
1328
|
+
const previous = getRoomRecord(id);
|
|
1230
1329
|
const recovered = await taskRoomService(opts.configuration).recoverRoom({
|
|
1231
1330
|
actor: { kind: 'local_control', surface: 'cli' }, roomId: id,
|
|
1232
1331
|
});
|
|
1233
1332
|
if (recovered.kind === 'deletion_worker_required') {
|
|
1234
1333
|
const settled = await launchRoomDeleteWorker(id, opts.configuration);
|
|
1334
|
+
if (previous)
|
|
1335
|
+
auditRoom('recover', previous, previous.state, settled.deleted ? 'deleted' : 'closing');
|
|
1235
1336
|
if (opts.json) {
|
|
1236
1337
|
console.log(JSON.stringify({
|
|
1237
1338
|
schema_version: 1, room: null, orchestration: null,
|
|
@@ -1252,6 +1353,8 @@ export function registerRoomCommands(parent, cOpt) {
|
|
|
1252
1353
|
return;
|
|
1253
1354
|
}
|
|
1254
1355
|
const { room: cowork, orchestration: r, issues: actions } = recovered;
|
|
1356
|
+
if (r)
|
|
1357
|
+
auditRoom('recover', r, previous?.state ?? 'unresolved', r.state);
|
|
1255
1358
|
if (opts.json) {
|
|
1256
1359
|
console.log(JSON.stringify({ schema_version: 1, room: cowork, orchestration: r ?? null, recovery_actions: actions }, null, 2));
|
|
1257
1360
|
return;
|
|
@@ -1,2 +1,3 @@
|
|
|
1
|
+
export declare function fleetWorkerEnv(source?: NodeJS.ProcessEnv): Record<string, string>;
|
|
1
2
|
/** Launch a Fleet CLI operation outside the caller's supervisor/session lifecycle. */
|
|
2
3
|
export declare function launchFleetWorker(args: string[], operation: string, configPath?: string): Promise<void>;
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { execFile, spawn } from 'node:child_process';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
|
|
4
|
-
export function
|
|
5
|
-
const cli = [process.argv[1], ...args, ...(configPath ? ['-c', configPath] : [])];
|
|
3
|
+
import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV } from '../fleet-proxy.js';
|
|
4
|
+
export function fleetWorkerEnv(source = process.env) {
|
|
6
5
|
const inheritedKeys = [
|
|
7
6
|
'HOME', 'PATH', 'XDG_RUNTIME_DIR', 'OURS_FLEET_HOME', 'OURS_CONFIG',
|
|
8
7
|
'OURS_PORT', 'OURS_STATE_DIR', 'OURS_API_TOKEN', 'OURS_COWORK_CONFIG',
|
|
9
8
|
];
|
|
10
|
-
const
|
|
11
|
-
|
|
9
|
+
const env = Object.fromEntries(inheritedKeys.flatMap(key => source[key] === undefined ? [] : [[key, source[key]]]));
|
|
10
|
+
// Explicit defense: trusted settle/recover/delete/close workers are effects
|
|
11
|
+
// nested inside the outer audited command, never fresh agent attempts.
|
|
12
|
+
delete env[FLEET_PROXY_STATE_DIR_ENV];
|
|
13
|
+
delete env[FLEET_PROXY_CALLER_ENV];
|
|
14
|
+
return env;
|
|
15
|
+
}
|
|
16
|
+
/** Launch a Fleet CLI operation outside the caller's supervisor/session lifecycle. */
|
|
17
|
+
export function launchFleetWorker(args, operation, configPath) {
|
|
18
|
+
const cli = [process.argv[1], ...args, ...(configPath ? ['-c', configPath] : [])];
|
|
19
|
+
const env = fleetWorkerEnv();
|
|
20
|
+
const inherited = Object.entries(env).map(([key, value]) => `${key}=${value}`);
|
|
12
21
|
const suffix = randomUUID().slice(0, 8);
|
|
13
22
|
const safeOperation = operation.replace(/[^A-Za-z0-9_.-]/g, '-');
|
|
14
23
|
return new Promise((resolve, reject) => {
|
|
@@ -29,10 +38,6 @@ export function launchFleetWorker(args, operation, configPath) {
|
|
|
29
38
|
], { timeout: 15_000 }, error => error ? reject(error) : resolve());
|
|
30
39
|
return;
|
|
31
40
|
}
|
|
32
|
-
const env = Object.fromEntries(inherited.map(value => {
|
|
33
|
-
const split = value.indexOf('=');
|
|
34
|
-
return [value.slice(0, split), value.slice(split + 1)];
|
|
35
|
-
}));
|
|
36
41
|
const child = spawn(process.execPath, cli, { detached: true, stdio: 'ignore', env });
|
|
37
42
|
child.once('error', reject);
|
|
38
43
|
child.once('spawn', () => { child.unref(); resolve(); });
|
package/dist/runner.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ResolvedRole } from './config.js';
|
|
2
2
|
import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
|
|
3
|
+
import { type DaemonGenerationProbe } from './daemon-recovery.js';
|
|
3
4
|
import { type Exec } from './exec.js';
|
|
4
5
|
import { RoleControlServer } from './session/control.js';
|
|
5
6
|
import type { AgentSession, ExitRecord, TurnResult } from './session/types.js';
|
|
@@ -15,6 +16,7 @@ export interface RunnerDeps {
|
|
|
15
16
|
log(line: string): void;
|
|
16
17
|
/** HTTP transport for the monitor's daemon long-poll (injectable for tests). */
|
|
17
18
|
fetch: FetchLike;
|
|
19
|
+
probeGeneration(env: NodeJS.ProcessEnv): Promise<DaemonGenerationProbe>;
|
|
18
20
|
/** Construct the supervisor mail monitor (injectable so tests stub it out). */
|
|
19
21
|
createMonitor(opts: MonitorOpts): MonitorHandle;
|
|
20
22
|
/** Construct trusted owner ingress (injectable for lifecycle tests). */
|
|
@@ -23,7 +25,7 @@ export interface RunnerDeps {
|
|
|
23
25
|
/** Neutral construction seam for deterministic runner lifecycle tests. */
|
|
24
26
|
startAgentSession(adapter: AgentSessionAdapter, options: AgentSessionStartOptions): Promise<AgentSession>;
|
|
25
27
|
/** Construct the authenticated role control route (injectable where sockets are unavailable). */
|
|
26
|
-
createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
|
|
28
|
+
createControlServer(stateDir: string, session: AgentSession, log: (line: string) => void): Pick<RoleControlServer, 'start' | 'close' | 'setFleetSpawner' | 'setFleetAuditor' | 'setOwnerChannel' | 'setConfigReloader' | 'setLoopManager'>;
|
|
27
29
|
/** Acquire the cross-process owner-channel binder lease before replacing the control socket. */
|
|
28
30
|
acquireOwnerBinder(stateDir: string, role: string, identity: string): Promise<OwnerBinderLease>;
|
|
29
31
|
/** Ask the still-authenticated predecessor to emit the fixed recovery notice. */
|
|
@@ -31,6 +33,11 @@ export interface RunnerDeps {
|
|
|
31
33
|
/** Lets a test (or a shutdown path) end the supervised restart loop. */
|
|
32
34
|
shouldStop?(): boolean;
|
|
33
35
|
}
|
|
36
|
+
export declare const SUPERVISOR_RECYCLE_REQUIRED = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
37
|
+
export declare class SupervisorRecycleRequiredError extends Error {
|
|
38
|
+
readonly code = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
39
|
+
constructor();
|
|
40
|
+
}
|
|
34
41
|
/** Environment injected only into the managed harness process. */
|
|
35
42
|
export declare function managedFleetProxyEnv(role: ResolvedRole, stateDir: string): Record<string, string>;
|
|
36
43
|
/**
|
|
@@ -176,4 +183,4 @@ export declare function runSupervised(name: string, opts?: {
|
|
|
176
183
|
allowResumeRotation?: boolean;
|
|
177
184
|
}, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
|
|
178
185
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
179
|
-
export declare function runTemp(name: string, deps?: Partial<RunnerDeps
|
|
186
|
+
export declare function runTemp(name: string, deps?: Partial<RunnerDeps>, attempt?: typeof runOnce): Promise<void>;
|
package/dist/runner.js
CHANGED
|
@@ -6,6 +6,8 @@ import { agentDir, stateRoot } from './paths.js';
|
|
|
6
6
|
import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolvePermissions, } from './config.js';
|
|
7
7
|
import { getAdapter } from './harness/registry.js';
|
|
8
8
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
|
+
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
+
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
9
11
|
import { realExec } from './exec.js';
|
|
10
12
|
import { resolveIsolation } from './isolation/policy.js';
|
|
11
13
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
@@ -24,6 +26,14 @@ import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, } from './fleet-prox
|
|
|
24
26
|
import { effectivePermissionMode } from './permissions.js';
|
|
25
27
|
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
26
28
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
29
|
+
export const SUPERVISOR_RECYCLE_REQUIRED = 'OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED';
|
|
30
|
+
export class SupervisorRecycleRequiredError extends Error {
|
|
31
|
+
code = SUPERVISOR_RECYCLE_REQUIRED;
|
|
32
|
+
constructor() {
|
|
33
|
+
super('owner channel requires a fresh supervisor process after unproven client quiescence');
|
|
34
|
+
this.name = 'SupervisorRecycleRequiredError';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
27
37
|
const defaultDeps = () => ({
|
|
28
38
|
exec: realExec,
|
|
29
39
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
@@ -38,6 +48,7 @@ const defaultDeps = () => ({
|
|
|
38
48
|
now: () => Date.now(),
|
|
39
49
|
log: line => process.stderr.write(line + '\n'),
|
|
40
50
|
fetch: (url, init) => globalThis.fetch(url, init),
|
|
51
|
+
probeGeneration: env => probeDaemonGeneration((url, init) => globalThis.fetch(url, init), env),
|
|
41
52
|
createMonitor: opts => createMonitor(opts),
|
|
42
53
|
createOwnerChannel: opts => new OwnerChannel(opts),
|
|
43
54
|
startAgentSession: (adapter, options) => adapter.start(options),
|
|
@@ -532,6 +543,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
532
543
|
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
533
544
|
deps: resolvedMonitorDeps,
|
|
534
545
|
}) : null;
|
|
546
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
547
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
548
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
535
549
|
if (monitor)
|
|
536
550
|
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
537
551
|
rmSync(exitFile, { force: true });
|
|
@@ -544,9 +558,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
544
558
|
let sessionClosed = false;
|
|
545
559
|
let ownerChannel;
|
|
546
560
|
let ownerBinder;
|
|
547
|
-
const pendingFleetSpawnNotices = [];
|
|
548
561
|
let loopManager;
|
|
549
562
|
let arbiter;
|
|
563
|
+
let recoveryController;
|
|
550
564
|
let reloadLoopConfig;
|
|
551
565
|
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
552
566
|
loop.name, loop.definitionHash, loop.promptHash,
|
|
@@ -607,21 +621,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
607
621
|
throw error;
|
|
608
622
|
}
|
|
609
623
|
control.setFleetSpawner(async (requested) => {
|
|
610
|
-
|
|
611
|
-
if (!role.owner_channel)
|
|
612
|
-
return event;
|
|
613
|
-
if (ownerChannel?.notifyFleetSpawn) {
|
|
614
|
-
try {
|
|
615
|
-
await ownerChannel.notifyFleetSpawn(event);
|
|
616
|
-
}
|
|
617
|
-
catch (error) {
|
|
618
|
-
deps.log(`[${name}] spawned-agent owner notice failed: `
|
|
619
|
-
+ `${error?.message ?? String(error)}`);
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
else
|
|
623
|
-
pendingFleetSpawnNotices.push(event);
|
|
624
|
-
return event;
|
|
624
|
+
return executeManagedSpawn(role, configPath, requested, deps.log);
|
|
625
625
|
});
|
|
626
626
|
resolvedMonitorDeps.delivery = {
|
|
627
627
|
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
@@ -743,18 +743,14 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
743
743
|
throw new Error(`[${name}] owner channel failed to start: `
|
|
744
744
|
+ `${error?.message ?? String(error)}`);
|
|
745
745
|
}
|
|
746
|
-
for (const event of pendingFleetSpawnNotices.splice(0)) {
|
|
747
|
-
try {
|
|
748
|
-
await ownerChannel.notifyFleetSpawn?.(event);
|
|
749
|
-
}
|
|
750
|
-
catch (error) {
|
|
751
|
-
deps.log(`[${name}] deferred spawned-agent owner notice failed: `
|
|
752
|
-
+ `${error?.message ?? String(error)}`);
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
746
|
}
|
|
756
|
-
if (ownerChannel)
|
|
747
|
+
if (ownerChannel) {
|
|
757
748
|
control.setOwnerChannel(ownerChannel);
|
|
749
|
+
control.setFleetAuditor({
|
|
750
|
+
begin: (requestId, argv) => ownerChannel.beginFleetCommandAudit(requestId, argv),
|
|
751
|
+
finish: input => ownerChannel.finishFleetCommandAudit(input),
|
|
752
|
+
});
|
|
753
|
+
}
|
|
758
754
|
reloadLoopConfig = async () => {
|
|
759
755
|
const nextRole = findRole(loadConfig(configPath), name);
|
|
760
756
|
const definitions = nextRole.loops ?? [];
|
|
@@ -801,16 +797,49 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
801
797
|
// The monitor loop lives exactly as long as the session: it starts once the
|
|
802
798
|
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
803
799
|
monitorLoop ??= monitor?.run(pid);
|
|
800
|
+
recoveryController = new RoleRecoveryController({
|
|
801
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
802
|
+
log: deps.log,
|
|
803
|
+
recoverAgent: async () => {
|
|
804
|
+
const evidence = await recoverAgentIdentity(arbiter, role.identity);
|
|
805
|
+
return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
|
|
806
|
+
},
|
|
807
|
+
recoverOwner: async (epoch) => {
|
|
808
|
+
if (!ownerChannel?.recover)
|
|
809
|
+
return { ok: true };
|
|
810
|
+
try {
|
|
811
|
+
await ownerChannel.recover(epoch);
|
|
812
|
+
return { ok: true };
|
|
813
|
+
}
|
|
814
|
+
catch (error) {
|
|
815
|
+
return { ok: false, reason: error instanceof Error
|
|
816
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
817
|
+
}
|
|
818
|
+
},
|
|
819
|
+
});
|
|
804
820
|
const start = deps.now();
|
|
805
821
|
let nextLoopReloadAt = deps.now() + 30_000;
|
|
806
822
|
let nextIdentityPollAt = deps.now();
|
|
823
|
+
let nextDaemonProbeAt = deps.now();
|
|
807
824
|
let lastReloadError = '';
|
|
808
825
|
let identityObserved = false;
|
|
809
826
|
let identityAbsentSince;
|
|
810
827
|
let retirementReason;
|
|
828
|
+
let supervisorRecycleRequired = false;
|
|
811
829
|
while (sessionHandle.isAlive()) {
|
|
812
830
|
await deps.sleep(temp ? 500 : 2000);
|
|
813
831
|
const now = deps.now();
|
|
832
|
+
if (now >= nextDaemonProbeAt) {
|
|
833
|
+
nextDaemonProbeAt = now + 2_000;
|
|
834
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
835
|
+
if (observation.kind === 'lost')
|
|
836
|
+
recoveryController.noteLoss(observation.reason);
|
|
837
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
838
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
839
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
840
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
841
|
+
}
|
|
842
|
+
}
|
|
814
843
|
if (temp && deps.shouldStop?.()) {
|
|
815
844
|
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
816
845
|
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
@@ -861,6 +890,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
861
890
|
control?.setLoopManager(undefined);
|
|
862
891
|
await loopManager.stop();
|
|
863
892
|
}
|
|
893
|
+
// Let already-settled path operations publish their aggregate result before
|
|
894
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
895
|
+
await Promise.resolve();
|
|
896
|
+
await Promise.resolve();
|
|
897
|
+
recoveryController?.cancel();
|
|
864
898
|
control?.setConfigReloader(undefined);
|
|
865
899
|
// Close the authenticated control route before releasing the binder lease;
|
|
866
900
|
// otherwise the predecessor can unlink the replacement's new socket.
|
|
@@ -871,7 +905,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
871
905
|
}
|
|
872
906
|
if (ownerChannel)
|
|
873
907
|
await ownerChannel.close();
|
|
874
|
-
|
|
908
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
909
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
910
|
+
supervisorRecycleRequired = true;
|
|
911
|
+
}
|
|
912
|
+
else
|
|
913
|
+
ownerBinder?.release();
|
|
875
914
|
if (monitor) {
|
|
876
915
|
monitor.stop();
|
|
877
916
|
await monitorLoop;
|
|
@@ -920,6 +959,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
920
959
|
}
|
|
921
960
|
else
|
|
922
961
|
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
962
|
+
if (supervisorRecycleRequired)
|
|
963
|
+
throw new SupervisorRecycleRequiredError();
|
|
923
964
|
return {
|
|
924
965
|
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
925
966
|
...(retirementReason ? { retirementReason } : {}),
|
|
@@ -996,6 +1037,16 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
996
1037
|
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
997
1038
|
}
|
|
998
1039
|
catch (e) {
|
|
1040
|
+
if (e instanceof SupervisorRecycleRequiredError) {
|
|
1041
|
+
writeRestartLedger(dir, {
|
|
1042
|
+
...ledger,
|
|
1043
|
+
lastReason: SUPERVISOR_RECYCLE_REQUIRED,
|
|
1044
|
+
nextDelayMs: 0,
|
|
1045
|
+
updatedAt: stamp(),
|
|
1046
|
+
});
|
|
1047
|
+
deps.log(`[${name}] supervisor recycle required: ${SUPERVISOR_RECYCLE_REQUIRED}`);
|
|
1048
|
+
throw e;
|
|
1049
|
+
}
|
|
999
1050
|
// A session that could not even start is an immediate failure like any
|
|
1000
1051
|
// other; it must count, or an unstartable role loops forever.
|
|
1001
1052
|
result = {
|
|
@@ -1089,7 +1140,7 @@ function fastFailSecsFor(name, configPath) {
|
|
|
1089
1140
|
}
|
|
1090
1141
|
}
|
|
1091
1142
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
1092
|
-
export async function runTemp(name, deps = {}) {
|
|
1143
|
+
export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
1093
1144
|
const dir = agentDir(name, true);
|
|
1094
1145
|
await markTempSupervisorActive(dir);
|
|
1095
1146
|
let signal;
|
|
@@ -1100,7 +1151,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1100
1151
|
let result;
|
|
1101
1152
|
let failure;
|
|
1102
1153
|
try {
|
|
1103
|
-
result = await
|
|
1154
|
+
result = await attempt(name, { temp: true }, {
|
|
1104
1155
|
...deps,
|
|
1105
1156
|
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1106
1157
|
});
|
|
@@ -1114,6 +1165,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1114
1165
|
const requested = requestedTempStopReason(dir);
|
|
1115
1166
|
const reason = requested
|
|
1116
1167
|
?? result?.retirementReason
|
|
1168
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1117
1169
|
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1118
1170
|
// A service-manager stop can make the child connection close before the
|
|
1119
1171
|
// runner reaches its normal loop. That is still a successful requested
|