@ours.network/fleet 1.0.5 → 1.1.0-nightly.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -133
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/capabilities.d.ts +0 -1
- package/dist/application/capabilities.js +3 -13
- package/dist/application/fleet-query-service.d.ts +0 -3
- package/dist/application/fleet-query-service.js +14 -25
- package/dist/application/role-creation-service.d.ts +5 -24
- package/dist/application/role-creation-service.js +27 -67
- package/dist/application/role-removal-service.js +6 -3
- package/dist/application/role-repository.d.ts +1 -2
- package/dist/application/role-repository.js +7 -7
- package/dist/application/session-control.d.ts +0 -12
- package/dist/application/session-control.js +0 -42
- package/dist/application/session-mutations.d.ts +5 -5
- package/dist/application/task-room-service.d.ts +1 -1
- package/dist/application/task-room-service.js +154 -16
- package/dist/application/types.d.ts +1 -8
- package/dist/briefing.js +12 -11
- package/dist/build-info.json +5 -5
- package/dist/cli.js +143 -63
- package/dist/config-yaml.d.ts +1 -1
- package/dist/config.d.ts +66 -6
- package/dist/config.js +398 -43
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +2 -2
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +103 -127
- package/dist/doctor.js +18 -11
- package/dist/fleet-command-audit.d.ts +160 -0
- package/dist/fleet-command-audit.js +433 -0
- package/dist/fleet-proxy.d.ts +3 -1
- package/dist/fleet-proxy.js +30 -8
- package/dist/generated-agent-source.d.ts +9 -0
- package/dist/generated-agent-source.js +53 -0
- package/dist/harness/acp-session-transport.d.ts +8 -0
- package/dist/harness/acp-session-transport.js +3 -0
- package/dist/harness/agent-session.d.ts +51 -0
- package/dist/harness/agent-session.js +1 -0
- package/dist/harness/claude-code-session.d.ts +24 -0
- package/dist/harness/claude-code-session.js +36 -0
- package/dist/harness/claude-code.d.ts +2 -1
- package/dist/harness/claude-code.js +41 -81
- package/dist/harness/codex-session.d.ts +24 -0
- package/dist/harness/codex-session.js +36 -0
- package/dist/harness/codex.d.ts +2 -1
- package/dist/harness/codex.js +55 -45
- package/dist/harness/registry.js +4 -0
- package/dist/harness/types.d.ts +6 -38
- package/dist/index.d.ts +4 -6
- package/dist/index.js +2 -4
- package/dist/init-guidance.d.ts +1 -0
- package/dist/init-guidance.js +6 -0
- package/dist/isolation/resources.d.ts +3 -7
- package/dist/isolation/resources.js +3 -7
- package/dist/model-env.d.ts +4 -6
- package/dist/model-env.js +5 -8
- package/dist/monitor.d.ts +1 -24
- package/dist/monitor.js +5 -140
- package/dist/ops.js +4 -2
- package/dist/owner-channel/channel.d.ts +58 -2
- package/dist/owner-channel/channel.js +269 -29
- package/dist/owner-channel/commands.d.ts +2 -0
- package/dist/owner-channel/commands.js +31 -13
- package/dist/owner-channel/state.d.ts +2 -0
- package/dist/owner-channel/state.js +8 -2
- package/dist/preset-bootstrap.d.ts +10 -0
- package/dist/preset-bootstrap.js +85 -0
- package/dist/reports/artifact.d.ts +10 -0
- package/dist/reports/artifact.js +32 -0
- package/dist/reports/index.d.ts +5 -0
- package/dist/reports/index.js +5 -0
- package/dist/reports/render.d.ts +6 -0
- package/dist/reports/render.js +240 -0
- package/dist/reports/service.d.ts +42 -0
- package/dist/reports/service.js +57 -0
- package/dist/reports/tasks.d.ts +52 -0
- package/dist/reports/tasks.js +177 -0
- package/dist/reports/types.d.ts +315 -0
- package/dist/reports/types.js +6 -0
- package/dist/resolved-plan.d.ts +3 -2
- package/dist/resolved-plan.js +21 -5
- package/dist/rooms-tasks/cli.js +225 -40
- package/dist/rooms-tasks/config.js +24 -13
- package/dist/rooms-tasks/external-worker.d.ts +1 -0
- package/dist/rooms-tasks/external-worker.js +14 -9
- package/dist/rooms-tasks/provision.js +98 -36
- package/dist/rooms-tasks/templates.d.ts +0 -1
- package/dist/rooms-tasks/templates.js +9 -76
- package/dist/rooms-tasks/types.d.ts +10 -16
- package/dist/rooms-tasks/types.js +1 -4
- package/dist/runner.d.ts +13 -16
- package/dist/runner.js +110 -122
- package/dist/sensitive-config.d.ts +2 -0
- package/dist/sensitive-config.js +4 -0
- package/dist/session/acp.d.ts +7 -2
- package/dist/session/acp.js +29 -6
- package/dist/session/activity.d.ts +2 -3
- package/dist/session/activity.js +2 -3
- package/dist/session/arbiter.d.ts +4 -4
- package/dist/session/control.d.ts +17 -4
- package/dist/session/control.js +35 -0
- package/dist/session/types.d.ts +11 -5
- package/dist/spawn.d.ts +8 -33
- package/dist/spawn.js +53 -213
- package/dist/supervisor/none.d.ts +1 -1
- package/dist/supervisor/none.js +2 -2
- package/dist/temp-lifecycle.d.ts +1 -1
- package/dist/tmux.d.ts +1 -14
- package/dist/tmux.js +1 -51
- package/dist/watchdog/config.d.ts +10 -7
- package/dist/watchdog/config.js +30 -18
- package/dist/watchdog/run.js +12 -36
- package/dist/web/auth.d.ts +1 -1
- package/dist/web/fleet-config-service.d.ts +11 -8
- package/dist/web/fleet-config-service.js +384 -183
- package/dist/web/runtime.js +4 -15
- package/dist/web/server.d.ts +1 -3
- package/dist/web/server.js +3 -14
- package/dist/web/topology-promote.js +26 -19
- package/dist/web/topology.js +2 -2
- package/dist/web/yaml-document-edit.js +2 -0
- package/dist/web-app/assets/index-59GF-gsZ.css +1 -0
- package/dist/web-app/assets/index-BbTc5KtZ.js +10 -0
- package/dist/web-app/index.html +2 -2
- package/examples/fleet/agents/Alice.yaml +11 -0
- package/examples/fleet/agents/FleetCoordinator.yaml +7 -0
- package/examples/fleet/brains/claude-default.yaml +5 -0
- package/examples/fleet/roles/coordinator.yaml +5 -0
- package/examples/fleet/roles/developer.yaml +3 -0
- package/examples/fleet.yaml +74 -0
- package/package.json +5 -8
- package/presets/fleet/agents/Agent.yaml +3 -0
- package/presets/fleet/agents/Architect.yaml +3 -0
- package/presets/fleet/agents/Critic.yaml +3 -0
- package/presets/fleet/agents/Developer.yaml +3 -0
- package/presets/fleet/agents/Secretary.yaml +3 -0
- package/presets/fleet/agents/Tester.yaml +3 -0
- package/presets/fleet/brains/claude-default.yaml +5 -0
- package/presets/fleet/roles/Agent.yaml +6 -0
- package/presets/fleet/roles/Architect.yaml +6 -0
- package/presets/fleet/roles/Critic.yaml +6 -0
- package/presets/fleet/roles/Developer.yaml +6 -0
- package/presets/fleet/roles/Secretary.yaml +6 -0
- package/presets/fleet/roles/Tester.yaml +6 -0
- package/presets/fleet/room_templates/pair.yaml +11 -0
- package/presets/fleet/room_templates/single.yaml +8 -0
- package/presets/fleet/room_templates/team.yaml +12 -0
- package/presets/fleet.yaml +7 -0
- package/presets/manifest.json +5 -0
- package/dist/session/tmux.d.ts +0 -28
- package/dist/session/tmux.js +0 -80
- package/dist/web/terminal/bridge.d.ts +0 -27
- package/dist/web/terminal/bridge.js +0 -317
- package/dist/web-app/assets/TerminalView-C_G1ID2P.js +0 -9
- package/dist/web-app/assets/index-BCBK78hw.js +0 -10
- package/dist/web-app/assets/index-DuC-xnX4.css +0 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
-
import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
|
|
3
|
+
import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { canonicalCid } from '../config.js';
|
|
6
|
+
import { DAEMON_RECOVERY_DEADLINE_MS } from '../daemon-recovery.js';
|
|
6
7
|
import { replaceFileAtomically } from '../atomic-file.js';
|
|
7
8
|
import { TaskRoomApplicationService } from '../application/task-room-service.js';
|
|
8
9
|
import { RoleLifecycleService } from '../application/role-command-service.js';
|
|
@@ -14,6 +15,7 @@ import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, } from '../session/t
|
|
|
14
15
|
import { VERSION } from '../version.js';
|
|
15
16
|
import { renderMarkdownFailure, renderMarkdownResult, roomStatus, taskStatus, } from '../rooms-tasks/markdown.js';
|
|
16
17
|
import { dispatchOwnerCommand, fleetCliOps, isOwnerCommandText, } from './commands.js';
|
|
18
|
+
import { FleetCommandAuditStore, lifecycleEventDigestBasis, renderFleetLifecycleEvent, } from '../fleet-command-audit.js';
|
|
17
19
|
import { OURS_BOUND_ELSEWHERE, OursSdkClient, oursErrorCode, } from './ours-client.js';
|
|
18
20
|
import { ownerNotices, } from './notices.js';
|
|
19
21
|
import { DuplicateSendError, OwnerAuthorizationState, OwnerChannelState, OwnerConversationState, } from './state.js';
|
|
@@ -21,6 +23,24 @@ import { OwnerTaskState, ownerTaskAuditId, ownerTaskDigest, } from './tasks.js';
|
|
|
21
23
|
import { AttachmentRecoveryState, admitAttachments, cleanupAttachmentRoot, parseIncomingAttachments, parseRetrievedAttachments, prepareAttachmentDirectory, recoveredAttachment, removeRequestDirectory, safeField, validateAttachmentSelection, validateAttachmentRelaySelection, writeRecoveredAttachment, } from './attachments.js';
|
|
22
24
|
import { acquireOwnerBinderLease, OWNER_BIND_HANDOFF_TIMEOUT_MS, } from './binder.js';
|
|
23
25
|
import { MessageRecoveryState } from './message-recovery.js';
|
|
26
|
+
export const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
|
|
27
|
+
export const OWNER_RECOVERY_DEGRADED = 'OWNER_RECOVERY_DEGRADED';
|
|
28
|
+
export class OwnerRecoveryDegradedError extends Error {
|
|
29
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
30
|
+
constructor() {
|
|
31
|
+
super('owner channel recovery is degraded');
|
|
32
|
+
this.name = 'OwnerRecoveryDegradedError';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export class OwnerRecoveryTimeoutError extends Error {
|
|
36
|
+
stage;
|
|
37
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
38
|
+
constructor(stage) {
|
|
39
|
+
super(`owner channel recovery timed out at ${stage}`);
|
|
40
|
+
this.stage = stage;
|
|
41
|
+
this.name = 'OwnerRecoveryTimeoutError';
|
|
42
|
+
}
|
|
43
|
+
}
|
|
24
44
|
const OWNER_UPDATE_MIN_INTERVAL_MS = 5_000;
|
|
25
45
|
const OWNER_UPDATE_MAX_COUNT = 20;
|
|
26
46
|
const OWNER_UPDATE_MAX_CHARS = 280;
|
|
@@ -76,11 +96,22 @@ export class OwnerChannel {
|
|
|
76
96
|
completionTasks = new Set();
|
|
77
97
|
activeRequests = new Map();
|
|
78
98
|
managementTail = Promise.resolve();
|
|
99
|
+
recoveryEpoch;
|
|
100
|
+
recoveryTask;
|
|
101
|
+
recoveryToken = 0;
|
|
102
|
+
recoveryQueued = 0;
|
|
103
|
+
watchGeneration = 0;
|
|
104
|
+
/** A timed-out client operation that must settle before replacement is safe. */
|
|
105
|
+
recoveryQuiescence;
|
|
106
|
+
releaseSafe = true;
|
|
79
107
|
ready = false;
|
|
108
|
+
startedOnce = false;
|
|
109
|
+
shutdownDegraded = false;
|
|
80
110
|
binder;
|
|
81
111
|
binderOwnedInternally = false;
|
|
82
112
|
fleetOps;
|
|
83
113
|
prepareRestart;
|
|
114
|
+
commandAudits;
|
|
84
115
|
constructor(options) {
|
|
85
116
|
this.options = options;
|
|
86
117
|
this.client = options.client ?? new OursSdkClient(options.env, line => options.log(`[${options.role}] owner channel ${line}`));
|
|
@@ -96,6 +127,7 @@ export class OwnerChannel {
|
|
|
96
127
|
await lifecycle.prepareRestart({ roleIds: [role], mode });
|
|
97
128
|
});
|
|
98
129
|
this.state = new OwnerChannelState(join(options.stateDir, '.owner-channel-state.json'));
|
|
130
|
+
this.commandAudits = new FleetCommandAuditStore(join(options.stateDir, '.owner-channel-command-audit.json'));
|
|
99
131
|
this.authorizations = new OwnerAuthorizationState(join(options.stateDir, '.owner-channel-owners.json'), options.config.owners);
|
|
100
132
|
this.conversations = new OwnerConversationState(join(options.stateDir, '.owner-channel-conversations.json'));
|
|
101
133
|
this.tasks = new OwnerTaskState(join(options.stateDir, '.owner-channel-tasks.json'));
|
|
@@ -124,6 +156,7 @@ export class OwnerChannel {
|
|
|
124
156
|
}
|
|
125
157
|
async start() {
|
|
126
158
|
this.stopping = false;
|
|
159
|
+
this.shutdownDegraded = false;
|
|
127
160
|
this.binderOwnedInternally = !this.options.binderLease;
|
|
128
161
|
this.binder = this.options.binderLease ?? await acquireOwnerBinderLease(this.options.stateDir, this.options.role, this.options.config.identity, this.options.binderDeps);
|
|
129
162
|
try {
|
|
@@ -164,11 +197,12 @@ export class OwnerChannel {
|
|
|
164
197
|
this.attachmentRecovery.cleanup(Date.now(), this.attachmentConfig.retention_ms);
|
|
165
198
|
void cleanupAttachmentRoot(this.attachmentRoot, Date.now(), this.attachmentConfig.retention_ms).catch(error => this.logError('attachment crash cleanup failed', error));
|
|
166
199
|
}
|
|
200
|
+
this.startedOnce = true;
|
|
167
201
|
this.ready = true;
|
|
168
202
|
// Do not make role startup wait for an old owner request to finish a turn.
|
|
169
203
|
// watchLoop itself drains before every establishment, including this first
|
|
170
204
|
// one, so there is no drain-to-tip race.
|
|
171
|
-
this.watchTask = this.watchLoop();
|
|
205
|
+
this.watchTask = this.watchLoop(++this.watchGeneration);
|
|
172
206
|
}
|
|
173
207
|
drain() {
|
|
174
208
|
this.drainRequested = true;
|
|
@@ -183,46 +217,239 @@ export class OwnerChannel {
|
|
|
183
217
|
return this.drainTask;
|
|
184
218
|
}
|
|
185
219
|
async close() {
|
|
220
|
+
if (this.recoveryTask)
|
|
221
|
+
this.shutdownDegraded = true;
|
|
186
222
|
this.stopping = true;
|
|
187
223
|
this.ready = false;
|
|
188
|
-
this.
|
|
189
|
-
|
|
224
|
+
this.recoveryToken++;
|
|
225
|
+
this.watchGeneration++;
|
|
226
|
+
const staleWatch = this.watchTask;
|
|
190
227
|
this.watchTask = undefined;
|
|
191
|
-
|
|
228
|
+
this.watchAbort?.abort();
|
|
229
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
230
|
+
const deadlineAt = now()
|
|
231
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
232
|
+
let disposed = false;
|
|
192
233
|
try {
|
|
193
|
-
await this.
|
|
234
|
+
await this.recoveryStage('shutdown_quiescence', Promise.all([
|
|
235
|
+
staleWatch?.catch(error => this.logError('watch shutdown failed', error)),
|
|
236
|
+
this.drainTask?.catch(error => this.logError('drain shutdown failed', error)),
|
|
237
|
+
this.managementTail.then(() => undefined),
|
|
238
|
+
this.recoveryQuiescence,
|
|
239
|
+
]), deadlineAt);
|
|
240
|
+
await this.recoveryStage('shutdown_close', this.client.close(), deadlineAt);
|
|
241
|
+
disposed = true;
|
|
242
|
+
this.writeShutdownState('closed', 'OWNER_SHUTDOWN_CLOSED', now());
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
this.shutdownDegraded = true;
|
|
246
|
+
this.releaseSafe = false;
|
|
247
|
+
const stage = error instanceof OwnerRecoveryTimeoutError ? error.stage : 'shutdown_unknown';
|
|
248
|
+
this.writeShutdownState('degraded', `OWNER_${stage.toUpperCase()}_TIMEOUT`, now());
|
|
249
|
+
this.options.log(`[${this.options.role}] owner shutdown degraded stage=${stage}`);
|
|
194
250
|
}
|
|
195
251
|
finally {
|
|
196
|
-
if (this.binderOwnedInternally)
|
|
252
|
+
if (disposed && this.binderOwnedInternally)
|
|
197
253
|
this.binder?.release();
|
|
198
|
-
|
|
254
|
+
if (disposed)
|
|
255
|
+
this.binder = undefined;
|
|
199
256
|
}
|
|
200
257
|
}
|
|
258
|
+
binderReleaseSafe() { return this.releaseSafe; }
|
|
201
259
|
manage(request) {
|
|
202
|
-
|
|
203
|
-
this.managementTail = run.then(() => undefined, () => undefined);
|
|
204
|
-
return run;
|
|
260
|
+
return this.queueManagement(() => this.manageNow(request));
|
|
205
261
|
}
|
|
206
262
|
notifyFleetSpawn(event) {
|
|
207
|
-
|
|
263
|
+
return this.queueManagement(async () => {
|
|
208
264
|
if (!this.ready || this.stopping)
|
|
209
265
|
throw new Error('owner-channel MCP client is unavailable');
|
|
210
|
-
const model = event.model ? `, model ${event.model}` : '';
|
|
211
|
-
const monitorPolicy = event.monitor.interrupt === true
|
|
212
|
-
? ' with interruption'
|
|
213
|
-
: event.monitor.interrupt === 'after_tool' ? ' with after-tool steering' : '';
|
|
214
|
-
const monitor = `${event.monitor.mode} monitor${monitorPolicy}`;
|
|
215
266
|
const permission = event.permissionMode
|
|
216
|
-
?
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
267
|
+
? `${event.permissionMode.fleetMode}/${event.permissionMode.nativeMode}` : undefined;
|
|
268
|
+
await this.sendProactiveMessage(renderFleetLifecycleEvent({ kind: 'agent_started', eventId: event.creationActionId,
|
|
269
|
+
id: event.role, name: event.role,
|
|
270
|
+
lifetime: event.lifetime, brain: event.brainSummary, role: event.roleSummary,
|
|
271
|
+
harness: event.harness, session: event.session, model: event.model,
|
|
272
|
+
permissions: permission, parent: event.caller, actionId: event.creationActionId,
|
|
273
|
+
inherited: event.inherited }), `fleet-spawn\0${event.creationActionId}`, 0);
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
beginFleetCommandAudit(requestId, argv) {
|
|
277
|
+
const run = this.managementTail.then(async () => {
|
|
278
|
+
let attempt = this.commandAudits.begin(requestId, this.options.role, argv);
|
|
279
|
+
if (attempt.invocation === 'delivered')
|
|
280
|
+
return attempt;
|
|
281
|
+
if (attempt.invocation === 'uncertain')
|
|
282
|
+
return attempt;
|
|
283
|
+
this.options.log(`[${this.options.role}] fleet proxy command ${attempt.correlationId} `
|
|
284
|
+
+ `route=${attempt.classification.route} decision=${attempt.classification.decision}`);
|
|
285
|
+
attempt = this.commandAudits.invocation(attempt.correlationId, this.options.role, 'delivered');
|
|
286
|
+
return attempt;
|
|
287
|
+
});
|
|
288
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
289
|
+
return run;
|
|
290
|
+
}
|
|
291
|
+
finishFleetCommandAudit(input) {
|
|
292
|
+
const run = this.managementTail.then(async () => {
|
|
293
|
+
let attempt = this.commandAudits.finish(input.correlationId, this.options.role, {
|
|
294
|
+
class: input.class, effect: input.effect,
|
|
295
|
+
...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
|
|
296
|
+
...(input.resourceIds ? { resourceIds: input.resourceIds } : {}),
|
|
297
|
+
...(input.presentations ? { presentations: input.presentations } : {}),
|
|
298
|
+
});
|
|
299
|
+
if (attempt.outcome?.delivery === 'delivered')
|
|
300
|
+
return attempt;
|
|
301
|
+
if (attempt.outcome?.delivery === 'uncertain')
|
|
302
|
+
return attempt;
|
|
303
|
+
let uncertain = false;
|
|
304
|
+
try {
|
|
305
|
+
for (const event of attempt.outcome?.presentations ?? []) {
|
|
306
|
+
try {
|
|
307
|
+
await this.sendProactiveMessage(renderFleetLifecycleEvent(event), `fleet-lifecycle\0${lifecycleEventDigestBasis(event)}`, 0);
|
|
308
|
+
}
|
|
309
|
+
catch (error) {
|
|
310
|
+
if (error instanceof DuplicateSendError) {
|
|
311
|
+
if (error.status !== 'delivered')
|
|
312
|
+
uncertain = true;
|
|
313
|
+
this.options.log(`[${this.options.role}] duplicate Fleet lifecycle event suppressed`);
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
throw error;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
catch (error) {
|
|
321
|
+
uncertain = true;
|
|
322
|
+
this.logError(`fleet lifecycle delivery uncertain correlation=${attempt.correlationId}`, error);
|
|
323
|
+
}
|
|
324
|
+
attempt = this.commandAudits.outcome(attempt.correlationId, this.options.role, uncertain ? 'uncertain' : 'delivered');
|
|
325
|
+
return attempt;
|
|
222
326
|
});
|
|
223
327
|
this.managementTail = run.then(() => undefined, () => undefined);
|
|
224
328
|
return run;
|
|
225
329
|
}
|
|
330
|
+
recover(epoch) {
|
|
331
|
+
if (this.stopping)
|
|
332
|
+
return Promise.reject(new Error('owner channel is stopping'));
|
|
333
|
+
if (this.recoveryEpoch === epoch && this.recoveryTask)
|
|
334
|
+
return this.recoveryTask;
|
|
335
|
+
this.recoveryEpoch = epoch;
|
|
336
|
+
this.ready = false;
|
|
337
|
+
const token = ++this.recoveryToken;
|
|
338
|
+
const watchGeneration = ++this.watchGeneration;
|
|
339
|
+
const staleWatch = this.watchTask;
|
|
340
|
+
this.watchTask = undefined;
|
|
341
|
+
this.watchAbort?.abort();
|
|
342
|
+
const superseded = () => this.stopping || token !== this.recoveryToken;
|
|
343
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
344
|
+
const deadlineAt = now()
|
|
345
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
346
|
+
const run = this.managementTail.then(async () => {
|
|
347
|
+
if (superseded())
|
|
348
|
+
throw new Error('owner recovery epoch superseded');
|
|
349
|
+
if (this.recoveryQuiescence) {
|
|
350
|
+
const debt = this.recoveryQuiescence;
|
|
351
|
+
await this.recoveryStage('prior_quiescence', debt, deadlineAt);
|
|
352
|
+
if (this.recoveryQuiescence === debt)
|
|
353
|
+
this.recoveryQuiescence = undefined;
|
|
354
|
+
}
|
|
355
|
+
// Quiesce every stale-client reader before replacing its transport. A
|
|
356
|
+
// watch can be inside the shared drain task when its abort lands.
|
|
357
|
+
await this.recoveryStage('stale_quiescence', Promise.all([
|
|
358
|
+
staleWatch?.catch(error => this.logError('stale watch shutdown failed', error)),
|
|
359
|
+
this.drainTask?.catch(error => this.logError('stale drain shutdown failed', error)),
|
|
360
|
+
]), deadlineAt);
|
|
361
|
+
if (superseded())
|
|
362
|
+
throw new Error('owner recovery epoch superseded');
|
|
363
|
+
await this.recoveryStage('close', this.client.close(), deadlineAt);
|
|
364
|
+
if (superseded())
|
|
365
|
+
throw new Error('owner recovery epoch superseded');
|
|
366
|
+
try {
|
|
367
|
+
await this.recoveryStage('start', this.client.start(), deadlineAt);
|
|
368
|
+
if (superseded())
|
|
369
|
+
throw new Error('owner recovery epoch superseded');
|
|
370
|
+
await this.recoveryStage('bind', this.client.bindIdentity(this.options.config.identity), deadlineAt);
|
|
371
|
+
if (superseded())
|
|
372
|
+
throw new Error('owner recovery epoch superseded');
|
|
373
|
+
// Journal-aware drain is the only operation allowed to mark owner mail
|
|
374
|
+
// read. The final listing is a typed, read-only channel probe.
|
|
375
|
+
await this.recoveryStage('drain', this.drain(), deadlineAt);
|
|
376
|
+
await this.recoveryStage('list', this.client.listIncomingMessages(), deadlineAt);
|
|
377
|
+
if (superseded())
|
|
378
|
+
throw new Error('owner recovery epoch superseded');
|
|
379
|
+
this.ready = true;
|
|
380
|
+
this.watchTask = this.watchLoop(watchGeneration);
|
|
381
|
+
}
|
|
382
|
+
catch (error) {
|
|
383
|
+
this.ready = false;
|
|
384
|
+
if (token === this.recoveryToken)
|
|
385
|
+
this.recoveryToken++;
|
|
386
|
+
// A timed-out operation may still be using this client. Never overlap
|
|
387
|
+
// its disposal; the recorded quiescence debt gates the next retry.
|
|
388
|
+
if (!this.recoveryQuiescence) {
|
|
389
|
+
try {
|
|
390
|
+
await this.recoveryStage('close_after_failure', this.client.close(), deadlineAt);
|
|
391
|
+
}
|
|
392
|
+
catch (closeError) {
|
|
393
|
+
this.logError('recovery client close failed', closeError);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
throw error;
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
400
|
+
const task = run.finally(() => {
|
|
401
|
+
if (this.recoveryTask === task)
|
|
402
|
+
this.recoveryTask = undefined;
|
|
403
|
+
});
|
|
404
|
+
this.recoveryTask = task;
|
|
405
|
+
return task;
|
|
406
|
+
}
|
|
407
|
+
recoveryStage(stage, operation, deadlineAt) {
|
|
408
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
409
|
+
const remaining = Math.max(0, deadlineAt - now());
|
|
410
|
+
const setTimer = this.options.recoveryDeps?.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
411
|
+
const clearTimer = this.options.recoveryDeps?.clearTimer ?? (timer => clearTimeout(timer));
|
|
412
|
+
let timer;
|
|
413
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
414
|
+
timer = setTimer(() => reject(new OwnerRecoveryTimeoutError(stage)), remaining);
|
|
415
|
+
});
|
|
416
|
+
return Promise.race([operation, timeout]).catch(error => {
|
|
417
|
+
if (error instanceof OwnerRecoveryTimeoutError) {
|
|
418
|
+
const debt = operation.then(() => undefined, () => undefined);
|
|
419
|
+
this.recoveryQuiescence = debt;
|
|
420
|
+
}
|
|
421
|
+
throw error;
|
|
422
|
+
}).finally(() => clearTimer(timer));
|
|
423
|
+
}
|
|
424
|
+
writeShutdownState(state, reason, at) {
|
|
425
|
+
try {
|
|
426
|
+
replaceFileAtomically(join(this.options.stateDir, '.owner-channel-shutdown.json'), JSON.stringify({ version: 1, state, reason, at: new Date(at).toISOString() }, null, 2) + '\n');
|
|
427
|
+
}
|
|
428
|
+
catch (error) {
|
|
429
|
+
this.logError('shutdown status write failed', error);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
queueManagement(operation) {
|
|
433
|
+
const deferred = this.recoveryTask !== undefined && !this.ready;
|
|
434
|
+
if (!this.ready && !deferred)
|
|
435
|
+
return Promise.reject(!this.startedOnce || (this.stopping && !this.shutdownDegraded)
|
|
436
|
+
? new Error('owner-channel MCP client is unavailable')
|
|
437
|
+
: new OwnerRecoveryDegradedError());
|
|
438
|
+
if (deferred && this.recoveryQueued >= OWNER_RECOVERY_QUEUE_CAPACITY)
|
|
439
|
+
return Promise.reject(new Error('owner recovery queue is full'));
|
|
440
|
+
if (deferred)
|
|
441
|
+
this.recoveryQueued++;
|
|
442
|
+
const run = this.managementTail.then(() => {
|
|
443
|
+
if (!this.ready || this.stopping)
|
|
444
|
+
throw new OwnerRecoveryDegradedError();
|
|
445
|
+
return operation();
|
|
446
|
+
});
|
|
447
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
448
|
+
return run.finally(() => {
|
|
449
|
+
if (deferred)
|
|
450
|
+
this.recoveryQueued = Math.max(0, this.recoveryQueued - 1);
|
|
451
|
+
});
|
|
452
|
+
}
|
|
226
453
|
async manageNow(request) {
|
|
227
454
|
if (!this.ready || this.stopping)
|
|
228
455
|
throw new Error('owner-channel MCP client is unavailable');
|
|
@@ -989,6 +1216,7 @@ export class OwnerChannel {
|
|
|
989
1216
|
*/
|
|
990
1217
|
async handleCommand(sender, text, wireId) {
|
|
991
1218
|
const ctx = {
|
|
1219
|
+
authenticatedCid: sender.id,
|
|
992
1220
|
role: this.options.role,
|
|
993
1221
|
harness: this.options.harness,
|
|
994
1222
|
version: VERSION,
|
|
@@ -1054,6 +1282,18 @@ export class OwnerChannel {
|
|
|
1054
1282
|
recentEvents: limit => this.options.session.eventsSince(0).slice(-limit),
|
|
1055
1283
|
readWorklogTail: maxChars => this.readWorklogTail(maxChars),
|
|
1056
1284
|
reply: async (replyText) => { await this.send(sender.id, replyText, wireId); },
|
|
1285
|
+
replyHtml: async (filename, html) => {
|
|
1286
|
+
const directory = join(this.options.stateDir, '.owner-channel-report-outbox', this.requestId(wireId));
|
|
1287
|
+
const path = join(directory, filename);
|
|
1288
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1289
|
+
try {
|
|
1290
|
+
await writeFile(path, html, { flag: 'wx', mode: 0o600 });
|
|
1291
|
+
await this.client.sendFile({ contact: sender.id, path, filename, replyToWireId: wireId });
|
|
1292
|
+
}
|
|
1293
|
+
finally {
|
|
1294
|
+
await rm(directory, { recursive: true, force: true });
|
|
1295
|
+
}
|
|
1296
|
+
},
|
|
1057
1297
|
};
|
|
1058
1298
|
try {
|
|
1059
1299
|
await dispatchOwnerCommand(text, ctx);
|
|
@@ -1876,7 +2116,7 @@ export class OwnerChannel {
|
|
|
1876
2116
|
}
|
|
1877
2117
|
return undefined;
|
|
1878
2118
|
}
|
|
1879
|
-
async watchLoop() {
|
|
2119
|
+
async watchLoop(generation) {
|
|
1880
2120
|
const sleep = this.options.binderDeps?.sleep
|
|
1881
2121
|
?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
1882
2122
|
const restored = this.readWatchState();
|
|
@@ -1885,7 +2125,7 @@ export class OwnerChannel {
|
|
|
1885
2125
|
state = this.writeWatchState(state, 'OWNER_WATCH_STATE_RECOVERED');
|
|
1886
2126
|
let delayMs = 1_000;
|
|
1887
2127
|
let attempts = 0;
|
|
1888
|
-
while (!this.stopping) {
|
|
2128
|
+
while (!this.stopping && generation === this.watchGeneration) {
|
|
1889
2129
|
const ctrl = new AbortController();
|
|
1890
2130
|
this.watchAbort = ctrl;
|
|
1891
2131
|
try {
|
|
@@ -1895,12 +2135,12 @@ export class OwnerChannel {
|
|
|
1895
2135
|
// fall into a gap. Persistent history plus fleet's body-free claim
|
|
1896
2136
|
// journal is authoritative; durable wire-ID dedupe makes hints harmless.
|
|
1897
2137
|
await this.drain();
|
|
1898
|
-
if (this.stopping)
|
|
2138
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1899
2139
|
return;
|
|
1900
2140
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTING', { reconnected: attempts > 0 });
|
|
1901
2141
|
attempts++;
|
|
1902
2142
|
for await (const _event of this.client.watchNotifications(this.options.config.identity, { since: 0, signal: ctrl.signal })) {
|
|
1903
|
-
if (this.stopping)
|
|
2143
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1904
2144
|
return;
|
|
1905
2145
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTED', { resetFailures: true });
|
|
1906
2146
|
delayMs = 1_000;
|
|
@@ -1909,11 +2149,11 @@ export class OwnerChannel {
|
|
|
1909
2149
|
// recorded before a managed request is dispatched.
|
|
1910
2150
|
await this.drain();
|
|
1911
2151
|
}
|
|
1912
|
-
if (!this.stopping)
|
|
2152
|
+
if (!this.stopping && generation === this.watchGeneration)
|
|
1913
2153
|
throw new Error('ours SDK notification stream ended');
|
|
1914
2154
|
}
|
|
1915
2155
|
catch (error) {
|
|
1916
|
-
if (this.stopping)
|
|
2156
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1917
2157
|
return;
|
|
1918
2158
|
// SDK 2 deliberately hides transport status behind its typed stream.
|
|
1919
2159
|
// Do not parse error prose to rediscover it: every failure follows the
|
|
@@ -25,6 +25,7 @@ export interface OwnerFleetOps {
|
|
|
25
25
|
* cannot name another agent or another recipient.
|
|
26
26
|
*/
|
|
27
27
|
export interface OwnerCommandContext {
|
|
28
|
+
authenticatedCid?: string;
|
|
28
29
|
role: string;
|
|
29
30
|
/** Harness id of the role (e.g. 'claude-code', 'codex'); gates forwarding. */
|
|
30
31
|
harness: string;
|
|
@@ -94,6 +95,7 @@ export interface OwnerCommandContext {
|
|
|
94
95
|
recentEvents(limit: number): SessionEvent[];
|
|
95
96
|
readWorklogTail(maxChars: number): Promise<string | undefined>;
|
|
96
97
|
reply(text: string): Promise<void>;
|
|
98
|
+
replyHtml(filename: string, html: string): Promise<void>;
|
|
97
99
|
}
|
|
98
100
|
export interface OwnerCommand {
|
|
99
101
|
/** Primary name without the leading slash. */
|
|
@@ -6,6 +6,16 @@ import { markdownCode, markdownProse, renderMarkdownFailure, renderMarkdownList,
|
|
|
6
6
|
import { OWNER_COMMENT_LABEL, ownerNotices } from './notices.js';
|
|
7
7
|
import { TaskRoomApplicationError } from '../application/task-room-service.js';
|
|
8
8
|
import { TaskListError } from '../rooms-tasks/task-lists.js';
|
|
9
|
+
import { createTasksReport } from '../reports/index.js';
|
|
10
|
+
async function ownerTaskReport(ctx) {
|
|
11
|
+
return createTasksReport({
|
|
12
|
+
viewer: { surface: 'messenger', authenticatedCid: ctx.authenticatedCid ?? 'owner-command-context', roomCids: [] },
|
|
13
|
+
collect: () => {
|
|
14
|
+
return { lists: ctx.listTaskLists(), tasks: ctx.listTasks() };
|
|
15
|
+
},
|
|
16
|
+
source: { name: 'ours-fleet', version: ctx.version },
|
|
17
|
+
});
|
|
18
|
+
}
|
|
9
19
|
/** A malformed invocation; the dispatcher answers it with annotated help. */
|
|
10
20
|
class OwnerCommandUsageError extends Error {
|
|
11
21
|
}
|
|
@@ -242,10 +252,18 @@ export const ownerCommands = [
|
|
|
242
252
|
execute: noArgs('/version', async (ctx) => ctx.reply(`ℹ️ ours-fleet ${ctx.version}`)),
|
|
243
253
|
},
|
|
244
254
|
{
|
|
245
|
-
name: 'tasks', usage: '/tasks [state]',
|
|
246
|
-
summary: 'list tasks
|
|
255
|
+
name: 'tasks', usage: '/tasks [state|html]',
|
|
256
|
+
summary: 'list tasks, or attach the HTML report with /tasks html',
|
|
247
257
|
execute: async (ctx, args) => {
|
|
248
258
|
const stateFilter = args?.trim() || undefined;
|
|
259
|
+
if (stateFilter === 'html') {
|
|
260
|
+
const artifact = await ownerTaskReport(ctx);
|
|
261
|
+
await ctx.replyHtml(artifact.metadata.filename, artifact.html);
|
|
262
|
+
await ctx.reply(`HTML report attached: ${artifact.metadata.filename}`);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (stateFilter && /\s/u.test(stateFilter))
|
|
266
|
+
throw new OwnerCommandUsageError('usage: /tasks [state|html]');
|
|
249
267
|
const tasks = ctx.listTasks(stateFilter && stateFilter !== 'all' ? { state: stateFilter } : undefined);
|
|
250
268
|
await ctx.reply(renderMarkdownList({
|
|
251
269
|
icon: '📋', title: 'Tasks', empty: 'No tasks found.',
|
|
@@ -309,6 +327,9 @@ export const ownerCommands = [
|
|
|
309
327
|
break;
|
|
310
328
|
}
|
|
311
329
|
case 'list': {
|
|
330
|
+
if (rest.some(value => value.startsWith('--')
|
|
331
|
+
&& value !== '--group-by-list' && !value.startsWith('--list=')))
|
|
332
|
+
throw new OwnerCommandUsageError('usage: /task list [state] [--list=<name>] [--group-by-list]');
|
|
312
333
|
const listFlag = rest.find(r => r.startsWith('--list='));
|
|
313
334
|
const list = trailingLines ?? listFlag?.slice('--list='.length);
|
|
314
335
|
const grouped = rest.includes('--group-by-list');
|
|
@@ -345,6 +366,8 @@ export const ownerCommands = [
|
|
|
345
366
|
break;
|
|
346
367
|
}
|
|
347
368
|
case 'lists': {
|
|
369
|
+
if (rest.length || trailingLines)
|
|
370
|
+
throw new OwnerCommandUsageError('usage: /task lists');
|
|
348
371
|
const lists = ctx.listTaskLists();
|
|
349
372
|
await ctx.reply(renderMarkdownList({ icon: '📚', title: 'Task lists', empty: 'No task lists found.',
|
|
350
373
|
records: lists.map(list => `${markdownCode(list.name)}${list.built_in ? ' — built-in' : ''}`) }));
|
|
@@ -388,7 +411,7 @@ export const ownerCommands = [
|
|
|
388
411
|
break;
|
|
389
412
|
}
|
|
390
413
|
case 'show': {
|
|
391
|
-
if (
|
|
414
|
+
if (rest.length !== 1 || rest[0].startsWith('--'))
|
|
392
415
|
throw new OwnerCommandUsageError('usage: /task show <id>');
|
|
393
416
|
await showTask(rest[0]);
|
|
394
417
|
break;
|
|
@@ -598,10 +621,7 @@ export const ownerCommands = [
|
|
|
598
621
|
const templates = ctx.listTemplateQueries();
|
|
599
622
|
if (!templates.length)
|
|
600
623
|
return ctx.reply('📐 No templates.');
|
|
601
|
-
const lines = templates.map(t => {
|
|
602
|
-
const tag = t.builtin ? ' (built-in)' : '';
|
|
603
|
-
return `${t.name}@${t.version}${tag} ${t.description}`;
|
|
604
|
-
});
|
|
624
|
+
const lines = templates.map(t => `${t.name}@${t.version} ${t.description} [file: ${t.sourceFile ?? 'manifest'}]`);
|
|
605
625
|
await ctx.reply(`📐 Templates:\n${lines.join('\n')}`);
|
|
606
626
|
}),
|
|
607
627
|
},
|
|
@@ -627,10 +647,11 @@ export const ownerCommands = [
|
|
|
627
647
|
const lines = [
|
|
628
648
|
`📐 Template: ${t.name}@${t.version}`,
|
|
629
649
|
`Description: ${t.description}`,
|
|
630
|
-
|
|
650
|
+
`Source: ${t.sourceFile ?? 'manifest'}`,
|
|
631
651
|
...(t.contract ? [`Contract: ${t.contract}`] : []),
|
|
632
652
|
'Members:',
|
|
633
|
-
...t.members.map(m => ` ${m.slot} (${m.role}) ×${m.count} →
|
|
653
|
+
...t.members.map(m => ` ${m.slot} (${m.role}) ×${m.count} → `
|
|
654
|
+
+ `${'ref' in m.agent ? `Agent ref: ${m.agent.ref}` : 'inline Agent'}`),
|
|
634
655
|
];
|
|
635
656
|
await ctx.reply(lines.join('\n'));
|
|
636
657
|
};
|
|
@@ -645,10 +666,7 @@ export const ownerCommands = [
|
|
|
645
666
|
const templates = ctx.listTemplateQueries();
|
|
646
667
|
if (!templates.length)
|
|
647
668
|
return ctx.reply('📐 No templates.');
|
|
648
|
-
const lines = templates.map(t => {
|
|
649
|
-
const tag = t.builtin ? ' (built-in)' : '';
|
|
650
|
-
return `${t.name}@${t.version}${tag} ${t.description}`;
|
|
651
|
-
});
|
|
669
|
+
const lines = templates.map(t => `${t.name}@${t.version} ${t.description} [file: ${t.sourceFile ?? 'manifest'}]`);
|
|
652
670
|
await ctx.reply(`📐 Templates:\n${lines.join('\n')}`);
|
|
653
671
|
break;
|
|
654
672
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** A send whose (dedupe-scoped) digest was already recorded: it must not repeat. */
|
|
2
2
|
export declare class DuplicateSendError extends Error {
|
|
3
|
+
readonly status: OwnerProactiveSend['status'];
|
|
4
|
+
constructor(status: OwnerProactiveSend['status']);
|
|
3
5
|
}
|
|
4
6
|
/** Durable bounded dedupe containing wire IDs only — never message or reply plaintext. */
|
|
5
7
|
export declare class OwnerChannelState {
|
|
@@ -5,6 +5,11 @@ import { replaceFileAtomically } from '../atomic-file.js';
|
|
|
5
5
|
import { canonicalCid } from '../config.js';
|
|
6
6
|
/** A send whose (dedupe-scoped) digest was already recorded: it must not repeat. */
|
|
7
7
|
export class DuplicateSendError extends Error {
|
|
8
|
+
status;
|
|
9
|
+
constructor(status) {
|
|
10
|
+
super('duplicate proactive owner message refused');
|
|
11
|
+
this.status = status;
|
|
12
|
+
}
|
|
8
13
|
}
|
|
9
14
|
/** Durable bounded dedupe containing wire IDs only — never message or reply plaintext. */
|
|
10
15
|
export class OwnerChannelState {
|
|
@@ -184,8 +189,9 @@ export class OwnerConversationState {
|
|
|
184
189
|
// 'all' scope serves wire-keyed idempotency: a crash replay must not
|
|
185
190
|
// deliver the same wire to a second owner after the route moved.
|
|
186
191
|
const scope = dedupe === 'all' ? this.sends : recent;
|
|
187
|
-
|
|
188
|
-
|
|
192
|
+
const duplicate = scope.find(send => send.digest === digest);
|
|
193
|
+
if (duplicate)
|
|
194
|
+
throw new DuplicateSendError(duplicate.status);
|
|
189
195
|
const last = recent.at(-1);
|
|
190
196
|
if (last && now - last.at < minIntervalMs)
|
|
191
197
|
throw new Error(`proactive owner messages are rate-limited to one every ${minIntervalMs}ms`);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface BootstrapResult {
|
|
2
|
+
revision: number;
|
|
3
|
+
sourceRoot: string;
|
|
4
|
+
configPath: string;
|
|
5
|
+
created: string[];
|
|
6
|
+
preserved: string[];
|
|
7
|
+
}
|
|
8
|
+
export declare const packagedPresetRoot: () => string;
|
|
9
|
+
/** Seed packaged defaults without replacing a single existing byte. */
|
|
10
|
+
export declare function bootstrapPresets(configuration?: string): BootstrapResult;
|