@ours.network/fleet 1.1.0-nightly.3 → 1.1.0-nightly.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +244 -40
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/capabilities.js +18 -6
- package/dist/application/fleet-query-service.js +20 -5
- package/dist/application/role-command-service.js +9 -0
- package/dist/application/role-creation-service.d.ts +4 -4
- package/dist/application/role-creation-service.js +19 -1
- package/dist/application/role-repository.d.ts +4 -2
- package/dist/application/role-repository.js +6 -3
- package/dist/application/session-control.d.ts +3 -1
- package/dist/application/session-control.js +4 -2
- package/dist/application/task-room-service.d.ts +93 -40
- package/dist/application/task-room-service.js +579 -173
- package/dist/briefing.js +85 -24
- package/dist/build-info.json +4 -4
- package/dist/canonical-json.d.ts +2 -0
- package/dist/canonical-json.js +10 -0
- package/dist/cli.js +217 -51
- package/dist/config-yaml.d.ts +1 -1
- package/dist/config.d.ts +38 -3
- package/dist/config.js +240 -28
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +7 -4
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +213 -42
- package/dist/doctor.d.ts +2 -0
- package/dist/doctor.js +54 -4
- package/dist/fleet-command-audit.d.ts +181 -0
- package/dist/fleet-command-audit.js +738 -0
- package/dist/fleet-proxy.d.ts +7 -2
- package/dist/harness/agent-session.d.ts +5 -0
- package/dist/harness/claude-code-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +6 -1
- package/dist/harness/claude-code.js +2 -0
- package/dist/harness/codex-app-server-proxy.d.ts +40 -0
- package/dist/harness/codex-app-server-proxy.js +348 -14
- package/dist/harness/codex-session.d.ts +11 -2
- package/dist/harness/codex-session.js +22 -2
- package/dist/harness/codex.d.ts +5 -1
- package/dist/harness/codex.js +110 -10
- package/dist/harness/types.d.ts +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/init-guidance.d.ts +1 -1
- package/dist/init-guidance.js +4 -3
- package/dist/init-wizard.d.ts +98 -0
- package/dist/init-wizard.js +631 -0
- package/dist/lifecycle-summary.d.ts +134 -0
- package/dist/lifecycle-summary.js +179 -0
- package/dist/loops/config.d.ts +10 -1
- package/dist/loops/config.js +38 -5
- package/dist/model-recovery.d.ts +1 -1
- package/dist/ops.js +2 -17
- package/dist/owner-channel/channel.d.ts +75 -3
- package/dist/owner-channel/channel.js +531 -123
- package/dist/owner-channel/commands.d.ts +28 -6
- package/dist/owner-channel/commands.js +112 -36
- package/dist/owner-channel/lifecycle-outbox.d.ts +24 -0
- package/dist/owner-channel/lifecycle-outbox.js +87 -0
- package/dist/owner-channel/ours-client.d.ts +19 -0
- package/dist/owner-channel/ours-client.js +3 -0
- package/dist/owner-channel/state.d.ts +2 -0
- package/dist/owner-channel/state.js +8 -2
- package/dist/permissions.js +1 -1
- package/dist/preset-bootstrap.d.ts +10 -0
- package/dist/preset-bootstrap.js +85 -0
- package/dist/preset-migration.d.ts +42 -0
- package/dist/preset-migration.js +450 -0
- package/dist/reports/artifact.d.ts +10 -0
- package/dist/reports/artifact.js +32 -0
- package/dist/reports/index.d.ts +5 -0
- package/dist/reports/index.js +5 -0
- package/dist/reports/render.d.ts +6 -0
- package/dist/reports/render.js +240 -0
- package/dist/reports/service.d.ts +42 -0
- package/dist/reports/service.js +57 -0
- package/dist/reports/tasks.d.ts +52 -0
- package/dist/reports/tasks.js +177 -0
- package/dist/reports/types.d.ts +315 -0
- package/dist/reports/types.js +6 -0
- package/dist/resolved-plan.js +13 -1
- package/dist/rooms-tasks/cli.d.ts +2 -0
- package/dist/rooms-tasks/cli.js +519 -261
- package/dist/rooms-tasks/close.d.ts +9 -1
- package/dist/rooms-tasks/close.js +23 -7
- package/dist/rooms-tasks/config.js +16 -29
- package/dist/rooms-tasks/cowork-adapter.d.ts +5 -0
- package/dist/rooms-tasks/cowork-adapter.js +13 -0
- package/dist/rooms-tasks/deletion.d.ts +51 -0
- package/dist/rooms-tasks/deletion.js +217 -0
- package/dist/rooms-tasks/external-worker.d.ts +9 -0
- package/dist/rooms-tasks/external-worker.js +52 -9
- package/dist/rooms-tasks/launch-snapshot.d.ts +19 -0
- package/dist/rooms-tasks/launch-snapshot.js +181 -0
- package/dist/rooms-tasks/member-overrides.d.ts +30 -0
- package/dist/rooms-tasks/member-overrides.js +204 -0
- package/dist/rooms-tasks/member-startup.d.ts +1 -0
- package/dist/rooms-tasks/member-startup.js +9 -2
- package/dist/rooms-tasks/provision.js +146 -56
- package/dist/rooms-tasks/room-state.d.ts +1 -0
- package/dist/rooms-tasks/room-state.js +12 -1
- package/dist/rooms-tasks/task-room-name.d.ts +12 -0
- package/dist/rooms-tasks/task-room-name.js +61 -0
- package/dist/rooms-tasks/task-state.d.ts +115 -3
- package/dist/rooms-tasks/task-state.js +361 -11
- package/dist/rooms-tasks/templates.d.ts +4 -2
- package/dist/rooms-tasks/templates.js +42 -78
- package/dist/rooms-tasks/terminal.d.ts +3 -0
- package/dist/rooms-tasks/terminal.js +10 -4
- package/dist/rooms-tasks/types.d.ts +94 -8
- package/dist/rooms-tasks/types.js +14 -1
- package/dist/runner.d.ts +12 -2
- package/dist/runner.js +167 -56
- package/dist/session/acp.d.ts +9 -1
- package/dist/session/acp.js +35 -9
- package/dist/session/arbiter.d.ts +2 -1
- package/dist/session/arbiter.js +2 -0
- package/dist/session/codex-app-server-transport.d.ts +53 -0
- package/dist/session/codex-app-server-transport.js +184 -0
- package/dist/session/codex-app-server.d.ts +122 -0
- package/dist/session/codex-app-server.js +1031 -0
- package/dist/session/control.d.ts +14 -1
- package/dist/session/control.js +51 -0
- package/dist/session/conversation-types.d.ts +4 -4
- package/dist/session/types.d.ts +24 -6
- package/dist/session/types.js +28 -0
- package/dist/spawn.d.ts +9 -0
- package/dist/spawn.js +77 -9
- package/dist/temp-lifecycle.d.ts +1 -1
- package/dist/web/fleet-config-service.d.ts +1 -0
- package/dist/web/fleet-config-service.js +27 -8
- package/dist/web/runtime.js +11 -6
- package/dist/web/server.js +28 -1
- package/dist/web-app/assets/{index-BbTc5KtZ.js → index-C0h3ALvs.js} +8 -8
- package/dist/web-app/index.html +1 -1
- package/package.json +5 -4
- package/presets/brain-catalog.json +17 -0
- package/presets/fleet/agent_templates/Critic.yaml +5 -0
- package/presets/fleet/agent_templates/Developer.yaml +5 -0
- package/presets/fleet/agent_templates/LocalCoordinator.yaml +50 -0
- package/presets/fleet/agents/FleetCoordinator.yaml +3 -0
- package/presets/fleet/brains/claude-default.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-fable-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-opus-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-high.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-low.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-max.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-medium.yaml +5 -0
- package/presets/fleet/brains/claude-sonnet-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-3-codex-spark-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-mini-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-4-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-5-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-luna-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-sol-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-5-6-terra-xhigh.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-high.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-low.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-max.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-medium.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-ultra.yaml +5 -0
- package/presets/fleet/brains/codex-gpt-6-astra-xhigh.yaml +5 -0
- package/presets/fleet/roles/Coordinator.yaml +30 -0
- package/presets/fleet/roles/Critic.yaml +28 -0
- package/presets/fleet/roles/Developer.yaml +28 -0
- package/presets/fleet/roles/LocalCoordinator.yaml +35 -0
- package/presets/fleet/room_templates/pair.yaml +13 -0
- package/presets/fleet/room_templates/single.yaml +9 -0
- package/presets/fleet/room_templates/team.yaml +16 -0
- package/presets/fleet.yaml +7 -0
- package/presets/manifest.json +6 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
-
import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
|
|
3
|
+
import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { canonicalCid } from '../config.js';
|
|
6
|
+
import { DAEMON_RECOVERY_DEADLINE_MS } from '../daemon-recovery.js';
|
|
6
7
|
import { replaceFileAtomically } from '../atomic-file.js';
|
|
7
8
|
import { TaskRoomApplicationService } from '../application/task-room-service.js';
|
|
8
9
|
import { RoleLifecycleService } from '../application/role-command-service.js';
|
|
@@ -10,17 +11,37 @@ import { RoleRepository } from '../application/role-repository.js';
|
|
|
10
11
|
import { FleetQueryService } from '../application/fleet-query-service.js';
|
|
11
12
|
import { interruptSession, queueSessionPrompt } from '../application/session-mutations.js';
|
|
12
13
|
import { pickBackend } from '../supervisor/index.js';
|
|
13
|
-
import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, } from '../session/types.js';
|
|
14
|
+
import { ACP_CANCEL_DEADLINE_EXCEEDED, CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED, SessionControlError, } from '../session/types.js';
|
|
14
15
|
import { VERSION } from '../version.js';
|
|
15
|
-
import { renderMarkdownFailure, renderMarkdownResult,
|
|
16
|
-
import { dispatchOwnerCommand, fleetCliOps, isOwnerCommandText, } from './commands.js';
|
|
16
|
+
import { renderMarkdownFailure, renderMarkdownResult, taskStatus, } from '../rooms-tasks/markdown.js';
|
|
17
|
+
import { dispatchOwnerCommand, fleetCliOps, isOwnerCommandText, ownerTypedCommandCatalog, ownerTypedCommandText, } from './commands.js';
|
|
18
|
+
import { beginFleetAuditCollection, consumeFleetAuditCollection, FleetCommandAuditStore, lifecycleEventDigestBasis, renderFleetLifecycleEvent, setFleetAuditLifecycleCheckpoint, } from '../fleet-command-audit.js';
|
|
17
19
|
import { OURS_BOUND_ELSEWHERE, OursSdkClient, oursErrorCode, } from './ours-client.js';
|
|
18
20
|
import { ownerNotices, } from './notices.js';
|
|
19
21
|
import { DuplicateSendError, OwnerAuthorizationState, OwnerChannelState, OwnerConversationState, } from './state.js';
|
|
22
|
+
import { FleetLifecycleOutbox } from './lifecycle-outbox.js';
|
|
20
23
|
import { OwnerTaskState, ownerTaskAuditId, ownerTaskDigest, } from './tasks.js';
|
|
21
24
|
import { AttachmentRecoveryState, admitAttachments, cleanupAttachmentRoot, parseIncomingAttachments, parseRetrievedAttachments, prepareAttachmentDirectory, recoveredAttachment, removeRequestDirectory, safeField, validateAttachmentSelection, validateAttachmentRelaySelection, writeRecoveredAttachment, } from './attachments.js';
|
|
22
25
|
import { acquireOwnerBinderLease, OWNER_BIND_HANDOFF_TIMEOUT_MS, } from './binder.js';
|
|
23
26
|
import { MessageRecoveryState } from './message-recovery.js';
|
|
27
|
+
export const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
|
|
28
|
+
export const OWNER_RECOVERY_DEGRADED = 'OWNER_RECOVERY_DEGRADED';
|
|
29
|
+
export class OwnerRecoveryDegradedError extends Error {
|
|
30
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
31
|
+
constructor() {
|
|
32
|
+
super('owner channel recovery is degraded');
|
|
33
|
+
this.name = 'OwnerRecoveryDegradedError';
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export class OwnerRecoveryTimeoutError extends Error {
|
|
37
|
+
stage;
|
|
38
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
39
|
+
constructor(stage) {
|
|
40
|
+
super(`owner channel recovery timed out at ${stage}`);
|
|
41
|
+
this.stage = stage;
|
|
42
|
+
this.name = 'OwnerRecoveryTimeoutError';
|
|
43
|
+
}
|
|
44
|
+
}
|
|
24
45
|
const OWNER_UPDATE_MIN_INTERVAL_MS = 5_000;
|
|
25
46
|
const OWNER_UPDATE_MAX_COUNT = 20;
|
|
26
47
|
const OWNER_UPDATE_MAX_CHARS = 280;
|
|
@@ -58,6 +79,8 @@ export class OwnerChannel {
|
|
|
58
79
|
* (a crash must replay them) but must not be queued twice while live.
|
|
59
80
|
*/
|
|
60
81
|
inFlight = new Set();
|
|
82
|
+
/** SDK handlers entered during the current getMessages call. */
|
|
83
|
+
typedHandlersEntered = new Set();
|
|
61
84
|
/** Wires already NACKed to the managed agent, so a history replay stays quiet. */
|
|
62
85
|
relayNacks = new Set();
|
|
63
86
|
/**
|
|
@@ -76,11 +99,23 @@ export class OwnerChannel {
|
|
|
76
99
|
completionTasks = new Set();
|
|
77
100
|
activeRequests = new Map();
|
|
78
101
|
managementTail = Promise.resolve();
|
|
102
|
+
recoveryEpoch;
|
|
103
|
+
recoveryTask;
|
|
104
|
+
recoveryToken = 0;
|
|
105
|
+
recoveryQueued = 0;
|
|
106
|
+
watchGeneration = 0;
|
|
107
|
+
/** A timed-out client operation that must settle before replacement is safe. */
|
|
108
|
+
recoveryQuiescence;
|
|
109
|
+
releaseSafe = true;
|
|
79
110
|
ready = false;
|
|
111
|
+
startedOnce = false;
|
|
112
|
+
shutdownDegraded = false;
|
|
80
113
|
binder;
|
|
81
114
|
binderOwnedInternally = false;
|
|
82
115
|
fleetOps;
|
|
83
116
|
prepareRestart;
|
|
117
|
+
commandAudits;
|
|
118
|
+
lifecycleOutbox;
|
|
84
119
|
constructor(options) {
|
|
85
120
|
this.options = options;
|
|
86
121
|
this.client = options.client ?? new OursSdkClient(options.env, line => options.log(`[${options.role}] owner channel ${line}`));
|
|
@@ -96,6 +131,8 @@ export class OwnerChannel {
|
|
|
96
131
|
await lifecycle.prepareRestart({ roleIds: [role], mode });
|
|
97
132
|
});
|
|
98
133
|
this.state = new OwnerChannelState(join(options.stateDir, '.owner-channel-state.json'));
|
|
134
|
+
this.commandAudits = new FleetCommandAuditStore(join(options.stateDir, '.owner-channel-command-audit.json'));
|
|
135
|
+
this.lifecycleOutbox = new FleetLifecycleOutbox(join(options.stateDir, '.owner-channel-lifecycle-outbox.json'));
|
|
99
136
|
this.authorizations = new OwnerAuthorizationState(join(options.stateDir, '.owner-channel-owners.json'), options.config.owners);
|
|
100
137
|
this.conversations = new OwnerConversationState(join(options.stateDir, '.owner-channel-conversations.json'));
|
|
101
138
|
this.tasks = new OwnerTaskState(join(options.stateDir, '.owner-channel-tasks.json'));
|
|
@@ -121,9 +158,12 @@ export class OwnerChannel {
|
|
|
121
158
|
options.log(`[${options.role}] owner message recovery state corrupt; message intake disabled`);
|
|
122
159
|
if (!this.attachmentRecovery.integrity())
|
|
123
160
|
options.log(`[${options.role}] owner attachment recovery state corrupt; attachments disabled`);
|
|
161
|
+
if (!this.lifecycleOutbox.integrity().ok)
|
|
162
|
+
options.log(`[${options.role}] owner lifecycle outbox corrupt; lifecycle delivery disabled`);
|
|
124
163
|
}
|
|
125
164
|
async start() {
|
|
126
165
|
this.stopping = false;
|
|
166
|
+
this.shutdownDegraded = false;
|
|
127
167
|
this.binderOwnedInternally = !this.options.binderLease;
|
|
128
168
|
this.binder = this.options.binderLease ?? await acquireOwnerBinderLease(this.options.stateDir, this.options.role, this.options.config.identity, this.options.binderDeps);
|
|
129
169
|
try {
|
|
@@ -135,6 +175,7 @@ export class OwnerChannel {
|
|
|
135
175
|
for (;;) {
|
|
136
176
|
try {
|
|
137
177
|
await this.client.bindIdentity(this.options.config.identity);
|
|
178
|
+
await this.registerTypedCommands();
|
|
138
179
|
break;
|
|
139
180
|
}
|
|
140
181
|
catch (error) {
|
|
@@ -164,11 +205,14 @@ export class OwnerChannel {
|
|
|
164
205
|
this.attachmentRecovery.cleanup(Date.now(), this.attachmentConfig.retention_ms);
|
|
165
206
|
void cleanupAttachmentRoot(this.attachmentRoot, Date.now(), this.attachmentConfig.retention_ms).catch(error => this.logError('attachment crash cleanup failed', error));
|
|
166
207
|
}
|
|
208
|
+
this.startedOnce = true;
|
|
167
209
|
this.ready = true;
|
|
210
|
+
await this.flushPendingFleetCommandAudits();
|
|
211
|
+
await this.flushPendingFleetLifecycle();
|
|
168
212
|
// Do not make role startup wait for an old owner request to finish a turn.
|
|
169
213
|
// watchLoop itself drains before every establishment, including this first
|
|
170
214
|
// one, so there is no drain-to-tip race.
|
|
171
|
-
this.watchTask = this.watchLoop();
|
|
215
|
+
this.watchTask = this.watchLoop(++this.watchGeneration);
|
|
172
216
|
}
|
|
173
217
|
drain() {
|
|
174
218
|
this.drainRequested = true;
|
|
@@ -183,46 +227,292 @@ export class OwnerChannel {
|
|
|
183
227
|
return this.drainTask;
|
|
184
228
|
}
|
|
185
229
|
async close() {
|
|
230
|
+
if (this.recoveryTask)
|
|
231
|
+
this.shutdownDegraded = true;
|
|
186
232
|
this.stopping = true;
|
|
187
233
|
this.ready = false;
|
|
188
|
-
this.
|
|
189
|
-
|
|
234
|
+
this.recoveryToken++;
|
|
235
|
+
this.watchGeneration++;
|
|
236
|
+
const staleWatch = this.watchTask;
|
|
190
237
|
this.watchTask = undefined;
|
|
191
|
-
|
|
238
|
+
this.watchAbort?.abort();
|
|
239
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
240
|
+
const deadlineAt = now()
|
|
241
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
242
|
+
let disposed = false;
|
|
192
243
|
try {
|
|
193
|
-
await this.
|
|
244
|
+
await this.recoveryStage('shutdown_quiescence', Promise.all([
|
|
245
|
+
staleWatch?.catch(error => this.logError('watch shutdown failed', error)),
|
|
246
|
+
this.drainTask?.catch(error => this.logError('drain shutdown failed', error)),
|
|
247
|
+
this.managementTail.then(() => undefined),
|
|
248
|
+
this.recoveryQuiescence,
|
|
249
|
+
]), deadlineAt);
|
|
250
|
+
await this.recoveryStage('shutdown_close', this.client.close(), deadlineAt);
|
|
251
|
+
disposed = true;
|
|
252
|
+
this.writeShutdownState('closed', 'OWNER_SHUTDOWN_CLOSED', now());
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
this.shutdownDegraded = true;
|
|
256
|
+
this.releaseSafe = false;
|
|
257
|
+
const stage = error instanceof OwnerRecoveryTimeoutError ? error.stage : 'shutdown_unknown';
|
|
258
|
+
this.writeShutdownState('degraded', `OWNER_${stage.toUpperCase()}_TIMEOUT`, now());
|
|
259
|
+
this.options.log(`[${this.options.role}] owner shutdown degraded stage=${stage}`);
|
|
194
260
|
}
|
|
195
261
|
finally {
|
|
196
|
-
if (this.binderOwnedInternally)
|
|
262
|
+
if (disposed && this.binderOwnedInternally)
|
|
197
263
|
this.binder?.release();
|
|
198
|
-
|
|
264
|
+
if (disposed)
|
|
265
|
+
this.binder = undefined;
|
|
199
266
|
}
|
|
200
267
|
}
|
|
268
|
+
binderReleaseSafe() { return this.releaseSafe; }
|
|
201
269
|
manage(request) {
|
|
202
|
-
|
|
203
|
-
this.managementTail = run.then(() => undefined, () => undefined);
|
|
204
|
-
return run;
|
|
270
|
+
return this.queueManagement(() => this.manageNow(request));
|
|
205
271
|
}
|
|
206
272
|
notifyFleetSpawn(event) {
|
|
207
|
-
|
|
273
|
+
return this.queueManagement(async () => {
|
|
208
274
|
if (!this.ready || this.stopping)
|
|
209
275
|
throw new Error('owner-channel MCP client is unavailable');
|
|
210
|
-
const model = event.model ? `, model ${event.model}` : '';
|
|
211
|
-
const monitorPolicy = event.monitor.interrupt === true
|
|
212
|
-
? ' with interruption'
|
|
213
|
-
: event.monitor.interrupt === 'after_tool' ? ' with after-tool steering' : '';
|
|
214
|
-
const monitor = `${event.monitor.mode} monitor${monitorPolicy}`;
|
|
215
276
|
const permission = event.permissionMode
|
|
216
|
-
?
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
277
|
+
? `${event.permissionMode.fleetMode}/${event.permissionMode.nativeMode}` : undefined;
|
|
278
|
+
await this.sendProactiveMessage(renderFleetLifecycleEvent({ kind: 'agent_started', eventId: event.creationActionId,
|
|
279
|
+
id: event.role, name: event.role,
|
|
280
|
+
lifetime: event.lifetime, brain: event.brainSummary, role: event.roleSummary,
|
|
281
|
+
harness: event.harness, session: event.session, model: event.model,
|
|
282
|
+
permissions: permission, parent: event.caller, actionId: event.creationActionId,
|
|
283
|
+
inherited: event.inherited,
|
|
284
|
+
...(event.configuration ? { configuration: event.configuration } : {}) }), `fleet-spawn\0${event.creationActionId}`, 0);
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
notifyFleetLifecycle(presentations) {
|
|
288
|
+
const run = this.managementTail.then(async () => {
|
|
289
|
+
// Acceptance is durable before delivery. A detached worker may exit
|
|
290
|
+
// while the Owner channel is reconnecting; startup/recovery flushes the
|
|
291
|
+
// pending event through the existing exactly-once send ledger.
|
|
292
|
+
this.lifecycleOutbox.enqueue(presentations);
|
|
293
|
+
if (this.ready && !this.stopping)
|
|
294
|
+
await this.flushPendingFleetLifecycle();
|
|
222
295
|
});
|
|
223
296
|
this.managementTail = run.then(() => undefined, () => undefined);
|
|
224
297
|
return run;
|
|
225
298
|
}
|
|
299
|
+
async flushPendingFleetLifecycle() {
|
|
300
|
+
for (const entry of this.lifecycleOutbox.pending()) {
|
|
301
|
+
try {
|
|
302
|
+
await this.sendProactiveMessage(renderFleetLifecycleEvent(entry.presentation), `fleet-lifecycle\0${lifecycleEventDigestBasis(entry.presentation)}`, 0);
|
|
303
|
+
this.lifecycleOutbox.finish(entry.digest, 'delivered');
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
if (error instanceof DuplicateSendError) {
|
|
307
|
+
this.lifecycleOutbox.finish(entry.digest, error.status === 'delivered' ? 'delivered' : 'uncertain');
|
|
308
|
+
this.options.log(`[${this.options.role}] duplicate Fleet lifecycle event suppressed`);
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
// If the send crossed the uncertain boundary, the proactive ledger
|
|
312
|
+
// will refuse it on the next flush. A pre-send outage stays pending.
|
|
313
|
+
this.logError('pending Fleet lifecycle delivery deferred', error);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
beginFleetCommandAudit(requestId, argv) {
|
|
319
|
+
const run = this.managementTail.then(async () => {
|
|
320
|
+
let attempt = this.commandAudits.begin(requestId, this.options.role, argv);
|
|
321
|
+
if (attempt.invocation === 'delivered')
|
|
322
|
+
return attempt;
|
|
323
|
+
if (attempt.invocation === 'uncertain')
|
|
324
|
+
return attempt;
|
|
325
|
+
this.options.log(`[${this.options.role}] fleet proxy command ${attempt.correlationId} `
|
|
326
|
+
+ `route=${attempt.classification.route} decision=${attempt.classification.decision}`);
|
|
327
|
+
attempt = this.commandAudits.invocation(attempt.correlationId, this.options.role, 'delivered');
|
|
328
|
+
return attempt;
|
|
329
|
+
});
|
|
330
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
331
|
+
return run;
|
|
332
|
+
}
|
|
333
|
+
finishFleetCommandAudit(input) {
|
|
334
|
+
const run = this.managementTail.then(async () => {
|
|
335
|
+
let attempt = this.commandAudits.finish(input.correlationId, this.options.role, {
|
|
336
|
+
class: input.class, effect: input.effect,
|
|
337
|
+
...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
|
|
338
|
+
...(input.resourceIds ? { resourceIds: input.resourceIds } : {}),
|
|
339
|
+
...(input.presentations ? { presentations: input.presentations } : {}),
|
|
340
|
+
});
|
|
341
|
+
if (attempt.outcome?.delivery === 'delivered')
|
|
342
|
+
return attempt;
|
|
343
|
+
if (attempt.outcome?.delivery === 'uncertain')
|
|
344
|
+
return attempt;
|
|
345
|
+
// Before the Owner sink is attached, ordinary commands have nothing to
|
|
346
|
+
// deliver. Lifecycle presentations remain in the never-attempted
|
|
347
|
+
// `sending` state and are flushed exactly once when start() makes the
|
|
348
|
+
// authenticated sink ready. A restart converts them to `uncertain`, so
|
|
349
|
+
// effects are never replayed after an unproven delivery boundary.
|
|
350
|
+
if (!attempt.outcome?.presentations?.length)
|
|
351
|
+
return this.commandAudits.outcome(attempt.correlationId, this.options.role, 'delivered');
|
|
352
|
+
if (!this.ready)
|
|
353
|
+
return attempt;
|
|
354
|
+
return this.deliverFleetCommandOutcome(attempt);
|
|
355
|
+
});
|
|
356
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
357
|
+
return run;
|
|
358
|
+
}
|
|
359
|
+
async flushPendingFleetCommandAudits() {
|
|
360
|
+
for (const attempt of this.commandAudits.list())
|
|
361
|
+
if (attempt.outcome?.delivery === 'sending')
|
|
362
|
+
await this.deliverFleetCommandOutcome(attempt);
|
|
363
|
+
}
|
|
364
|
+
async deliverFleetCommandOutcome(attempt) {
|
|
365
|
+
let uncertain = false;
|
|
366
|
+
try {
|
|
367
|
+
for (const event of attempt.outcome?.presentations ?? []) {
|
|
368
|
+
try {
|
|
369
|
+
await this.sendProactiveMessage(renderFleetLifecycleEvent(event), `fleet-lifecycle\0${lifecycleEventDigestBasis(event)}`, 0);
|
|
370
|
+
}
|
|
371
|
+
catch (error) {
|
|
372
|
+
if (error instanceof DuplicateSendError) {
|
|
373
|
+
if (error.status !== 'delivered')
|
|
374
|
+
uncertain = true;
|
|
375
|
+
this.options.log(`[${this.options.role}] duplicate Fleet lifecycle event suppressed`);
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
throw error;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
catch (error) {
|
|
383
|
+
uncertain = true;
|
|
384
|
+
this.logError(`fleet lifecycle delivery uncertain correlation=${attempt.correlationId}`, error);
|
|
385
|
+
}
|
|
386
|
+
attempt = this.commandAudits.outcome(attempt.correlationId, this.options.role, uncertain ? 'uncertain' : 'delivered');
|
|
387
|
+
return attempt;
|
|
388
|
+
}
|
|
389
|
+
recover(epoch) {
|
|
390
|
+
if (this.stopping)
|
|
391
|
+
return Promise.reject(new Error('owner channel is stopping'));
|
|
392
|
+
if (this.recoveryEpoch === epoch && this.recoveryTask)
|
|
393
|
+
return this.recoveryTask;
|
|
394
|
+
this.recoveryEpoch = epoch;
|
|
395
|
+
this.ready = false;
|
|
396
|
+
const token = ++this.recoveryToken;
|
|
397
|
+
const watchGeneration = ++this.watchGeneration;
|
|
398
|
+
const staleWatch = this.watchTask;
|
|
399
|
+
this.watchTask = undefined;
|
|
400
|
+
this.watchAbort?.abort();
|
|
401
|
+
const superseded = () => this.stopping || token !== this.recoveryToken;
|
|
402
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
403
|
+
const deadlineAt = now()
|
|
404
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
405
|
+
const run = this.managementTail.then(async () => {
|
|
406
|
+
if (superseded())
|
|
407
|
+
throw new Error('owner recovery epoch superseded');
|
|
408
|
+
if (this.recoveryQuiescence) {
|
|
409
|
+
const debt = this.recoveryQuiescence;
|
|
410
|
+
await this.recoveryStage('prior_quiescence', debt, deadlineAt);
|
|
411
|
+
if (this.recoveryQuiescence === debt)
|
|
412
|
+
this.recoveryQuiescence = undefined;
|
|
413
|
+
}
|
|
414
|
+
// Quiesce every stale-client reader before replacing its transport. A
|
|
415
|
+
// watch can be inside the shared drain task when its abort lands.
|
|
416
|
+
await this.recoveryStage('stale_quiescence', Promise.all([
|
|
417
|
+
staleWatch?.catch(error => this.logError('stale watch shutdown failed', error)),
|
|
418
|
+
this.drainTask?.catch(error => this.logError('stale drain shutdown failed', error)),
|
|
419
|
+
]), deadlineAt);
|
|
420
|
+
if (superseded())
|
|
421
|
+
throw new Error('owner recovery epoch superseded');
|
|
422
|
+
await this.recoveryStage('close', this.client.close(), deadlineAt);
|
|
423
|
+
if (superseded())
|
|
424
|
+
throw new Error('owner recovery epoch superseded');
|
|
425
|
+
try {
|
|
426
|
+
await this.recoveryStage('start', this.client.start(), deadlineAt);
|
|
427
|
+
if (superseded())
|
|
428
|
+
throw new Error('owner recovery epoch superseded');
|
|
429
|
+
await this.recoveryStage('bind', this.client.bindIdentity(this.options.config.identity), deadlineAt);
|
|
430
|
+
if (superseded())
|
|
431
|
+
throw new Error('owner recovery epoch superseded');
|
|
432
|
+
await this.recoveryStage('register_commands', this.registerTypedCommands(), deadlineAt);
|
|
433
|
+
if (superseded())
|
|
434
|
+
throw new Error('owner recovery epoch superseded');
|
|
435
|
+
// Journal-aware drain is the only operation allowed to mark owner mail
|
|
436
|
+
// read. The final listing is a typed, read-only channel probe.
|
|
437
|
+
await this.recoveryStage('drain', this.drain(), deadlineAt);
|
|
438
|
+
await this.recoveryStage('list', this.client.listIncomingMessages(), deadlineAt);
|
|
439
|
+
if (superseded())
|
|
440
|
+
throw new Error('owner recovery epoch superseded');
|
|
441
|
+
this.ready = true;
|
|
442
|
+
await this.flushPendingFleetLifecycle();
|
|
443
|
+
this.watchTask = this.watchLoop(watchGeneration);
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
this.ready = false;
|
|
447
|
+
if (token === this.recoveryToken)
|
|
448
|
+
this.recoveryToken++;
|
|
449
|
+
// A timed-out operation may still be using this client. Never overlap
|
|
450
|
+
// its disposal; the recorded quiescence debt gates the next retry.
|
|
451
|
+
if (!this.recoveryQuiescence) {
|
|
452
|
+
try {
|
|
453
|
+
await this.recoveryStage('close_after_failure', this.client.close(), deadlineAt);
|
|
454
|
+
}
|
|
455
|
+
catch (closeError) {
|
|
456
|
+
this.logError('recovery client close failed', closeError);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
throw error;
|
|
460
|
+
}
|
|
461
|
+
});
|
|
462
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
463
|
+
const task = run.finally(() => {
|
|
464
|
+
if (this.recoveryTask === task)
|
|
465
|
+
this.recoveryTask = undefined;
|
|
466
|
+
});
|
|
467
|
+
this.recoveryTask = task;
|
|
468
|
+
return task;
|
|
469
|
+
}
|
|
470
|
+
recoveryStage(stage, operation, deadlineAt) {
|
|
471
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
472
|
+
const remaining = Math.max(0, deadlineAt - now());
|
|
473
|
+
const setTimer = this.options.recoveryDeps?.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
474
|
+
const clearTimer = this.options.recoveryDeps?.clearTimer ?? (timer => clearTimeout(timer));
|
|
475
|
+
let timer;
|
|
476
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
477
|
+
timer = setTimer(() => reject(new OwnerRecoveryTimeoutError(stage)), remaining);
|
|
478
|
+
});
|
|
479
|
+
return Promise.race([operation, timeout]).catch(error => {
|
|
480
|
+
if (error instanceof OwnerRecoveryTimeoutError) {
|
|
481
|
+
const debt = operation.then(() => undefined, () => undefined);
|
|
482
|
+
this.recoveryQuiescence = debt;
|
|
483
|
+
}
|
|
484
|
+
throw error;
|
|
485
|
+
}).finally(() => clearTimer(timer));
|
|
486
|
+
}
|
|
487
|
+
writeShutdownState(state, reason, at) {
|
|
488
|
+
try {
|
|
489
|
+
replaceFileAtomically(join(this.options.stateDir, '.owner-channel-shutdown.json'), JSON.stringify({ version: 1, state, reason, at: new Date(at).toISOString() }, null, 2) + '\n');
|
|
490
|
+
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
this.logError('shutdown status write failed', error);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
queueManagement(operation) {
|
|
496
|
+
const deferred = this.recoveryTask !== undefined && !this.ready;
|
|
497
|
+
if (!this.ready && !deferred)
|
|
498
|
+
return Promise.reject(!this.startedOnce || (this.stopping && !this.shutdownDegraded)
|
|
499
|
+
? new Error('owner-channel MCP client is unavailable')
|
|
500
|
+
: new OwnerRecoveryDegradedError());
|
|
501
|
+
if (deferred && this.recoveryQueued >= OWNER_RECOVERY_QUEUE_CAPACITY)
|
|
502
|
+
return Promise.reject(new Error('owner recovery queue is full'));
|
|
503
|
+
if (deferred)
|
|
504
|
+
this.recoveryQueued++;
|
|
505
|
+
const run = this.managementTail.then(() => {
|
|
506
|
+
if (!this.ready || this.stopping)
|
|
507
|
+
throw new OwnerRecoveryDegradedError();
|
|
508
|
+
return operation();
|
|
509
|
+
});
|
|
510
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
511
|
+
return run.finally(() => {
|
|
512
|
+
if (deferred)
|
|
513
|
+
this.recoveryQueued = Math.max(0, this.recoveryQueued - 1);
|
|
514
|
+
});
|
|
515
|
+
}
|
|
226
516
|
async manageNow(request) {
|
|
227
517
|
if (!this.ready || this.stopping)
|
|
228
518
|
throw new Error('owner-channel MCP client is unavailable');
|
|
@@ -600,27 +890,61 @@ export class OwnerChannel {
|
|
|
600
890
|
throw new Error('the ours daemon returned invalid unread message metadata');
|
|
601
891
|
const preflight = listed.slice(0, OWNER_MESSAGE_BATCH_LIMIT);
|
|
602
892
|
const now = Date.now();
|
|
603
|
-
const
|
|
604
|
-
|
|
605
|
-
if (
|
|
893
|
+
const metadata = preflight.map(item => ({ item, claim: this.messageClaim(item, now),
|
|
894
|
+
kind: item.message_kind }));
|
|
895
|
+
if (metadata.some(item => !['text', 'command', 'command_result'].includes(item.kind)))
|
|
896
|
+
throw new Error('the ours daemon returned an unknown unread message kind');
|
|
897
|
+
// The SDK marks the entire batch read before invoking typed handlers. Claim
|
|
898
|
+
// every text AND command row first: if an earlier lifecycle handler exits
|
|
899
|
+
// this process, a later command is recovered from persistent history even
|
|
900
|
+
// though its SDK handler was never entered. Command-result rows are inert.
|
|
901
|
+
const recoverable = metadata.filter(item => item.kind !== 'command_result');
|
|
902
|
+
const claims = recoverable.map(item => item.claim);
|
|
903
|
+
const textClaims = metadata.filter(item => item.kind === 'text').map(item => item.claim);
|
|
904
|
+
const textClaimKeys = new Set(textClaims.map(item => `${item.seq}\0${item.wireId}`));
|
|
905
|
+
const preflightKeys = new Set(metadata.map(item => `${item.claim.seq}\0${item.claim.wireId}`));
|
|
906
|
+
if (preflightKeys.size !== metadata.length)
|
|
606
907
|
throw new Error('the ours daemon returned duplicate unread message metadata');
|
|
607
908
|
this.messageRecovery.claim(claims);
|
|
608
909
|
let fresh = [];
|
|
609
910
|
let remaining = 0;
|
|
610
911
|
// SDK batchLimit rejects zero. An empty preflight is a read-only drain.
|
|
611
|
-
if (
|
|
612
|
-
const payload = await this.client.getMessages(
|
|
613
|
-
if (!payload || !Array.isArray(payload.messages)
|
|
912
|
+
if (preflight.length) {
|
|
913
|
+
const payload = await this.client.getMessages(preflight.length);
|
|
914
|
+
if (!payload || !Array.isArray(payload.messages) || !Array.isArray(payload.command_results)
|
|
915
|
+
|| !Number.isSafeInteger(payload.commands_handled) || payload.commands_handled < 0
|
|
614
916
|
|| !Number.isSafeInteger(payload.remaining) || payload.remaining < 0)
|
|
615
917
|
throw new Error('the ours daemon returned an invalid claimed message batch');
|
|
616
918
|
fresh = payload.messages.map(message => this.historyMessage(message));
|
|
617
919
|
remaining = payload.remaining;
|
|
618
|
-
const expected =
|
|
920
|
+
const expected = textClaimKeys;
|
|
619
921
|
const actual = new Set(fresh.map(item => `${item.seq}\0${item.wire_id}`));
|
|
620
|
-
if (fresh.length !==
|
|
922
|
+
if (fresh.length !== textClaims.length || actual.size !== fresh.length
|
|
621
923
|
|| actual.size !== expected.size
|
|
622
924
|
|| [...expected].some(item => !actual.has(item)))
|
|
623
925
|
throw new Error('the ours daemon claimed a different message batch than fleet journaled');
|
|
926
|
+
const expectedCommands = metadata.filter(item => item.kind === 'command');
|
|
927
|
+
if (payload.commands_handled !== expectedCommands.length)
|
|
928
|
+
throw new Error('the ours daemon handled a different typed-command batch than fleet inspected');
|
|
929
|
+
const expectedResults = new Set(metadata.filter(item => item.kind === 'command_result')
|
|
930
|
+
.map(item => `${item.claim.seq}\0${item.claim.wireId}`));
|
|
931
|
+
const actualResults = new Set(payload.command_results
|
|
932
|
+
.map(item => `${item.seq}\0${item.wire_id}`));
|
|
933
|
+
if (actualResults.size !== payload.command_results.length
|
|
934
|
+
|| actualResults.size !== expectedResults.size
|
|
935
|
+
|| [...expectedResults].some(item => !actualResults.has(item)))
|
|
936
|
+
throw new Error('the ours daemon returned a different typed-command result batch than fleet inspected');
|
|
937
|
+
for (const item of expectedCommands) {
|
|
938
|
+
// Invalid envelopes and unregistered commands never enter a handler;
|
|
939
|
+
// the SDK has already returned handler_failed, so consume those now.
|
|
940
|
+
// Entered harness commands keep their claim until async completion.
|
|
941
|
+
if (!this.typedHandlersEntered.has(item.claim.wireId))
|
|
942
|
+
this.state.remember(item.claim.wireId);
|
|
943
|
+
this.typedHandlersEntered.delete(item.claim.wireId);
|
|
944
|
+
}
|
|
945
|
+
for (const item of metadata)
|
|
946
|
+
if (item.kind === 'command_result')
|
|
947
|
+
this.state.remember(item.claim.wireId);
|
|
624
948
|
}
|
|
625
949
|
const merged = new Map();
|
|
626
950
|
for (const message of [...recovered, ...fresh]) {
|
|
@@ -876,6 +1200,8 @@ export class OwnerChannel {
|
|
|
876
1200
|
const wireId = this.wireId(message);
|
|
877
1201
|
if (!wireId || this.state.has(wireId) || this.inFlight.has(wireId))
|
|
878
1202
|
return false;
|
|
1203
|
+
if (message.message_kind === 'command')
|
|
1204
|
+
return this.handleRecoveredTypedCommand(message, wireId);
|
|
879
1205
|
const sender = this.sender(message);
|
|
880
1206
|
if (this.isAgentSender(sender.id)) {
|
|
881
1207
|
try {
|
|
@@ -943,12 +1269,13 @@ export class OwnerChannel {
|
|
|
943
1269
|
catch (error) {
|
|
944
1270
|
await rm(outbox, { recursive: true, force: true });
|
|
945
1271
|
if (error instanceof SessionControlError
|
|
946
|
-
&& error.reasonCode === ACP_CANCEL_DEADLINE_EXCEEDED
|
|
1272
|
+
&& (error.reasonCode === ACP_CANCEL_DEADLINE_EXCEEDED
|
|
1273
|
+
|| error.reasonCode === CODEX_APP_SERVER_CANCEL_DEADLINE_EXCEEDED)) {
|
|
947
1274
|
// drainAll journaled this authenticated message before delivery. The
|
|
948
1275
|
// adapter generation is terminating, so leave the wire unhandled and
|
|
949
1276
|
// body-free: the resumed owner channel recovers it exactly once.
|
|
950
1277
|
this.options.log(`[${this.options.role}] owner request ${requestId.slice(0, 12)} `
|
|
951
|
-
+ `held for adapter resume reason=${
|
|
1278
|
+
+ `held for adapter resume reason=${error.reasonCode}`);
|
|
952
1279
|
return false;
|
|
953
1280
|
}
|
|
954
1281
|
this.logError('request delivery failed', error);
|
|
@@ -988,7 +1315,21 @@ export class OwnerChannel {
|
|
|
988
1315
|
* agent itself can execute /force-restart, /model, or any other command.
|
|
989
1316
|
*/
|
|
990
1317
|
async handleCommand(sender, text, wireId) {
|
|
1318
|
+
const provisioningCommand = async (run) => {
|
|
1319
|
+
beginFleetAuditCollection();
|
|
1320
|
+
setFleetAuditLifecycleCheckpoint(events => this.notifyFleetLifecycle(events));
|
|
1321
|
+
try {
|
|
1322
|
+
return await run();
|
|
1323
|
+
}
|
|
1324
|
+
finally {
|
|
1325
|
+
setFleetAuditLifecycleCheckpoint(undefined);
|
|
1326
|
+
const remaining = consumeFleetAuditCollection().presentations;
|
|
1327
|
+
if (remaining?.length)
|
|
1328
|
+
await this.notifyFleetLifecycle(remaining);
|
|
1329
|
+
}
|
|
1330
|
+
};
|
|
991
1331
|
const ctx = {
|
|
1332
|
+
authenticatedCid: sender.id,
|
|
992
1333
|
role: this.options.role,
|
|
993
1334
|
harness: this.options.harness,
|
|
994
1335
|
version: VERSION,
|
|
@@ -1006,16 +1347,36 @@ export class OwnerChannel {
|
|
|
1006
1347
|
},
|
|
1007
1348
|
fleetList: () => this.fleetOps.list(),
|
|
1008
1349
|
closeRoom: roomId => this.closeRoomFromOwner(sender, roomId, wireId),
|
|
1009
|
-
recoverRoom: roomId => this.recoverRoomFromOwner(sender, roomId, wireId),
|
|
1010
1350
|
terminalTask: (taskId, kind, outcome) => this.terminalTaskFromOwner(sender, taskId, kind, outcome, wireId),
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1351
|
+
createTask: input => provisioningCommand(async () => {
|
|
1352
|
+
const service = new TaskRoomApplicationService(this.options.configPath);
|
|
1353
|
+
const task = await service.createTask({
|
|
1354
|
+
...input,
|
|
1355
|
+
actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id },
|
|
1356
|
+
});
|
|
1357
|
+
if (!input.backlog && !input.noRoom && task.room_id) {
|
|
1358
|
+
const outcome = service.taskProvisioningOutcome(task.task_id);
|
|
1359
|
+
if (outcome.kind === 'in_progress' && !outcome.next_action)
|
|
1360
|
+
await this.fleetOps.provisionTask(task.task_id);
|
|
1361
|
+
}
|
|
1362
|
+
return task;
|
|
1015
1363
|
}),
|
|
1016
|
-
startTask: taskId =>
|
|
1017
|
-
|
|
1364
|
+
startTask: taskId => provisioningCommand(async () => {
|
|
1365
|
+
const service = new TaskRoomApplicationService(this.options.configPath);
|
|
1366
|
+
const task = await service.startTask({
|
|
1367
|
+
actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id }, taskId,
|
|
1368
|
+
});
|
|
1369
|
+
// A zero-duration observation repairs a ready notice after a crash or
|
|
1370
|
+
// duplicate start without extending the start command's existing wait.
|
|
1371
|
+
const outcome = await service.awaitTaskProvisioning({
|
|
1372
|
+
actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id },
|
|
1373
|
+
taskId: task.task_id, waitMs: 0,
|
|
1374
|
+
});
|
|
1375
|
+
if (outcome.kind === 'in_progress' && !outcome.next_action)
|
|
1376
|
+
await this.fleetOps.provisionTask(task.task_id);
|
|
1377
|
+
return outcome;
|
|
1018
1378
|
}),
|
|
1379
|
+
taskProvisioningOutcome: taskId => new TaskRoomApplicationService(this.options.configPath).taskProvisioningOutcome(taskId),
|
|
1019
1380
|
listTasks: filter => new TaskRoomApplicationService(this.options.configPath).listTasks(filter),
|
|
1020
1381
|
groupedTasks: filter => new TaskRoomApplicationService(this.options.configPath).groupedTasks(filter),
|
|
1021
1382
|
listTaskLists: () => new TaskRoomApplicationService(this.options.configPath).listTaskLists(),
|
|
@@ -1041,19 +1402,29 @@ export class OwnerChannel {
|
|
|
1041
1402
|
reviewTask: taskId => new TaskRoomApplicationService(this.options.configPath).reviewTask({
|
|
1042
1403
|
actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id }, taskId,
|
|
1043
1404
|
}),
|
|
1044
|
-
deleteTask: taskId =>
|
|
1045
|
-
actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id }, taskId,
|
|
1046
|
-
}),
|
|
1405
|
+
deleteTask: taskId => this.deleteTaskFromOwner(sender, taskId, wireId),
|
|
1047
1406
|
listRoomQueries: filter => new TaskRoomApplicationService(this.options.configPath).listRooms(filter),
|
|
1048
1407
|
getRoomQuery: id => new TaskRoomApplicationService(this.options.configPath).getRoomDetail(id),
|
|
1049
1408
|
listTemplateQueries: () => new TaskRoomApplicationService(this.options.configPath).listTemplates(),
|
|
1050
1409
|
getTemplateQuery: name => new TaskRoomApplicationService(this.options.configPath).getTemplate(name),
|
|
1051
|
-
createRoom: input => new TaskRoomApplicationService(this.options.configPath).createRoom({
|
|
1410
|
+
createRoom: input => provisioningCommand(() => new TaskRoomApplicationService(this.options.configPath).createRoom({
|
|
1052
1411
|
...input, actor: { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id },
|
|
1053
|
-
}),
|
|
1412
|
+
})),
|
|
1054
1413
|
recentEvents: limit => this.options.session.eventsSince(0).slice(-limit),
|
|
1055
1414
|
readWorklogTail: maxChars => this.readWorklogTail(maxChars),
|
|
1056
1415
|
reply: async (replyText) => { await this.send(sender.id, replyText, wireId); },
|
|
1416
|
+
replyHtml: async (filename, html) => {
|
|
1417
|
+
const directory = join(this.options.stateDir, '.owner-channel-report-outbox', this.requestId(wireId));
|
|
1418
|
+
const path = join(directory, filename);
|
|
1419
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
1420
|
+
try {
|
|
1421
|
+
await writeFile(path, html, { flag: 'wx', mode: 0o600 });
|
|
1422
|
+
await this.client.sendFile({ contact: sender.id, path, filename, replyToWireId: wireId });
|
|
1423
|
+
}
|
|
1424
|
+
finally {
|
|
1425
|
+
await rm(directory, { recursive: true, force: true });
|
|
1426
|
+
}
|
|
1427
|
+
},
|
|
1057
1428
|
};
|
|
1058
1429
|
try {
|
|
1059
1430
|
await dispatchOwnerCommand(text, ctx);
|
|
@@ -1066,6 +1437,84 @@ export class OwnerChannel {
|
|
|
1066
1437
|
if (!this.inFlight.has(wireId))
|
|
1067
1438
|
this.state.remember(wireId);
|
|
1068
1439
|
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Register one typed adapter per primary slash command. The adapter performs
|
|
1442
|
+
* the live CID check before constructing the same slash text and entering the
|
|
1443
|
+
* existing dispatcher; its null SDK result avoids duplicating the ordinary
|
|
1444
|
+
* owner-channel replies that remain the command's result contract.
|
|
1445
|
+
*/
|
|
1446
|
+
registerTypedCommands() {
|
|
1447
|
+
const commands = ownerTypedCommandCatalog().map(definition => ({
|
|
1448
|
+
name: definition.name,
|
|
1449
|
+
description: definition.description,
|
|
1450
|
+
input_schema: definition.input_schema,
|
|
1451
|
+
handler: (input, context) => {
|
|
1452
|
+
this.typedHandlersEntered.add(context.request_wire_id);
|
|
1453
|
+
return this.handleTypedCommand(definition.name, input, context);
|
|
1454
|
+
},
|
|
1455
|
+
}));
|
|
1456
|
+
return this.client.registerCommands(commands);
|
|
1457
|
+
}
|
|
1458
|
+
async handleRecoveredTypedCommand(message, wireId) {
|
|
1459
|
+
let payload;
|
|
1460
|
+
try {
|
|
1461
|
+
const parsed = JSON.parse(String(message.body ?? message.text ?? ''));
|
|
1462
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)
|
|
1463
|
+
|| Object.keys(parsed).length !== 2
|
|
1464
|
+
|| typeof parsed.command !== 'string'
|
|
1465
|
+
|| !('arguments' in parsed))
|
|
1466
|
+
throw new Error('invalid command payload');
|
|
1467
|
+
payload = parsed;
|
|
1468
|
+
}
|
|
1469
|
+
catch {
|
|
1470
|
+
// The original SDK attempt would have emitted handler_failed without
|
|
1471
|
+
// entering Fleet. Recovery mirrors that terminal no-effect outcome.
|
|
1472
|
+
this.state.remember(wireId);
|
|
1473
|
+
return true;
|
|
1474
|
+
}
|
|
1475
|
+
const sender = this.sender(message);
|
|
1476
|
+
try {
|
|
1477
|
+
await this.handleTypedCommand(payload.command, payload.arguments, {
|
|
1478
|
+
sender_cid: sender.id, sender_name: sender.name, request_wire_id: wireId,
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1481
|
+
catch (error) {
|
|
1482
|
+
// Expected handler failures (unauthorized/invalid typed input) are
|
|
1483
|
+
// terminal and already recorded, just as the SDK swallows them.
|
|
1484
|
+
if (!this.state.has(wireId))
|
|
1485
|
+
throw error;
|
|
1486
|
+
}
|
|
1487
|
+
return true;
|
|
1488
|
+
}
|
|
1489
|
+
async handleTypedCommand(name, input, context) {
|
|
1490
|
+
const sender = { id: context.sender_cid, name: context.sender_name };
|
|
1491
|
+
const wireId = context.request_wire_id;
|
|
1492
|
+
if (!this.isEffectiveOwner(sender.id)) {
|
|
1493
|
+
this.options.log(`[${this.options.role}] owner channel ignored unauthorized typed-command sender `
|
|
1494
|
+
+ `${sender.id || '<unknown>'}`);
|
|
1495
|
+
await this.warnOwnerOfUnauthorizedSender(sender.id);
|
|
1496
|
+
if (wireId)
|
|
1497
|
+
this.state.remember(wireId);
|
|
1498
|
+
throw new Error('typed owner command denied');
|
|
1499
|
+
}
|
|
1500
|
+
try {
|
|
1501
|
+
try {
|
|
1502
|
+
this.conversations.recordInbound(sender.id, wireId);
|
|
1503
|
+
}
|
|
1504
|
+
catch (error) {
|
|
1505
|
+
this.logError('owner conversation route update failed', error);
|
|
1506
|
+
}
|
|
1507
|
+
await this.handleCommand(sender, ownerTypedCommandText(name, input), wireId);
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
catch (error) {
|
|
1511
|
+
// Structural typed-input failures are terminal handler_failed outcomes,
|
|
1512
|
+
// not indefinitely replayable work.
|
|
1513
|
+
if (wireId)
|
|
1514
|
+
this.state.remember(wireId);
|
|
1515
|
+
throw error;
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1069
1518
|
/** Queue raw slash text to the harness and report the turn's outcome. */
|
|
1070
1519
|
async runHarnessCommand(sender, command, wireId) {
|
|
1071
1520
|
const requestId = this.requestId(wireId);
|
|
@@ -1113,7 +1562,7 @@ export class OwnerChannel {
|
|
|
1113
1562
|
await this.send(sender.id, renderMarkdownFailure({
|
|
1114
1563
|
kind: 'pending', subject: `/room delete ${roomId} ${roomId}`,
|
|
1115
1564
|
detail: 'The deletion request was accepted and is still being settled.',
|
|
1116
|
-
action: `Run /room
|
|
1565
|
+
action: `Run /room delete ${roomId} ${roomId} again if deletion remains pending.`,
|
|
1117
1566
|
}), wireId);
|
|
1118
1567
|
this.state.remember(wireId);
|
|
1119
1568
|
try {
|
|
@@ -1126,78 +1575,35 @@ export class OwnerChannel {
|
|
|
1126
1575
|
throw error;
|
|
1127
1576
|
}
|
|
1128
1577
|
}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const actor = { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id };
|
|
1132
|
-
const result = await app.recoverRoom({ actor, roomId });
|
|
1133
|
-
if (result.kind === 'deletion_worker_required') {
|
|
1134
|
-
await this.send(sender.id, renderMarkdownFailure({ kind: 'pending',
|
|
1135
|
-
subject: `/room recover ${roomId}`,
|
|
1136
|
-
detail: 'The deletion recovery is still being settled.',
|
|
1137
|
-
action: `Run /room recover ${roomId} if deletion remains pending.` }), wireId);
|
|
1138
|
-
this.state.remember(wireId);
|
|
1139
|
-
try {
|
|
1140
|
-
await this.fleetOps.closeRoom(roomId);
|
|
1141
|
-
}
|
|
1142
|
-
catch (error) {
|
|
1143
|
-
await app.recordRoomSettlementError({ actor, roomId,
|
|
1144
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1145
|
-
recoveryHint: `External delete worker failed to start. Retry /room delete ${roomId} ${roomId}.` });
|
|
1146
|
-
throw error;
|
|
1147
|
-
}
|
|
1148
|
-
return;
|
|
1149
|
-
}
|
|
1150
|
-
const r = result.orchestration;
|
|
1151
|
-
await this.send(sender.id, renderMarkdownResult({ icon: '🛟', title: 'Room recovery',
|
|
1152
|
-
fields: [{ label: 'Room', value: result.room.room_id, kind: 'code' },
|
|
1153
|
-
{ label: 'Status', value: roomStatus(result.room.state), kind: 'markdown' },
|
|
1154
|
-
...(r ? [{ label: 'Saga', value: r.saga.phase, kind: 'code' }] : [])],
|
|
1155
|
-
sections: result.issues.length ? [{ heading: 'Next steps', items: result.issues }]
|
|
1156
|
-
: [{ heading: 'Result', items: ['No recovery action is needed.'] }] }), wireId);
|
|
1157
|
-
this.state.remember(wireId);
|
|
1158
|
-
}
|
|
1159
|
-
/** Carry a task terminal request through a worker that survives this role. */
|
|
1160
|
-
async recoverTaskFromOwner(sender, taskId, wireId) {
|
|
1578
|
+
/** Accept a permanent any-state deletion, acknowledge it, then hand cleanup to a durable worker. */
|
|
1579
|
+
async deleteTaskFromOwner(sender, taskId, wireId) {
|
|
1161
1580
|
const app = new TaskRoomApplicationService(this.options.configPath);
|
|
1162
1581
|
const actor = { kind: 'authenticated_owner', surface: 'messenger', cid: sender.id };
|
|
1163
|
-
const
|
|
1164
|
-
if (
|
|
1165
|
-
await this.send(sender.id,
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
action: `Run /task recover ${taskId} if it remains pending.`,
|
|
1582
|
+
const accepted = await app.requestTaskDeletion({ actor, taskId });
|
|
1583
|
+
if (accepted.status === 'already_absent') {
|
|
1584
|
+
await this.send(sender.id, renderMarkdownResult({
|
|
1585
|
+
icon: '🗑️', title: 'Task already absent',
|
|
1586
|
+
fields: [{ label: 'ID', value: taskId, kind: 'code' }],
|
|
1169
1587
|
}), wireId);
|
|
1170
1588
|
this.state.remember(wireId);
|
|
1171
|
-
try {
|
|
1172
|
-
await this.fleetOps.recoverTask(taskId);
|
|
1173
|
-
}
|
|
1174
|
-
catch (error) {
|
|
1175
|
-
await app.recordSettlementError({ actor, taskId,
|
|
1176
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1177
|
-
recoveryHint: `External settle worker failed to start. Retry /task recover ${taskId}.` });
|
|
1178
|
-
throw error;
|
|
1179
|
-
}
|
|
1180
1589
|
return;
|
|
1181
1590
|
}
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
:
|
|
1185
|
-
|
|
1186
|
-
: issue.code === 'member_failed' ? `Member failed at step ${issue.stepIndex}`
|
|
1187
|
-
: issue.code === 'resume_failed' ? `Resume failed: ${issue.error}`
|
|
1188
|
-
: issue.code === 'provisioning_resumed' ? 'Provisioning resumed successfully'
|
|
1189
|
-
: issue.code);
|
|
1190
|
-
await this.send(sender.id, renderMarkdownResult({
|
|
1191
|
-
icon: '🛟', title: 'Task recovery',
|
|
1192
|
-
fields: [{ label: 'Task', value: task.task_id, kind: 'code' },
|
|
1193
|
-
{ label: 'Status', value: taskStatus(task.state), kind: 'markdown' },
|
|
1194
|
-
...(room ? [{ label: 'Room', value: room.room_id, kind: 'code' },
|
|
1195
|
-
{ label: 'Room status', value: roomStatus(room.state), kind: 'markdown' },
|
|
1196
|
-
{ label: 'Saga', value: room.saga.phase, kind: 'code' }] : [])],
|
|
1197
|
-
sections: hints.length ? [{ heading: 'Next steps', items: hints }]
|
|
1198
|
-
: [{ heading: 'Result', items: ['No automated recovery action is available.'] }],
|
|
1591
|
+
await this.send(sender.id, renderMarkdownFailure({
|
|
1592
|
+
kind: 'pending', subject: `/task delete ${taskId} ${taskId}`,
|
|
1593
|
+
detail: 'The deletion was accepted; cleanup is settling in the background.',
|
|
1594
|
+
action: `Repeat /task delete ${taskId} ${taskId} if it remains pending.`,
|
|
1199
1595
|
}), wireId);
|
|
1200
1596
|
this.state.remember(wireId);
|
|
1597
|
+
try {
|
|
1598
|
+
await this.fleetOps.settleTaskDeletion(taskId);
|
|
1599
|
+
}
|
|
1600
|
+
catch (error) {
|
|
1601
|
+
await app.recordDeletionError({
|
|
1602
|
+
actor, taskId, error: error instanceof Error ? error.message : String(error),
|
|
1603
|
+
recoveryHint: `External delete worker failed to start. Retry /task delete ${taskId} ${taskId}.`,
|
|
1604
|
+
});
|
|
1605
|
+
throw error;
|
|
1606
|
+
}
|
|
1201
1607
|
}
|
|
1202
1608
|
async terminalTaskFromOwner(sender, taskId, kind, outcome, wireId) {
|
|
1203
1609
|
const app = new TaskRoomApplicationService(this.options.configPath);
|
|
@@ -1219,7 +1625,7 @@ export class OwnerChannel {
|
|
|
1219
1625
|
await this.send(sender.id, renderMarkdownFailure({
|
|
1220
1626
|
kind: 'pending', subject: `/task ${kind === 'done' ? 'done' : 'cancel'} ${taskId}`,
|
|
1221
1627
|
detail: 'The terminal request was accepted and is still being settled.',
|
|
1222
|
-
action: `
|
|
1628
|
+
action: `Repeat /task ${kind === 'done' ? 'done' : 'cancel'} ${taskId} if it remains pending.`,
|
|
1223
1629
|
}), wireId);
|
|
1224
1630
|
this.state.remember(wireId);
|
|
1225
1631
|
try {
|
|
@@ -1228,7 +1634,7 @@ export class OwnerChannel {
|
|
|
1228
1634
|
catch (error) {
|
|
1229
1635
|
await app.recordSettlementError({
|
|
1230
1636
|
actor, taskId, error: error instanceof Error ? error.message : String(error),
|
|
1231
|
-
recoveryHint: `External settle worker failed to start. Retry the identical task command
|
|
1637
|
+
recoveryHint: `External settle worker failed to start. Retry the identical task command.`,
|
|
1232
1638
|
});
|
|
1233
1639
|
throw error;
|
|
1234
1640
|
}
|
|
@@ -1686,7 +2092,8 @@ export class OwnerChannel {
|
|
|
1686
2092
|
startProgress(event);
|
|
1687
2093
|
// Automatic commentary is an ACP phase extension. Other backends and
|
|
1688
2094
|
// older adapters retain their established final-only behavior.
|
|
1689
|
-
if (this.options.session.
|
|
2095
|
+
if (this.options.session.capabilities?.messagePhases
|
|
2096
|
+
?? this.options.session.backend === 'acp')
|
|
1690
2097
|
acceptCommentary(event);
|
|
1691
2098
|
})
|
|
1692
2099
|
: () => undefined;
|
|
@@ -1728,7 +2135,8 @@ export class OwnerChannel {
|
|
|
1728
2135
|
enabled: this.commentsEnabled,
|
|
1729
2136
|
baseline: this.commentsBaseline,
|
|
1730
2137
|
// Only the ACP backend emits the phase marker commentary relaying needs.
|
|
1731
|
-
supported: this.options.session.
|
|
2138
|
+
supported: this.options.session.capabilities?.messagePhases
|
|
2139
|
+
?? this.options.session.backend === 'acp',
|
|
1732
2140
|
};
|
|
1733
2141
|
}
|
|
1734
2142
|
/** Model-authored commentary only; raw protocol/tool data never reaches here. */
|
|
@@ -1876,7 +2284,7 @@ export class OwnerChannel {
|
|
|
1876
2284
|
}
|
|
1877
2285
|
return undefined;
|
|
1878
2286
|
}
|
|
1879
|
-
async watchLoop() {
|
|
2287
|
+
async watchLoop(generation) {
|
|
1880
2288
|
const sleep = this.options.binderDeps?.sleep
|
|
1881
2289
|
?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
1882
2290
|
const restored = this.readWatchState();
|
|
@@ -1885,7 +2293,7 @@ export class OwnerChannel {
|
|
|
1885
2293
|
state = this.writeWatchState(state, 'OWNER_WATCH_STATE_RECOVERED');
|
|
1886
2294
|
let delayMs = 1_000;
|
|
1887
2295
|
let attempts = 0;
|
|
1888
|
-
while (!this.stopping) {
|
|
2296
|
+
while (!this.stopping && generation === this.watchGeneration) {
|
|
1889
2297
|
const ctrl = new AbortController();
|
|
1890
2298
|
this.watchAbort = ctrl;
|
|
1891
2299
|
try {
|
|
@@ -1895,12 +2303,12 @@ export class OwnerChannel {
|
|
|
1895
2303
|
// fall into a gap. Persistent history plus fleet's body-free claim
|
|
1896
2304
|
// journal is authoritative; durable wire-ID dedupe makes hints harmless.
|
|
1897
2305
|
await this.drain();
|
|
1898
|
-
if (this.stopping)
|
|
2306
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1899
2307
|
return;
|
|
1900
2308
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTING', { reconnected: attempts > 0 });
|
|
1901
2309
|
attempts++;
|
|
1902
2310
|
for await (const _event of this.client.watchNotifications(this.options.config.identity, { since: 0, signal: ctrl.signal })) {
|
|
1903
|
-
if (this.stopping)
|
|
2311
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1904
2312
|
return;
|
|
1905
2313
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTED', { resetFailures: true });
|
|
1906
2314
|
delayMs = 1_000;
|
|
@@ -1909,11 +2317,11 @@ export class OwnerChannel {
|
|
|
1909
2317
|
// recorded before a managed request is dispatched.
|
|
1910
2318
|
await this.drain();
|
|
1911
2319
|
}
|
|
1912
|
-
if (!this.stopping)
|
|
2320
|
+
if (!this.stopping && generation === this.watchGeneration)
|
|
1913
2321
|
throw new Error('ours SDK notification stream ended');
|
|
1914
2322
|
}
|
|
1915
2323
|
catch (error) {
|
|
1916
|
-
if (this.stopping)
|
|
2324
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1917
2325
|
return;
|
|
1918
2326
|
// SDK 2 deliberately hides transport status behind its typed stream.
|
|
1919
2327
|
// Do not parse error prose to rediscover it: every failure follows the
|