mtech-ai-local-runner 0.1.31 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -1
- package/dist/src/adapters/claude-code/adapter.js +76 -17
- package/dist/src/adapters/codex/adapter.js +131 -30
- package/dist/src/adapters/codex/exec-launch-host.js +28 -11
- package/dist/src/adapters/codex/stream-parser.js +8 -17
- package/dist/src/adapters/completion-report.js +8 -5
- package/dist/src/daemon/daemon.js +62 -4
- package/dist/src/daemon/execution-coordinator.js +325 -111
- package/dist/src/daemon/job-poller.js +4 -1
- package/dist/src/daemon/manual-codex.js +10 -3
- package/dist/src/daemon/preparation-lease.js +142 -0
- package/dist/src/daemon/runtime.js +51 -25
- package/dist/src/daemon/terminal-recovery.js +29 -5
- package/dist/src/gateway-client.js +53 -22
- package/dist/src/process/node-process-group-host.js +72 -52
- package/dist/src/process/owned-process-guardian.js +140 -0
- package/dist/src/publication/installed-host.js +31 -1
- package/dist/src/publication/production-host.js +7 -3
- package/dist/src/publication/server-ci-error.js +15 -0
- package/dist/src/publication/server-ci-host.js +160 -0
- package/dist/src/runner.js +33 -4
- package/dist/src/safety/codex-daemon-host.js +2 -2
- package/dist/src/safety/execution-failure.js +4 -0
- package/dist/src/safety/terminal-evidence.js +67 -16
- package/dist/src/safety/unsupported-supervisor-command.js +12 -0
- package/dist/src/state-store.js +50 -7
- package/package.json +1 -1
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { PublicationOutcomeUnknownError } from '../publication/publisher.js';
|
|
2
2
|
import { executionError, executionFailure } from '../safety/execution-failure.js';
|
|
3
|
+
import { isUnsupportedSupervisorCommand } from '../safety/unsupported-supervisor-command.js';
|
|
3
4
|
import { requireSecurityRequirements } from '../safety/security-requirements.js';
|
|
4
5
|
import { GatewayToolError } from "../gateway-client.js";
|
|
5
6
|
import { createHash } from "node:crypto";
|
|
6
7
|
import { completionReportDiagnostic } from "../adapters/completion-report.js";
|
|
7
8
|
import { PROTOCOL_VERSION, isTerminalEventType, validateEvent, MAX_EVENT_SUMMARY_CHARS } from "../protocol.js";
|
|
9
|
+
import { DAEMON_BLOCKED_REASONS, DAEMON_CANCELLED_REASONS } from "../state-store.js";
|
|
10
|
+
import { PreparationDeadlineError, PreparationLease, validateJobLease } from './preparation-lease.js';
|
|
8
11
|
function canonical(value) {
|
|
9
12
|
if (Array.isArray(value))
|
|
10
13
|
return value.map(canonical);
|
|
@@ -41,10 +44,19 @@ function immutable(value) {
|
|
|
41
44
|
}
|
|
42
45
|
return value;
|
|
43
46
|
}
|
|
44
|
-
class PreparationDeadlineError extends Error {
|
|
45
|
-
}
|
|
46
47
|
class LocalFixHostRequired extends Error {
|
|
47
48
|
}
|
|
49
|
+
function daemonOutcome(event) {
|
|
50
|
+
const source = event.eventType;
|
|
51
|
+
const suppliedReason = event.payload?.reason;
|
|
52
|
+
const reason = source === 'blocked' ? 'executor_blocked'
|
|
53
|
+
: source === 'question' ? 'executor_question' : source === 'scope_conflict' ? 'scope_conflict'
|
|
54
|
+
: source === 'cancelled' ? (DAEMON_CANCELLED_REASONS.includes(suppliedReason) ? suppliedReason : 'cancelled')
|
|
55
|
+
: source === 'failed' && DAEMON_BLOCKED_REASONS.includes(suppliedReason) ? suppliedReason : undefined;
|
|
56
|
+
const eventType = ['blocked', 'question', 'scope_conflict'].includes(source) ? 'failed' : source;
|
|
57
|
+
return { event: { ...event, eventType, summary: event.summary.slice(0, MAX_EVENT_SUMMARY_CHARS), ...(reason ? { payload: { ...event.payload, reason } } : {}) },
|
|
58
|
+
status: eventType === 'completed' ? 'completed' : eventType === 'cancelled' ? 'cancelled' : reason ? 'blocked' : 'failed', ...(reason ? { reason } : {}) };
|
|
59
|
+
}
|
|
48
60
|
export class ExecutionCoordinator {
|
|
49
61
|
deps;
|
|
50
62
|
constructor(deps) {
|
|
@@ -61,22 +73,12 @@ export class ExecutionCoordinator {
|
|
|
61
73
|
const isLocalFix = job.taskPackage.local_fix_scope !== undefined;
|
|
62
74
|
const lifecycle = { frozen: false, current: true };
|
|
63
75
|
const deadline = Date.now() + (this.deps.preparationTimeoutMs ?? 900_000);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
let timer;
|
|
69
|
-
try {
|
|
70
|
-
return await Promise.race([operation(), new Promise((_, reject) => {
|
|
71
|
-
timer = setTimeout(() => reject(new PreparationDeadlineError()), remaining);
|
|
72
|
-
})]);
|
|
73
|
-
}
|
|
74
|
-
finally {
|
|
75
|
-
if (timer)
|
|
76
|
-
clearTimeout(timer);
|
|
77
|
-
}
|
|
78
|
-
};
|
|
76
|
+
let leaseExpiresAt = job.leaseExpiresAt;
|
|
77
|
+
const preparation = new PreparationLease(job, deadline, signal => this.deps.gateway.renewJob(job, signal), expiry => { this.deps.gateway.confirmJobLease?.(job, expiry); leaseExpiresAt = expiry; });
|
|
78
|
+
const step = (operation) => preparation.step(operation);
|
|
79
|
+
let bindingAttempted = false;
|
|
79
80
|
try {
|
|
81
|
+
await preparation.start();
|
|
80
82
|
const pkg = job.taskPackage;
|
|
81
83
|
if (isLocalFix && (!this.deps.localFixHost || pkg.repository?.access !== "write"))
|
|
82
84
|
throw new LocalFixHostRequired();
|
|
@@ -87,7 +89,7 @@ export class ExecutionCoordinator {
|
|
|
87
89
|
requireSecurityRequirements(pkg.security_requirements, { adapterId, adapterVersion });
|
|
88
90
|
if (!/^[a-f0-9]{64}$/.test(job.launchFingerprint ?? '') || job.gatewayContractFingerprint !== expectedContractFingerprint
|
|
89
91
|
|| job.gatewayTaskPackageFingerprint !== fingerprint(pkg))
|
|
90
|
-
throw new
|
|
92
|
+
throw new GatewayToolError("Claimed launch fingerprints differ", 'local_agent_jobs_wait', 'JOB_LAUNCH_FENCE_INVALID');
|
|
91
93
|
stage = "workspace";
|
|
92
94
|
workspace = await step(() => this.deps.workspaces.create({
|
|
93
95
|
jobId: job.id,
|
|
@@ -97,21 +99,35 @@ export class ExecutionCoordinator {
|
|
|
97
99
|
...(isLocalFix ? { workingBranch: text(repository?.working_branch, "approved fix branch") } : {})
|
|
98
100
|
}));
|
|
99
101
|
stage = "adapter";
|
|
100
|
-
adapter = await step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }));
|
|
101
|
-
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest });
|
|
102
|
+
adapter = await preparation.step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }), late => late.cancel());
|
|
103
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest }));
|
|
102
104
|
stage = "shadow";
|
|
103
|
-
const gatewayShadow = await step(() => this.deps.gateway.shadow(job));
|
|
105
|
+
const gatewayShadow = await step(() => this.deps.gateway.shadow(job, preparation.signal));
|
|
104
106
|
stage = "fingerprint";
|
|
105
107
|
if (contractFingerprint(gatewayShadow.contractPayload) !== expectedContractFingerprint || gatewayShadow.contractFingerprint !== expectedContractFingerprint || gatewayShadow.taskPackageFingerprint !== fingerprint(pkg)) {
|
|
106
|
-
throw new
|
|
108
|
+
throw new GatewayToolError("contract or task-package fingerprint mismatch before binding", 'work_execution_contract_show', 'JOB_SHADOW_FENCE_INVALID');
|
|
107
109
|
}
|
|
108
110
|
stage = "binding";
|
|
109
|
-
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" });
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" }));
|
|
112
|
+
preparation.beginBinding();
|
|
113
|
+
bindingAttempted = true;
|
|
114
|
+
binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }, preparation.signal));
|
|
115
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest }));
|
|
116
|
+
await preparation.stop();
|
|
117
|
+
preparation.assert();
|
|
113
118
|
}
|
|
114
119
|
catch (error) {
|
|
120
|
+
if (stage === 'fingerprint')
|
|
121
|
+
preparation.fail(error);
|
|
122
|
+
if (error instanceof GatewayToolError && !['GATEWAY_TRANSPORT_ERROR', 'GATEWAY_HTTP_500', 'GATEWAY_HTTP_502', 'GATEWAY_HTTP_503', 'GATEWAY_HTTP_504'].includes(error.code))
|
|
123
|
+
preparation.fail(error);
|
|
124
|
+
try {
|
|
125
|
+
await preparation.stop();
|
|
126
|
+
}
|
|
127
|
+
catch { /* The original error and latched fence remain authoritative. */ }
|
|
128
|
+
preparation.dispose();
|
|
129
|
+
if (adapter)
|
|
130
|
+
await adapter.cancel();
|
|
115
131
|
const diagnostic = { stage, code: "PREPARATION_FAILED" };
|
|
116
132
|
if (error instanceof LocalFixHostRequired)
|
|
117
133
|
diagnostic.code = "LOCAL_FIX_HOST_REQUIRED";
|
|
@@ -122,50 +138,137 @@ export class ExecutionCoordinator {
|
|
|
122
138
|
diagnostic.code = error.code;
|
|
123
139
|
}
|
|
124
140
|
this.deps.reportPreparationFailure?.(diagnostic);
|
|
125
|
-
if (
|
|
141
|
+
if (bindingAttempted && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
|
|
126
142
|
// The server may have committed the binding. Never fabricate a blocked result
|
|
127
143
|
// or repeat the mutation before the deterministic binding reference is reconciled.
|
|
128
144
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
129
145
|
phase: "binding_uncertain", preparationFailure: diagnostic });
|
|
130
146
|
throw new Error("Binding outcome requires reconciliation");
|
|
131
147
|
}
|
|
148
|
+
if (!preparation.hasLiveFence || (error instanceof PreparationDeadlineError && stage === 'adapter'))
|
|
149
|
+
throw error;
|
|
132
150
|
// Retain this exact safe terminal payload until Gateway acknowledges it.
|
|
133
151
|
const completionRequest = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
134
152
|
status: "blocked", result: { preparation_failure: { ...diagnostic } } };
|
|
135
153
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
136
154
|
phase: "terminal_pending", preparationFailure: diagnostic, completionRequest: structuredClone(completionRequest) });
|
|
155
|
+
if (!preparation.hasLiveFence) {
|
|
156
|
+
preparation.dispose();
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
137
159
|
await this.deps.gateway.complete(completionRequest);
|
|
138
160
|
await this.deps.state.clear();
|
|
161
|
+
preparation.dispose();
|
|
139
162
|
return;
|
|
140
163
|
}
|
|
141
|
-
|
|
164
|
+
const executor = adapter;
|
|
142
165
|
let heartbeatFailure;
|
|
166
|
+
const evidenceAbort = new AbortController();
|
|
167
|
+
const controlAbort = new AbortController();
|
|
168
|
+
let leaseBoundary;
|
|
169
|
+
let rejectFence;
|
|
170
|
+
const lostFence = new Promise((_, reject) => { rejectFence = reject; });
|
|
171
|
+
void lostFence.catch(() => { });
|
|
143
172
|
let executionStage = 'binding_model';
|
|
144
173
|
let requestedTerminal;
|
|
174
|
+
let rejectControlStop;
|
|
175
|
+
const controlStopped = new Promise((_, reject) => { rejectControlStop = reject; });
|
|
176
|
+
void controlStopped.catch(() => { });
|
|
177
|
+
let publicationPending = false;
|
|
145
178
|
let commandCursor = 0;
|
|
146
179
|
let controlsStopped = false;
|
|
147
180
|
let controlsInFlight;
|
|
181
|
+
let executorStop;
|
|
182
|
+
let stopUnconfirmed = false;
|
|
183
|
+
let rejectUnconfirmedStop;
|
|
184
|
+
const unconfirmedStop = new Promise((_, reject) => { rejectUnconfirmedStop = reject; });
|
|
185
|
+
void unconfirmedStop.catch(() => { });
|
|
186
|
+
const stopExecutor = () => {
|
|
187
|
+
if (!executorStop) {
|
|
188
|
+
try {
|
|
189
|
+
executorStop = executor.cancel();
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
executorStop = Promise.reject(error);
|
|
193
|
+
}
|
|
194
|
+
void executorStop.catch(error => { stopUnconfirmed = true; rejectUnconfirmedStop(error); failFence(error); });
|
|
195
|
+
}
|
|
196
|
+
return executorStop;
|
|
197
|
+
};
|
|
198
|
+
const failFence = (error) => {
|
|
199
|
+
if (heartbeatFailure)
|
|
200
|
+
return;
|
|
201
|
+
lifecycle.current = false;
|
|
202
|
+
heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
|
|
203
|
+
controlAbort.abort();
|
|
204
|
+
evidenceAbort.abort();
|
|
205
|
+
rejectFence(heartbeatFailure);
|
|
206
|
+
// Cancellation cannot queue behind transport or generator cleanup.
|
|
207
|
+
void stopExecutor().catch(() => { });
|
|
208
|
+
};
|
|
209
|
+
const requireControlFence = () => {
|
|
210
|
+
if (!heartbeatFailure && (leaseExpiresAt === undefined || Date.now() >= leaseExpiresAt * 1000))
|
|
211
|
+
failFence(new GatewayToolError('Job lease expired', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID'));
|
|
212
|
+
if (heartbeatFailure)
|
|
213
|
+
throw heartbeatFailure;
|
|
214
|
+
};
|
|
215
|
+
const armLeaseBoundary = () => {
|
|
216
|
+
if (leaseBoundary)
|
|
217
|
+
clearTimeout(leaseBoundary);
|
|
218
|
+
leaseBoundary = setTimeout(() => { try {
|
|
219
|
+
requireControlFence();
|
|
220
|
+
}
|
|
221
|
+
catch { /* The latched fence rejects all pending operations. */ } }, Math.max(0, (leaseExpiresAt ?? 0) * 1000 - Date.now()));
|
|
222
|
+
leaseBoundary.unref?.();
|
|
223
|
+
};
|
|
224
|
+
const fenced = (work) => Promise.race([work, lostFence]);
|
|
225
|
+
armLeaseBoundary();
|
|
148
226
|
const readControls = async () => {
|
|
227
|
+
requireControlFence();
|
|
149
228
|
if (!this.deps.gateway.readCommands || controlsStopped || requestedTerminal)
|
|
150
229
|
return;
|
|
151
230
|
if (controlsInFlight)
|
|
152
231
|
return controlsInFlight;
|
|
153
232
|
controlsInFlight = (async () => {
|
|
154
|
-
|
|
233
|
+
const commands = await fenced(this.deps.gateway.readCommands(binding.bindingId, commandCursor, controlAbort.signal));
|
|
234
|
+
requireControlFence();
|
|
235
|
+
for (const command of commands) {
|
|
236
|
+
requireControlFence();
|
|
155
237
|
if (controlsStopped)
|
|
156
238
|
return;
|
|
239
|
+
let commandTerminal;
|
|
157
240
|
if (["cancel", "pause", "request_replan"].includes(command.command)) {
|
|
158
241
|
lifecycle.current = false;
|
|
159
|
-
|
|
160
|
-
|
|
242
|
+
commandTerminal = { eventType: "cancelled", summary: `supervisor ${command.command} acknowledged`,
|
|
243
|
+
payload: { reason: command.command === 'pause' ? 'paused' : command.command === 'request_replan' ? 'replan_requested' : 'cancelled' } };
|
|
244
|
+
await stopExecutor();
|
|
161
245
|
}
|
|
162
246
|
else {
|
|
163
|
-
|
|
247
|
+
try {
|
|
248
|
+
await fenced(adapter.command(command));
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
requireControlFence();
|
|
252
|
+
// Classify the host capability outcome before executionError wraps it.
|
|
253
|
+
if (!isUnsupportedSupervisorCommand(error))
|
|
254
|
+
throw error;
|
|
255
|
+
lifecycle.current = false;
|
|
256
|
+
await stopExecutor();
|
|
257
|
+
requireControlFence();
|
|
258
|
+
commandTerminal = { eventType: 'failed', summary: 'supervisor command is unsupported for the approved launch',
|
|
259
|
+
payload: { reason: 'unsupported_supervisor_command' } };
|
|
260
|
+
}
|
|
164
261
|
}
|
|
262
|
+
requireControlFence();
|
|
165
263
|
if (!this.deps.gateway.acknowledgeCommand)
|
|
166
264
|
throw new Error("daemon command acknowledgement is unavailable");
|
|
167
|
-
await this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence);
|
|
265
|
+
await fenced(this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence, controlAbort.signal));
|
|
266
|
+
requireControlFence();
|
|
168
267
|
commandCursor = command.sequence;
|
|
268
|
+
if (commandTerminal) {
|
|
269
|
+
requestedTerminal = commandTerminal;
|
|
270
|
+
rejectControlStop(new Error('Supervisor stopped execution'));
|
|
271
|
+
}
|
|
169
272
|
if (requestedTerminal)
|
|
170
273
|
break;
|
|
171
274
|
}
|
|
@@ -180,24 +283,60 @@ export class ExecutionCoordinator {
|
|
|
180
283
|
controlsInFlight = undefined;
|
|
181
284
|
}
|
|
182
285
|
};
|
|
286
|
+
let heartbeatInFlight;
|
|
183
287
|
const heartbeatTimer = this.deps.gateway.heartbeat || this.deps.gateway.readCommands ? setInterval(() => {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
await
|
|
188
|
-
|
|
288
|
+
if (requestedTerminal || heartbeatInFlight)
|
|
289
|
+
return;
|
|
290
|
+
heartbeatInFlight = (async () => {
|
|
291
|
+
await readControls();
|
|
292
|
+
requireControlFence();
|
|
293
|
+
if (!requestedTerminal && this.deps.gateway.heartbeat) {
|
|
294
|
+
const renewed = await fenced(this.deps.gateway.heartbeat(binding.bindingId, job, controlAbort.signal));
|
|
295
|
+
requireControlFence();
|
|
296
|
+
const expiry = validateJobLease(renewed, job, true);
|
|
297
|
+
if (!['running', 'review_pending'].includes(renewed.job.status))
|
|
298
|
+
throw new GatewayToolError('Running Job lease status is invalid', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID');
|
|
299
|
+
this.deps.gateway.confirmJobLease?.(job, expiry);
|
|
300
|
+
leaseExpiresAt = expiry;
|
|
301
|
+
armLeaseBoundary();
|
|
302
|
+
}
|
|
303
|
+
})().catch((error) => {
|
|
304
|
+
failFence(error);
|
|
305
|
+
}).finally(() => { heartbeatInFlight = undefined; });
|
|
189
306
|
}, 30_000) : undefined;
|
|
190
307
|
heartbeatTimer?.unref();
|
|
191
308
|
let terminal;
|
|
192
309
|
let failureDiagnostic;
|
|
193
310
|
let sequence = 0;
|
|
194
311
|
let checkpointSeen = false;
|
|
312
|
+
let eventUncertain = false;
|
|
313
|
+
const stop = async (ignoreControlFailure = false) => {
|
|
314
|
+
controlsStopped = true;
|
|
315
|
+
const cancelled = stopExecutor();
|
|
316
|
+
if (stopUnconfirmed)
|
|
317
|
+
await cancelled;
|
|
318
|
+
let controlFailure;
|
|
319
|
+
try {
|
|
320
|
+
if (controlsInFlight)
|
|
321
|
+
await controlsInFlight;
|
|
322
|
+
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
controlFailure = error;
|
|
325
|
+
}
|
|
326
|
+
if (heartbeatInFlight)
|
|
327
|
+
await heartbeatInFlight;
|
|
328
|
+
await cancelled;
|
|
329
|
+
if (!ignoreControlFailure && (controlFailure || heartbeatFailure))
|
|
330
|
+
throw controlFailure || heartbeatFailure;
|
|
331
|
+
};
|
|
195
332
|
const append = async (event) => {
|
|
196
333
|
const previousStage = executionStage;
|
|
197
334
|
let payload = event.payload;
|
|
198
335
|
if (isTerminalEventType(event.eventType)) {
|
|
199
336
|
executionStage = 'terminal_evidence';
|
|
200
|
-
|
|
337
|
+
requireControlFence();
|
|
338
|
+
const evidence = localFixPublication && typeof localFixPublication.diff_hash === 'string' && typeof localFixPublication.changed_file_manifest_hash === 'string' ? { diffHash: localFixPublication.diff_hash, changedFileManifestHash: localFixPublication.changed_file_manifest_hash } : await fenced(this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha, signal: evidenceAbort.signal }));
|
|
339
|
+
requireControlFence();
|
|
201
340
|
payload = { ...payload, diff_hash: evidence.diffHash, changed_file_manifest_hash: evidence.changedFileManifestHash };
|
|
202
341
|
}
|
|
203
342
|
// A vendor report cannot supply host observations, even when its shape looks valid.
|
|
@@ -224,112 +363,183 @@ export class ExecutionCoordinator {
|
|
|
224
363
|
validateEvent(normalized);
|
|
225
364
|
if (!this.deps.gateway.appendEvent)
|
|
226
365
|
throw new Error("daemon event reporting is unavailable");
|
|
227
|
-
|
|
366
|
+
// Keep lease renewal responsive during evidence collection, then settle
|
|
367
|
+
// its last fence before the terminal event consumes the execution lease.
|
|
368
|
+
if (isTerminalEventType(event.eventType)) {
|
|
369
|
+
if (heartbeatTimer)
|
|
370
|
+
clearInterval(heartbeatTimer);
|
|
371
|
+
if (heartbeatInFlight)
|
|
372
|
+
await heartbeatInFlight;
|
|
373
|
+
}
|
|
374
|
+
requireControlFence();
|
|
375
|
+
eventUncertain = true;
|
|
376
|
+
await fenced(this.deps.gateway.appendEvent(binding.bindingId, normalized));
|
|
377
|
+
requireControlFence();
|
|
228
378
|
sequence = normalized.sequence;
|
|
229
379
|
executionStage = 'state_save';
|
|
230
380
|
if (event.eventType === "checkpoint_ready")
|
|
231
381
|
checkpointSeen = true;
|
|
232
382
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, eventCursor: sequence, ...(isTerminalEventType(event.eventType) ? { terminalEventType: event.eventType } : {}), workspaceManifest: workspace.manifest });
|
|
383
|
+
eventUncertain = false;
|
|
233
384
|
executionStage = previousStage;
|
|
234
385
|
};
|
|
386
|
+
const publicationStep = async (operation) => {
|
|
387
|
+
requireControlFence();
|
|
388
|
+
if (requestedTerminal)
|
|
389
|
+
throw new Error('Supervisor stopped execution');
|
|
390
|
+
const value = await fenced(Promise.race([operation(), controlStopped]));
|
|
391
|
+
requireControlFence();
|
|
392
|
+
if (requestedTerminal)
|
|
393
|
+
throw new Error('Supervisor stopped execution');
|
|
394
|
+
return value;
|
|
395
|
+
};
|
|
235
396
|
try {
|
|
236
397
|
if (!job.taskPackage.resolved_model_id
|
|
237
398
|
|| binding.resolvedModelId !== job.taskPackage.resolved_model_id)
|
|
238
399
|
throw new Error("Binding model differs from approved launch");
|
|
239
400
|
executionStage = 'adapter_start';
|
|
240
|
-
|
|
401
|
+
preparation.assert();
|
|
402
|
+
requireControlFence();
|
|
403
|
+
if (isLocalFix) {
|
|
404
|
+
await readControls();
|
|
405
|
+
if (!requestedTerminal)
|
|
406
|
+
fixSession = await publicationStep(() => this.deps.localFixHost.prepare({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, lifecycle: {
|
|
407
|
+
isFrozen: () => lifecycle.frozen,
|
|
408
|
+
isCurrent: () => lifecycle.current && !requestedTerminal && !heartbeatFailure && leaseExpiresAt !== undefined && Date.now() < leaseExpiresAt * 1000,
|
|
409
|
+
securityReceipt: () => adapter.securityReceipt?.(),
|
|
410
|
+
} }).then(async (session) => {
|
|
411
|
+
if (!lifecycle.current || requestedTerminal || heartbeatFailure) {
|
|
412
|
+
await session.close();
|
|
413
|
+
throw new Error('Local fix preparation lost its lease');
|
|
414
|
+
}
|
|
415
|
+
return session;
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
418
|
+
if (!requestedTerminal) {
|
|
419
|
+
await fenced(Promise.race([adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId }), unconfirmedStop]));
|
|
420
|
+
}
|
|
241
421
|
executionStage = 'commands';
|
|
242
422
|
await readControls();
|
|
243
423
|
executionStage = 'event_stream';
|
|
244
424
|
if (adapter.events && !requestedTerminal) {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
if (controlsInFlight)
|
|
255
|
-
await controlsInFlight;
|
|
425
|
+
const iterator = adapter.events()[Symbol.asyncIterator]();
|
|
426
|
+
try {
|
|
427
|
+
while (true) {
|
|
428
|
+
const next = await fenced(Promise.race([iterator.next(), unconfirmedStop]));
|
|
429
|
+
if (next.done)
|
|
430
|
+
break;
|
|
431
|
+
const event = next.value;
|
|
432
|
+
if (heartbeatFailure)
|
|
433
|
+
throw heartbeatFailure;
|
|
256
434
|
if (requestedTerminal)
|
|
257
435
|
break;
|
|
258
|
-
if (event.eventType
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
if (
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
else
|
|
267
|
-
await adapter.cancel();
|
|
268
|
-
lifecycle.frozen = true;
|
|
269
|
-
await readControls();
|
|
270
|
-
if (requestedTerminal || heartbeatFailure)
|
|
271
|
-
break;
|
|
272
|
-
executionStage = 'local_fix_publication';
|
|
273
|
-
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } });
|
|
274
|
-
localFixPublication = await fixSession.finalize();
|
|
275
|
-
if (this.deps.gateway.syncPublicationCursor) {
|
|
276
|
-
try {
|
|
277
|
-
const cursor = await this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence);
|
|
278
|
-
if (!Number.isSafeInteger(cursor) || cursor < sequence)
|
|
279
|
-
throw new Error('Invalid event cursor');
|
|
280
|
-
sequence = cursor;
|
|
436
|
+
if (isTerminalEventType(event.eventType)) {
|
|
437
|
+
if (event.eventType === "completed" && fixSession) {
|
|
438
|
+
// Stop only this owned model before running verified host publication.
|
|
439
|
+
// Controls/heartbeat remain active throughout the publication phase.
|
|
440
|
+
if (this.deps.localFixHost?.requiresOwnedFreeze) {
|
|
441
|
+
if (!adapter.freezeForPublication)
|
|
442
|
+
throw new Error('LOCAL_FIX_OWNED_FREEZE_UNAVAILABLE');
|
|
443
|
+
await publicationStep(() => adapter.freezeForPublication());
|
|
281
444
|
}
|
|
282
|
-
|
|
283
|
-
|
|
445
|
+
else
|
|
446
|
+
await publicationStep(() => stopExecutor());
|
|
447
|
+
lifecycle.frozen = true;
|
|
448
|
+
await readControls();
|
|
449
|
+
if (requestedTerminal || heartbeatFailure)
|
|
450
|
+
break;
|
|
451
|
+
executionStage = 'local_fix_publication';
|
|
452
|
+
if (!checkpointSeen)
|
|
453
|
+
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
454
|
+
await publicationStep(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } }));
|
|
455
|
+
publicationPending = true;
|
|
456
|
+
localFixPublication = await publicationStep(() => fixSession.finalize());
|
|
457
|
+
if (this.deps.gateway.syncPublicationCursor) {
|
|
458
|
+
try {
|
|
459
|
+
const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence));
|
|
460
|
+
if (!Number.isSafeInteger(cursor) || cursor < sequence)
|
|
461
|
+
throw new Error('Invalid event cursor');
|
|
462
|
+
sequence = cursor;
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
throw new PublicationOutcomeUnknownError();
|
|
466
|
+
}
|
|
284
467
|
}
|
|
468
|
+
if (controlsInFlight)
|
|
469
|
+
await controlsInFlight;
|
|
470
|
+
requireControlFence();
|
|
471
|
+
if (requestedTerminal)
|
|
472
|
+
throw new PublicationOutcomeUnknownError();
|
|
473
|
+
executionStage = 'event_stream';
|
|
285
474
|
}
|
|
475
|
+
}
|
|
476
|
+
if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict", "question"].includes(event.eventType)) {
|
|
477
|
+
terminal = event;
|
|
478
|
+
void stopExecutor().catch(() => { });
|
|
479
|
+
// Resolve already-issued controls before choosing the terminal receipt,
|
|
480
|
+
// and stop writes before the generator's deferred cleanup or evidence.
|
|
286
481
|
if (controlsInFlight)
|
|
287
|
-
await controlsInFlight;
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
executionStage = 'event_stream';
|
|
482
|
+
await Promise.race([controlsInFlight, unconfirmedStop]);
|
|
483
|
+
await stop();
|
|
484
|
+
break;
|
|
291
485
|
}
|
|
292
|
-
|
|
486
|
+
await append(event);
|
|
487
|
+
await readControls();
|
|
293
488
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
489
|
+
}
|
|
490
|
+
finally {
|
|
491
|
+
// A failed owned stop cannot wait behind a blocked generator next().
|
|
492
|
+
// Keep disposal attached, but return to durable suspension promptly.
|
|
493
|
+
if (stopUnconfirmed || heartbeatFailure) {
|
|
494
|
+
try {
|
|
495
|
+
void Promise.resolve(iterator.return?.()).catch(() => { });
|
|
496
|
+
}
|
|
497
|
+
catch { /* Unconfirmed cleanup already fences reporting. */ }
|
|
299
498
|
}
|
|
300
|
-
|
|
499
|
+
else
|
|
500
|
+
await iterator.return?.();
|
|
301
501
|
}
|
|
302
502
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
503
|
+
await stop();
|
|
504
|
+
if (publicationPending && requestedTerminal)
|
|
505
|
+
throw new PublicationOutcomeUnknownError();
|
|
506
|
+
if (requestedTerminal)
|
|
306
507
|
terminal = requestedTerminal;
|
|
307
|
-
await append(terminal);
|
|
308
|
-
}
|
|
309
|
-
if (heartbeatFailure)
|
|
310
|
-
throw heartbeatFailure;
|
|
311
508
|
if (!terminal) {
|
|
312
509
|
terminal = { eventType: "failed", summary: "adapter ended without a terminal event" };
|
|
313
|
-
await append(terminal);
|
|
314
510
|
}
|
|
511
|
+
terminal = daemonOutcome(terminal).event;
|
|
512
|
+
if (terminal.eventType === 'completed' && !checkpointSeen)
|
|
513
|
+
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
514
|
+
await append(terminal);
|
|
315
515
|
}
|
|
316
516
|
catch (error) {
|
|
317
|
-
|
|
517
|
+
await stop(true);
|
|
518
|
+
if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
|
|
318
519
|
// The provider may already have committed/pushed/opened a PR. Do not
|
|
319
520
|
// release this run as failed or let restart begin another model session.
|
|
320
521
|
throw new Error('Local publication outcome requires reconciliation');
|
|
321
522
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
523
|
+
if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
|
|
524
|
+
terminal = daemonOutcome(requestedTerminal).event;
|
|
525
|
+
}
|
|
526
|
+
else {
|
|
527
|
+
// Do not complete a job whose terminal evidence was not acknowledged.
|
|
528
|
+
// A safe bounded reason prevents raw adapter/transport errors leaking.
|
|
529
|
+
const diagnostic = executionFailure(error, executionStage);
|
|
530
|
+
failureDiagnostic = diagnostic;
|
|
531
|
+
// Log before reporting so an unavailable Gateway does not erase the cause.
|
|
532
|
+
try {
|
|
533
|
+
this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
|
|
534
|
+
}
|
|
535
|
+
catch { /* logging must not change terminal reconciliation */ }
|
|
536
|
+
if (heartbeatFailure)
|
|
537
|
+
throw error;
|
|
538
|
+
if (eventUncertain || ['GATEWAY_TOOL_REJECTED', 'GATEWAY_TRANSPORT_ERROR', 'WORK_LOOP_CANCELLED'].includes(diagnostic.code)
|
|
539
|
+
|| diagnostic.stage === 'terminal_evidence')
|
|
540
|
+
throw error;
|
|
541
|
+
terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
|
|
330
542
|
}
|
|
331
|
-
catch { /* logging must not change terminal reconciliation */ }
|
|
332
|
-
terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
|
|
333
543
|
await append(terminal);
|
|
334
544
|
}
|
|
335
545
|
finally {
|
|
@@ -337,19 +547,23 @@ export class ExecutionCoordinator {
|
|
|
337
547
|
if (heartbeatTimer)
|
|
338
548
|
clearInterval(heartbeatTimer);
|
|
339
549
|
lifecycle.current = false;
|
|
550
|
+
if (leaseBoundary)
|
|
551
|
+
clearTimeout(leaseBoundary);
|
|
340
552
|
await fixSession?.close();
|
|
341
553
|
}
|
|
342
|
-
const status
|
|
343
|
-
const diagnostic = status === "failed" ||
|
|
344
|
-
const reportDiagnostic = diagnostic && (
|
|
554
|
+
const { status, reason } = daemonOutcome(terminal);
|
|
555
|
+
const diagnostic = status === "failed" || status === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
|
|
556
|
+
const reportDiagnostic = diagnostic && (status === "blocked"
|
|
345
557
|
? diagnostic.code === "CODEX_COMPLETION_REPORT_BLOCKED" : diagnostic.code !== "CODEX_COMPLETION_REPORT_BLOCKED") ? diagnostic : undefined;
|
|
346
558
|
const securityReceipt = adapter.securityReceipt?.();
|
|
347
559
|
const reads = safeLocalReadObservations(adapter.localReadObservations?.());
|
|
348
|
-
const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
|
|
560
|
+
const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reason ? { reason } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
|
|
349
561
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
350
562
|
phase: "terminal_pending", bindingId: binding.bindingId, terminalEventType: terminal.eventType,
|
|
351
563
|
completionRequest: structuredClone(completionRequest) });
|
|
564
|
+
requireControlFence();
|
|
352
565
|
await this.deps.gateway.complete(completionRequest);
|
|
566
|
+
requireControlFence();
|
|
353
567
|
await this.deps.state.clear();
|
|
354
568
|
}
|
|
355
569
|
}
|
|
@@ -20,7 +20,10 @@ export class JobPoller {
|
|
|
20
20
|
this.failures = 0;
|
|
21
21
|
}
|
|
22
22
|
catch {
|
|
23
|
-
|
|
23
|
+
try {
|
|
24
|
+
this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
|
|
25
|
+
}
|
|
26
|
+
catch { /* Observational logging cannot bypass stop/backoff or discard recovery state. */ }
|
|
24
27
|
// Unknown execution/terminal outcomes must reconcile before any new claim.
|
|
25
28
|
if (stage === "execute") {
|
|
26
29
|
this.stop();
|