mtech-ai-local-runner 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -1
- package/dist/src/adapters/claude-code/adapter.js +76 -17
- package/dist/src/adapters/codex/adapter.js +146 -30
- package/dist/src/adapters/codex/exec-launch-host.js +50 -11
- package/dist/src/adapters/codex/stream-parser.js +8 -17
- package/dist/src/adapters/completion-report.js +8 -5
- package/dist/src/cli.js +23 -0
- package/dist/src/config.js +4 -4
- package/dist/src/daemon/daemon.js +62 -4
- package/dist/src/daemon/execution-coordinator.js +351 -77
- package/dist/src/daemon/job-poller.js +4 -1
- package/dist/src/daemon/manual-codex.js +10 -3
- package/dist/src/daemon/preparation-lease.js +142 -0
- package/dist/src/daemon/runtime.js +65 -23
- package/dist/src/daemon/terminal-recovery.js +29 -5
- package/dist/src/gateway-client.js +120 -22
- package/dist/src/platform/macos/launchd.js +13 -2
- package/dist/src/process/node-process-group-host.js +89 -34
- package/dist/src/process/owned-process-guardian.js +140 -0
- package/dist/src/publication/candidate.js +95 -0
- package/dist/src/publication/execution-host.js +55 -0
- package/dist/src/publication/git-credential-broker.js +150 -0
- package/dist/src/publication/git-credential-helper.js +29 -0
- package/dist/src/publication/git-snapshot.js +321 -0
- package/dist/src/publication/git-transport.js +156 -0
- package/dist/src/publication/installed-host.js +120 -0
- package/dist/src/publication/isolated-verifier.js +67 -0
- package/dist/src/publication/journal.js +87 -0
- package/dist/src/publication/production-host.js +155 -0
- package/dist/src/publication/project-registration.js +49 -0
- package/dist/src/publication/publisher.js +166 -0
- package/dist/src/publication/recovery.js +53 -0
- package/dist/src/publication/runtime-release.js +91 -0
- package/dist/src/publication/server-ci-error.js +15 -0
- package/dist/src/publication/server-ci-host.js +160 -0
- package/dist/src/runner.js +33 -4
- package/dist/src/safety/codex-daemon-host.js +2 -2
- package/dist/src/safety/command-policy-core.js +124 -0
- package/dist/src/safety/command-policy.js +1 -124
- package/dist/src/safety/execution-failure.js +4 -0
- package/dist/src/safety/job-tool-policy.js +9 -0
- package/dist/src/safety/local-tool-policy.js +1 -1
- package/dist/src/safety/terminal-evidence.js +67 -16
- package/dist/src/safety/unsupported-supervisor-command.js +12 -0
- package/dist/src/state-store.js +50 -7
- package/dist/src/workspace/workspace-manager.js +14 -0
- package/package.json +3 -2
- package/verification/README.md +63 -0
- package/verification/runner-policy.cjs +24 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import { PublicationOutcomeUnknownError } from '../publication/publisher.js';
|
|
1
2
|
import { executionError, executionFailure } from '../safety/execution-failure.js';
|
|
3
|
+
import { isUnsupportedSupervisorCommand } from '../safety/unsupported-supervisor-command.js';
|
|
2
4
|
import { requireSecurityRequirements } from '../safety/security-requirements.js';
|
|
3
5
|
import { GatewayToolError } from "../gateway-client.js";
|
|
4
6
|
import { createHash } from "node:crypto";
|
|
5
7
|
import { completionReportDiagnostic } from "../adapters/completion-report.js";
|
|
6
8
|
import { PROTOCOL_VERSION, isTerminalEventType, validateEvent, MAX_EVENT_SUMMARY_CHARS } from "../protocol.js";
|
|
9
|
+
import { DAEMON_BLOCKED_REASONS, DAEMON_CANCELLED_REASONS } from "../state-store.js";
|
|
10
|
+
import { PreparationDeadlineError, PreparationLease, validateJobLease } from './preparation-lease.js';
|
|
7
11
|
function canonical(value) {
|
|
8
12
|
if (Array.isArray(value))
|
|
9
13
|
return value.map(canonical);
|
|
@@ -40,7 +44,18 @@ function immutable(value) {
|
|
|
40
44
|
}
|
|
41
45
|
return value;
|
|
42
46
|
}
|
|
43
|
-
class
|
|
47
|
+
class LocalFixHostRequired extends Error {
|
|
48
|
+
}
|
|
49
|
+
function daemonOutcome(event) {
|
|
50
|
+
const source = event.eventType;
|
|
51
|
+
const suppliedReason = event.payload?.reason;
|
|
52
|
+
const reason = source === 'blocked' ? 'executor_blocked'
|
|
53
|
+
: source === 'question' ? 'executor_question' : source === 'scope_conflict' ? 'scope_conflict'
|
|
54
|
+
: source === 'cancelled' ? (DAEMON_CANCELLED_REASONS.includes(suppliedReason) ? suppliedReason : 'cancelled')
|
|
55
|
+
: source === 'failed' && DAEMON_BLOCKED_REASONS.includes(suppliedReason) ? suppliedReason : undefined;
|
|
56
|
+
const eventType = ['blocked', 'question', 'scope_conflict'].includes(source) ? 'failed' : source;
|
|
57
|
+
return { event: { ...event, eventType, summary: event.summary.slice(0, MAX_EVENT_SUMMARY_CHARS), ...(reason ? { payload: { ...event.payload, reason } } : {}) },
|
|
58
|
+
status: eventType === 'completed' ? 'completed' : eventType === 'cancelled' ? 'cancelled' : reason ? 'blocked' : 'failed', ...(reason ? { reason } : {}) };
|
|
44
59
|
}
|
|
45
60
|
export class ExecutionCoordinator {
|
|
46
61
|
deps;
|
|
@@ -53,24 +68,20 @@ export class ExecutionCoordinator {
|
|
|
53
68
|
let workspace;
|
|
54
69
|
let adapter;
|
|
55
70
|
let binding;
|
|
71
|
+
let fixSession;
|
|
72
|
+
let localFixPublication;
|
|
73
|
+
const isLocalFix = job.taskPackage.local_fix_scope !== undefined;
|
|
74
|
+
const lifecycle = { frozen: false, current: true };
|
|
56
75
|
const deadline = Date.now() + (this.deps.preparationTimeoutMs ?? 900_000);
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
let timer;
|
|
62
|
-
try {
|
|
63
|
-
return await Promise.race([operation(), new Promise((_, reject) => {
|
|
64
|
-
timer = setTimeout(() => reject(new PreparationDeadlineError()), remaining);
|
|
65
|
-
})]);
|
|
66
|
-
}
|
|
67
|
-
finally {
|
|
68
|
-
if (timer)
|
|
69
|
-
clearTimeout(timer);
|
|
70
|
-
}
|
|
71
|
-
};
|
|
76
|
+
let leaseExpiresAt = job.leaseExpiresAt;
|
|
77
|
+
const preparation = new PreparationLease(job, deadline, signal => this.deps.gateway.renewJob(job, signal), expiry => { this.deps.gateway.confirmJobLease?.(job, expiry); leaseExpiresAt = expiry; });
|
|
78
|
+
const step = (operation) => preparation.step(operation);
|
|
79
|
+
let bindingAttempted = false;
|
|
72
80
|
try {
|
|
81
|
+
await preparation.start();
|
|
73
82
|
const pkg = job.taskPackage;
|
|
83
|
+
if (isLocalFix && (!this.deps.localFixHost || pkg.repository?.access !== "write"))
|
|
84
|
+
throw new LocalFixHostRequired();
|
|
74
85
|
const repository = pkg.repository;
|
|
75
86
|
const adapterId = text(pkg.runtime_adapter, "runtime adapter");
|
|
76
87
|
const adapterVersion = text(pkg.runtime_adapter_version, "runtime adapter version");
|
|
@@ -78,29 +89,48 @@ export class ExecutionCoordinator {
|
|
|
78
89
|
requireSecurityRequirements(pkg.security_requirements, { adapterId, adapterVersion });
|
|
79
90
|
if (!/^[a-f0-9]{64}$/.test(job.launchFingerprint ?? '') || job.gatewayContractFingerprint !== expectedContractFingerprint
|
|
80
91
|
|| job.gatewayTaskPackageFingerprint !== fingerprint(pkg))
|
|
81
|
-
throw new
|
|
92
|
+
throw new GatewayToolError("Claimed launch fingerprints differ", 'local_agent_jobs_wait', 'JOB_LAUNCH_FENCE_INVALID');
|
|
82
93
|
stage = "workspace";
|
|
83
94
|
workspace = await step(() => this.deps.workspaces.create({
|
|
84
95
|
jobId: job.id,
|
|
85
96
|
repositoryKey: text(repository?.key, "repository key"),
|
|
86
97
|
providerUrl: text(repository?.provider_url, "repository provider URL"),
|
|
87
98
|
baseCommitSha: text(repository?.base_commit_sha, "repository SHA"),
|
|
99
|
+
...(isLocalFix ? { workingBranch: text(repository?.working_branch, "approved fix branch") } : {})
|
|
88
100
|
}));
|
|
89
101
|
stage = "adapter";
|
|
90
|
-
adapter = await step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }));
|
|
91
|
-
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest });
|
|
102
|
+
adapter = await preparation.step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }), late => late.cancel());
|
|
103
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest }));
|
|
92
104
|
stage = "shadow";
|
|
93
|
-
const gatewayShadow = await step(() => this.deps.gateway.shadow(job));
|
|
105
|
+
const gatewayShadow = await step(() => this.deps.gateway.shadow(job, preparation.signal));
|
|
94
106
|
stage = "fingerprint";
|
|
95
107
|
if (contractFingerprint(gatewayShadow.contractPayload) !== expectedContractFingerprint || gatewayShadow.contractFingerprint !== expectedContractFingerprint || gatewayShadow.taskPackageFingerprint !== fingerprint(pkg)) {
|
|
96
|
-
throw new
|
|
108
|
+
throw new GatewayToolError("contract or task-package fingerprint mismatch before binding", 'work_execution_contract_show', 'JOB_SHADOW_FENCE_INVALID');
|
|
97
109
|
}
|
|
98
110
|
stage = "binding";
|
|
99
|
-
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" });
|
|
100
|
-
|
|
111
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" }));
|
|
112
|
+
preparation.beginBinding();
|
|
113
|
+
bindingAttempted = true;
|
|
114
|
+
binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }, preparation.signal));
|
|
115
|
+
await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest }));
|
|
116
|
+
await preparation.stop();
|
|
117
|
+
preparation.assert();
|
|
101
118
|
}
|
|
102
119
|
catch (error) {
|
|
120
|
+
if (stage === 'fingerprint')
|
|
121
|
+
preparation.fail(error);
|
|
122
|
+
if (error instanceof GatewayToolError && !['GATEWAY_TRANSPORT_ERROR', 'GATEWAY_HTTP_500', 'GATEWAY_HTTP_502', 'GATEWAY_HTTP_503', 'GATEWAY_HTTP_504'].includes(error.code))
|
|
123
|
+
preparation.fail(error);
|
|
124
|
+
try {
|
|
125
|
+
await preparation.stop();
|
|
126
|
+
}
|
|
127
|
+
catch { /* The original error and latched fence remain authoritative. */ }
|
|
128
|
+
preparation.dispose();
|
|
129
|
+
if (adapter)
|
|
130
|
+
await adapter.cancel();
|
|
103
131
|
const diagnostic = { stage, code: "PREPARATION_FAILED" };
|
|
132
|
+
if (error instanceof LocalFixHostRequired)
|
|
133
|
+
diagnostic.code = "LOCAL_FIX_HOST_REQUIRED";
|
|
104
134
|
if (error instanceof PreparationDeadlineError)
|
|
105
135
|
diagnostic.code = "PREPARATION_DEADLINE_EXCEEDED";
|
|
106
136
|
if (error instanceof GatewayToolError) {
|
|
@@ -108,49 +138,137 @@ export class ExecutionCoordinator {
|
|
|
108
138
|
diagnostic.code = error.code;
|
|
109
139
|
}
|
|
110
140
|
this.deps.reportPreparationFailure?.(diagnostic);
|
|
111
|
-
if (
|
|
141
|
+
if (bindingAttempted && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
|
|
112
142
|
// The server may have committed the binding. Never fabricate a blocked result
|
|
113
143
|
// or repeat the mutation before the deterministic binding reference is reconciled.
|
|
114
144
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
115
145
|
phase: "binding_uncertain", preparationFailure: diagnostic });
|
|
116
146
|
throw new Error("Binding outcome requires reconciliation");
|
|
117
147
|
}
|
|
148
|
+
if (!preparation.hasLiveFence || (error instanceof PreparationDeadlineError && stage === 'adapter'))
|
|
149
|
+
throw error;
|
|
118
150
|
// Retain this exact safe terminal payload until Gateway acknowledges it.
|
|
119
151
|
const completionRequest = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
120
152
|
status: "blocked", result: { preparation_failure: { ...diagnostic } } };
|
|
121
153
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
122
154
|
phase: "terminal_pending", preparationFailure: diagnostic, completionRequest: structuredClone(completionRequest) });
|
|
155
|
+
if (!preparation.hasLiveFence) {
|
|
156
|
+
preparation.dispose();
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
123
159
|
await this.deps.gateway.complete(completionRequest);
|
|
124
160
|
await this.deps.state.clear();
|
|
161
|
+
preparation.dispose();
|
|
125
162
|
return;
|
|
126
163
|
}
|
|
127
|
-
|
|
164
|
+
const executor = adapter;
|
|
128
165
|
let heartbeatFailure;
|
|
166
|
+
const evidenceAbort = new AbortController();
|
|
167
|
+
const controlAbort = new AbortController();
|
|
168
|
+
let leaseBoundary;
|
|
169
|
+
let rejectFence;
|
|
170
|
+
const lostFence = new Promise((_, reject) => { rejectFence = reject; });
|
|
171
|
+
void lostFence.catch(() => { });
|
|
129
172
|
let executionStage = 'binding_model';
|
|
130
173
|
let requestedTerminal;
|
|
174
|
+
let rejectControlStop;
|
|
175
|
+
const controlStopped = new Promise((_, reject) => { rejectControlStop = reject; });
|
|
176
|
+
void controlStopped.catch(() => { });
|
|
177
|
+
let publicationPending = false;
|
|
131
178
|
let commandCursor = 0;
|
|
132
179
|
let controlsStopped = false;
|
|
133
180
|
let controlsInFlight;
|
|
181
|
+
let executorStop;
|
|
182
|
+
let stopUnconfirmed = false;
|
|
183
|
+
let rejectUnconfirmedStop;
|
|
184
|
+
const unconfirmedStop = new Promise((_, reject) => { rejectUnconfirmedStop = reject; });
|
|
185
|
+
void unconfirmedStop.catch(() => { });
|
|
186
|
+
const stopExecutor = () => {
|
|
187
|
+
if (!executorStop) {
|
|
188
|
+
try {
|
|
189
|
+
executorStop = executor.cancel();
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
executorStop = Promise.reject(error);
|
|
193
|
+
}
|
|
194
|
+
void executorStop.catch(error => { stopUnconfirmed = true; rejectUnconfirmedStop(error); failFence(error); });
|
|
195
|
+
}
|
|
196
|
+
return executorStop;
|
|
197
|
+
};
|
|
198
|
+
const failFence = (error) => {
|
|
199
|
+
if (heartbeatFailure)
|
|
200
|
+
return;
|
|
201
|
+
lifecycle.current = false;
|
|
202
|
+
heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
|
|
203
|
+
controlAbort.abort();
|
|
204
|
+
evidenceAbort.abort();
|
|
205
|
+
rejectFence(heartbeatFailure);
|
|
206
|
+
// Cancellation cannot queue behind transport or generator cleanup.
|
|
207
|
+
void stopExecutor().catch(() => { });
|
|
208
|
+
};
|
|
209
|
+
const requireControlFence = () => {
|
|
210
|
+
if (!heartbeatFailure && (leaseExpiresAt === undefined || Date.now() >= leaseExpiresAt * 1000))
|
|
211
|
+
failFence(new GatewayToolError('Job lease expired', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID'));
|
|
212
|
+
if (heartbeatFailure)
|
|
213
|
+
throw heartbeatFailure;
|
|
214
|
+
};
|
|
215
|
+
const armLeaseBoundary = () => {
|
|
216
|
+
if (leaseBoundary)
|
|
217
|
+
clearTimeout(leaseBoundary);
|
|
218
|
+
leaseBoundary = setTimeout(() => { try {
|
|
219
|
+
requireControlFence();
|
|
220
|
+
}
|
|
221
|
+
catch { /* The latched fence rejects all pending operations. */ } }, Math.max(0, (leaseExpiresAt ?? 0) * 1000 - Date.now()));
|
|
222
|
+
leaseBoundary.unref?.();
|
|
223
|
+
};
|
|
224
|
+
const fenced = (work) => Promise.race([work, lostFence]);
|
|
225
|
+
armLeaseBoundary();
|
|
134
226
|
const readControls = async () => {
|
|
227
|
+
requireControlFence();
|
|
135
228
|
if (!this.deps.gateway.readCommands || controlsStopped || requestedTerminal)
|
|
136
229
|
return;
|
|
137
230
|
if (controlsInFlight)
|
|
138
231
|
return controlsInFlight;
|
|
139
232
|
controlsInFlight = (async () => {
|
|
140
|
-
|
|
233
|
+
const commands = await fenced(this.deps.gateway.readCommands(binding.bindingId, commandCursor, controlAbort.signal));
|
|
234
|
+
requireControlFence();
|
|
235
|
+
for (const command of commands) {
|
|
236
|
+
requireControlFence();
|
|
141
237
|
if (controlsStopped)
|
|
142
238
|
return;
|
|
239
|
+
let commandTerminal;
|
|
143
240
|
if (["cancel", "pause", "request_replan"].includes(command.command)) {
|
|
144
|
-
|
|
145
|
-
|
|
241
|
+
lifecycle.current = false;
|
|
242
|
+
commandTerminal = { eventType: "cancelled", summary: `supervisor ${command.command} acknowledged`,
|
|
243
|
+
payload: { reason: command.command === 'pause' ? 'paused' : command.command === 'request_replan' ? 'replan_requested' : 'cancelled' } };
|
|
244
|
+
await stopExecutor();
|
|
146
245
|
}
|
|
147
246
|
else {
|
|
148
|
-
|
|
247
|
+
try {
|
|
248
|
+
await fenced(adapter.command(command));
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
requireControlFence();
|
|
252
|
+
// Classify the host capability outcome before executionError wraps it.
|
|
253
|
+
if (!isUnsupportedSupervisorCommand(error))
|
|
254
|
+
throw error;
|
|
255
|
+
lifecycle.current = false;
|
|
256
|
+
await stopExecutor();
|
|
257
|
+
requireControlFence();
|
|
258
|
+
commandTerminal = { eventType: 'failed', summary: 'supervisor command is unsupported for the approved launch',
|
|
259
|
+
payload: { reason: 'unsupported_supervisor_command' } };
|
|
260
|
+
}
|
|
149
261
|
}
|
|
262
|
+
requireControlFence();
|
|
150
263
|
if (!this.deps.gateway.acknowledgeCommand)
|
|
151
264
|
throw new Error("daemon command acknowledgement is unavailable");
|
|
152
|
-
await this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence);
|
|
265
|
+
await fenced(this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence, controlAbort.signal));
|
|
266
|
+
requireControlFence();
|
|
153
267
|
commandCursor = command.sequence;
|
|
268
|
+
if (commandTerminal) {
|
|
269
|
+
requestedTerminal = commandTerminal;
|
|
270
|
+
rejectControlStop(new Error('Supervisor stopped execution'));
|
|
271
|
+
}
|
|
154
272
|
if (requestedTerminal)
|
|
155
273
|
break;
|
|
156
274
|
}
|
|
@@ -165,30 +283,70 @@ export class ExecutionCoordinator {
|
|
|
165
283
|
controlsInFlight = undefined;
|
|
166
284
|
}
|
|
167
285
|
};
|
|
286
|
+
let heartbeatInFlight;
|
|
168
287
|
const heartbeatTimer = this.deps.gateway.heartbeat || this.deps.gateway.readCommands ? setInterval(() => {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
288
|
+
if (requestedTerminal || heartbeatInFlight)
|
|
289
|
+
return;
|
|
290
|
+
heartbeatInFlight = (async () => {
|
|
291
|
+
await readControls();
|
|
292
|
+
requireControlFence();
|
|
293
|
+
if (!requestedTerminal && this.deps.gateway.heartbeat) {
|
|
294
|
+
const renewed = await fenced(this.deps.gateway.heartbeat(binding.bindingId, job, controlAbort.signal));
|
|
295
|
+
requireControlFence();
|
|
296
|
+
const expiry = validateJobLease(renewed, job, true);
|
|
297
|
+
if (!['running', 'review_pending'].includes(renewed.job.status))
|
|
298
|
+
throw new GatewayToolError('Running Job lease status is invalid', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID');
|
|
299
|
+
this.deps.gateway.confirmJobLease?.(job, expiry);
|
|
300
|
+
leaseExpiresAt = expiry;
|
|
301
|
+
armLeaseBoundary();
|
|
302
|
+
}
|
|
303
|
+
})().catch((error) => {
|
|
304
|
+
failFence(error);
|
|
305
|
+
}).finally(() => { heartbeatInFlight = undefined; });
|
|
173
306
|
}, 30_000) : undefined;
|
|
174
307
|
heartbeatTimer?.unref();
|
|
175
308
|
let terminal;
|
|
176
309
|
let failureDiagnostic;
|
|
177
310
|
let sequence = 0;
|
|
178
311
|
let checkpointSeen = false;
|
|
312
|
+
let eventUncertain = false;
|
|
313
|
+
const stop = async (ignoreControlFailure = false) => {
|
|
314
|
+
controlsStopped = true;
|
|
315
|
+
const cancelled = stopExecutor();
|
|
316
|
+
if (stopUnconfirmed)
|
|
317
|
+
await cancelled;
|
|
318
|
+
let controlFailure;
|
|
319
|
+
try {
|
|
320
|
+
if (controlsInFlight)
|
|
321
|
+
await controlsInFlight;
|
|
322
|
+
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
controlFailure = error;
|
|
325
|
+
}
|
|
326
|
+
if (heartbeatInFlight)
|
|
327
|
+
await heartbeatInFlight;
|
|
328
|
+
await cancelled;
|
|
329
|
+
if (!ignoreControlFailure && (controlFailure || heartbeatFailure))
|
|
330
|
+
throw controlFailure || heartbeatFailure;
|
|
331
|
+
};
|
|
179
332
|
const append = async (event) => {
|
|
180
333
|
const previousStage = executionStage;
|
|
181
334
|
let payload = event.payload;
|
|
182
335
|
if (isTerminalEventType(event.eventType)) {
|
|
183
336
|
executionStage = 'terminal_evidence';
|
|
184
|
-
|
|
337
|
+
requireControlFence();
|
|
338
|
+
const evidence = localFixPublication && typeof localFixPublication.diff_hash === 'string' && typeof localFixPublication.changed_file_manifest_hash === 'string' ? { diffHash: localFixPublication.diff_hash, changedFileManifestHash: localFixPublication.changed_file_manifest_hash } : await fenced(this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha, signal: evidenceAbort.signal }));
|
|
339
|
+
requireControlFence();
|
|
185
340
|
payload = { ...payload, diff_hash: evidence.diffHash, changed_file_manifest_hash: evidence.changedFileManifestHash };
|
|
186
341
|
}
|
|
187
342
|
// A vendor report cannot supply host observations, even when its shape looks valid.
|
|
188
343
|
if (payload) {
|
|
189
344
|
payload = { ...payload };
|
|
190
345
|
delete payload.local_read_observations;
|
|
346
|
+
delete payload.local_fix_publication;
|
|
191
347
|
}
|
|
348
|
+
if (event.eventType === "completed" && localFixPublication)
|
|
349
|
+
payload = { ...payload, local_fix_publication: structuredClone(localFixPublication) };
|
|
192
350
|
if (isTerminalEventType(event.eventType) || event.eventType === 'blocked' || event.eventType === 'scope_conflict') {
|
|
193
351
|
const reads = safeLocalReadObservations(adapter.localReadObservations?.());
|
|
194
352
|
if (reads)
|
|
@@ -205,91 +363,207 @@ export class ExecutionCoordinator {
|
|
|
205
363
|
validateEvent(normalized);
|
|
206
364
|
if (!this.deps.gateway.appendEvent)
|
|
207
365
|
throw new Error("daemon event reporting is unavailable");
|
|
208
|
-
|
|
366
|
+
// Keep lease renewal responsive during evidence collection, then settle
|
|
367
|
+
// its last fence before the terminal event consumes the execution lease.
|
|
368
|
+
if (isTerminalEventType(event.eventType)) {
|
|
369
|
+
if (heartbeatTimer)
|
|
370
|
+
clearInterval(heartbeatTimer);
|
|
371
|
+
if (heartbeatInFlight)
|
|
372
|
+
await heartbeatInFlight;
|
|
373
|
+
}
|
|
374
|
+
requireControlFence();
|
|
375
|
+
eventUncertain = true;
|
|
376
|
+
await fenced(this.deps.gateway.appendEvent(binding.bindingId, normalized));
|
|
377
|
+
requireControlFence();
|
|
209
378
|
sequence = normalized.sequence;
|
|
210
379
|
executionStage = 'state_save';
|
|
211
380
|
if (event.eventType === "checkpoint_ready")
|
|
212
381
|
checkpointSeen = true;
|
|
213
382
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, eventCursor: sequence, ...(isTerminalEventType(event.eventType) ? { terminalEventType: event.eventType } : {}), workspaceManifest: workspace.manifest });
|
|
383
|
+
eventUncertain = false;
|
|
214
384
|
executionStage = previousStage;
|
|
215
385
|
};
|
|
386
|
+
const publicationStep = async (operation) => {
|
|
387
|
+
requireControlFence();
|
|
388
|
+
if (requestedTerminal)
|
|
389
|
+
throw new Error('Supervisor stopped execution');
|
|
390
|
+
const value = await fenced(Promise.race([operation(), controlStopped]));
|
|
391
|
+
requireControlFence();
|
|
392
|
+
if (requestedTerminal)
|
|
393
|
+
throw new Error('Supervisor stopped execution');
|
|
394
|
+
return value;
|
|
395
|
+
};
|
|
216
396
|
try {
|
|
217
397
|
if (!job.taskPackage.resolved_model_id
|
|
218
398
|
|| binding.resolvedModelId !== job.taskPackage.resolved_model_id)
|
|
219
399
|
throw new Error("Binding model differs from approved launch");
|
|
220
400
|
executionStage = 'adapter_start';
|
|
221
|
-
|
|
401
|
+
preparation.assert();
|
|
402
|
+
requireControlFence();
|
|
403
|
+
if (isLocalFix) {
|
|
404
|
+
await readControls();
|
|
405
|
+
if (!requestedTerminal)
|
|
406
|
+
fixSession = await publicationStep(() => this.deps.localFixHost.prepare({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, lifecycle: {
|
|
407
|
+
isFrozen: () => lifecycle.frozen,
|
|
408
|
+
isCurrent: () => lifecycle.current && !requestedTerminal && !heartbeatFailure && leaseExpiresAt !== undefined && Date.now() < leaseExpiresAt * 1000,
|
|
409
|
+
securityReceipt: () => adapter.securityReceipt?.(),
|
|
410
|
+
} }).then(async (session) => {
|
|
411
|
+
if (!lifecycle.current || requestedTerminal || heartbeatFailure) {
|
|
412
|
+
await session.close();
|
|
413
|
+
throw new Error('Local fix preparation lost its lease');
|
|
414
|
+
}
|
|
415
|
+
return session;
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
418
|
+
if (!requestedTerminal) {
|
|
419
|
+
await fenced(Promise.race([adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId }), unconfirmedStop]));
|
|
420
|
+
}
|
|
222
421
|
executionStage = 'commands';
|
|
223
422
|
await readControls();
|
|
224
423
|
executionStage = 'event_stream';
|
|
225
424
|
if (adapter.events && !requestedTerminal) {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
if (controlsInFlight)
|
|
236
|
-
await controlsInFlight;
|
|
425
|
+
const iterator = adapter.events()[Symbol.asyncIterator]();
|
|
426
|
+
try {
|
|
427
|
+
while (true) {
|
|
428
|
+
const next = await fenced(Promise.race([iterator.next(), unconfirmedStop]));
|
|
429
|
+
if (next.done)
|
|
430
|
+
break;
|
|
431
|
+
const event = next.value;
|
|
432
|
+
if (heartbeatFailure)
|
|
433
|
+
throw heartbeatFailure;
|
|
237
434
|
if (requestedTerminal)
|
|
238
435
|
break;
|
|
239
|
-
|
|
436
|
+
if (isTerminalEventType(event.eventType)) {
|
|
437
|
+
if (event.eventType === "completed" && fixSession) {
|
|
438
|
+
// Stop only this owned model before running verified host publication.
|
|
439
|
+
// Controls/heartbeat remain active throughout the publication phase.
|
|
440
|
+
if (this.deps.localFixHost?.requiresOwnedFreeze) {
|
|
441
|
+
if (!adapter.freezeForPublication)
|
|
442
|
+
throw new Error('LOCAL_FIX_OWNED_FREEZE_UNAVAILABLE');
|
|
443
|
+
await publicationStep(() => adapter.freezeForPublication());
|
|
444
|
+
}
|
|
445
|
+
else
|
|
446
|
+
await publicationStep(() => stopExecutor());
|
|
447
|
+
lifecycle.frozen = true;
|
|
448
|
+
await readControls();
|
|
449
|
+
if (requestedTerminal || heartbeatFailure)
|
|
450
|
+
break;
|
|
451
|
+
executionStage = 'local_fix_publication';
|
|
452
|
+
if (!checkpointSeen)
|
|
453
|
+
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
454
|
+
await publicationStep(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } }));
|
|
455
|
+
publicationPending = true;
|
|
456
|
+
localFixPublication = await publicationStep(() => fixSession.finalize());
|
|
457
|
+
if (this.deps.gateway.syncPublicationCursor) {
|
|
458
|
+
try {
|
|
459
|
+
const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence));
|
|
460
|
+
if (!Number.isSafeInteger(cursor) || cursor < sequence)
|
|
461
|
+
throw new Error('Invalid event cursor');
|
|
462
|
+
sequence = cursor;
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
throw new PublicationOutcomeUnknownError();
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
if (controlsInFlight)
|
|
469
|
+
await controlsInFlight;
|
|
470
|
+
requireControlFence();
|
|
471
|
+
if (requestedTerminal)
|
|
472
|
+
throw new PublicationOutcomeUnknownError();
|
|
473
|
+
executionStage = 'event_stream';
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict", "question"].includes(event.eventType)) {
|
|
477
|
+
terminal = event;
|
|
478
|
+
void stopExecutor().catch(() => { });
|
|
479
|
+
// Resolve already-issued controls before choosing the terminal receipt,
|
|
480
|
+
// and stop writes before the generator's deferred cleanup or evidence.
|
|
481
|
+
if (controlsInFlight)
|
|
482
|
+
await Promise.race([controlsInFlight, unconfirmedStop]);
|
|
483
|
+
await stop();
|
|
484
|
+
break;
|
|
485
|
+
}
|
|
486
|
+
await append(event);
|
|
487
|
+
await readControls();
|
|
240
488
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
489
|
+
}
|
|
490
|
+
finally {
|
|
491
|
+
// A failed owned stop cannot wait behind a blocked generator next().
|
|
492
|
+
// Keep disposal attached, but return to durable suspension promptly.
|
|
493
|
+
if (stopUnconfirmed || heartbeatFailure) {
|
|
494
|
+
try {
|
|
495
|
+
void Promise.resolve(iterator.return?.()).catch(() => { });
|
|
496
|
+
}
|
|
497
|
+
catch { /* Unconfirmed cleanup already fences reporting. */ }
|
|
246
498
|
}
|
|
247
|
-
|
|
499
|
+
else
|
|
500
|
+
await iterator.return?.();
|
|
248
501
|
}
|
|
249
502
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
503
|
+
await stop();
|
|
504
|
+
if (publicationPending && requestedTerminal)
|
|
505
|
+
throw new PublicationOutcomeUnknownError();
|
|
506
|
+
if (requestedTerminal)
|
|
253
507
|
terminal = requestedTerminal;
|
|
254
|
-
await append(terminal);
|
|
255
|
-
}
|
|
256
|
-
if (heartbeatFailure)
|
|
257
|
-
throw heartbeatFailure;
|
|
258
508
|
if (!terminal) {
|
|
259
509
|
terminal = { eventType: "failed", summary: "adapter ended without a terminal event" };
|
|
260
|
-
await append(terminal);
|
|
261
510
|
}
|
|
511
|
+
terminal = daemonOutcome(terminal).event;
|
|
512
|
+
if (terminal.eventType === 'completed' && !checkpointSeen)
|
|
513
|
+
await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
|
|
514
|
+
await append(terminal);
|
|
262
515
|
}
|
|
263
516
|
catch (error) {
|
|
264
|
-
await
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
517
|
+
await stop(true);
|
|
518
|
+
if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
|
|
519
|
+
// The provider may already have committed/pushed/opened a PR. Do not
|
|
520
|
+
// release this run as failed or let restart begin another model session.
|
|
521
|
+
throw new Error('Local publication outcome requires reconciliation');
|
|
522
|
+
}
|
|
523
|
+
if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
|
|
524
|
+
terminal = daemonOutcome(requestedTerminal).event;
|
|
525
|
+
}
|
|
526
|
+
else {
|
|
527
|
+
// Do not complete a job whose terminal evidence was not acknowledged.
|
|
528
|
+
// A safe bounded reason prevents raw adapter/transport errors leaking.
|
|
529
|
+
const diagnostic = executionFailure(error, executionStage);
|
|
530
|
+
failureDiagnostic = diagnostic;
|
|
531
|
+
// Log before reporting so an unavailable Gateway does not erase the cause.
|
|
532
|
+
try {
|
|
533
|
+
this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
|
|
534
|
+
}
|
|
535
|
+
catch { /* logging must not change terminal reconciliation */ }
|
|
536
|
+
if (heartbeatFailure)
|
|
537
|
+
throw error;
|
|
538
|
+
if (eventUncertain || ['GATEWAY_TOOL_REJECTED', 'GATEWAY_TRANSPORT_ERROR', 'WORK_LOOP_CANCELLED'].includes(diagnostic.code)
|
|
539
|
+
|| diagnostic.stage === 'terminal_evidence')
|
|
540
|
+
throw error;
|
|
541
|
+
terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
|
|
272
542
|
}
|
|
273
|
-
catch { /* logging must not change terminal reconciliation */ }
|
|
274
|
-
terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
|
|
275
543
|
await append(terminal);
|
|
276
544
|
}
|
|
277
545
|
finally {
|
|
278
546
|
controlsStopped = true;
|
|
279
547
|
if (heartbeatTimer)
|
|
280
548
|
clearInterval(heartbeatTimer);
|
|
549
|
+
lifecycle.current = false;
|
|
550
|
+
if (leaseBoundary)
|
|
551
|
+
clearTimeout(leaseBoundary);
|
|
552
|
+
await fixSession?.close();
|
|
281
553
|
}
|
|
282
|
-
const status
|
|
283
|
-
const diagnostic = status === "failed" ||
|
|
284
|
-
const reportDiagnostic = diagnostic && (
|
|
554
|
+
const { status, reason } = daemonOutcome(terminal);
|
|
555
|
+
const diagnostic = status === "failed" || status === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
|
|
556
|
+
const reportDiagnostic = diagnostic && (status === "blocked"
|
|
285
557
|
? diagnostic.code === "CODEX_COMPLETION_REPORT_BLOCKED" : diagnostic.code !== "CODEX_COMPLETION_REPORT_BLOCKED") ? diagnostic : undefined;
|
|
286
558
|
const securityReceipt = adapter.securityReceipt?.();
|
|
287
559
|
const reads = safeLocalReadObservations(adapter.localReadObservations?.());
|
|
288
|
-
const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
|
|
560
|
+
const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reason ? { reason } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
|
|
289
561
|
await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
|
|
290
562
|
phase: "terminal_pending", bindingId: binding.bindingId, terminalEventType: terminal.eventType,
|
|
291
563
|
completionRequest: structuredClone(completionRequest) });
|
|
564
|
+
requireControlFence();
|
|
292
565
|
await this.deps.gateway.complete(completionRequest);
|
|
566
|
+
requireControlFence();
|
|
293
567
|
await this.deps.state.clear();
|
|
294
568
|
}
|
|
295
569
|
}
|
|
@@ -20,7 +20,10 @@ export class JobPoller {
|
|
|
20
20
|
this.failures = 0;
|
|
21
21
|
}
|
|
22
22
|
catch {
|
|
23
|
-
|
|
23
|
+
try {
|
|
24
|
+
this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
|
|
25
|
+
}
|
|
26
|
+
catch { /* Observational logging cannot bypass stop/backoff or discard recovery state. */ }
|
|
24
27
|
// Unknown execution/terminal outcomes must reconcile before any new claim.
|
|
25
28
|
if (stage === "execute") {
|
|
26
29
|
this.stop();
|