mtech-ai-local-runner 0.1.30 → 0.1.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +64 -1
  2. package/dist/src/adapters/claude-code/adapter.js +76 -17
  3. package/dist/src/adapters/codex/adapter.js +146 -30
  4. package/dist/src/adapters/codex/exec-launch-host.js +50 -11
  5. package/dist/src/adapters/codex/stream-parser.js +8 -17
  6. package/dist/src/adapters/completion-report.js +8 -5
  7. package/dist/src/cli.js +23 -0
  8. package/dist/src/config.js +4 -4
  9. package/dist/src/daemon/daemon.js +62 -4
  10. package/dist/src/daemon/execution-coordinator.js +351 -77
  11. package/dist/src/daemon/job-poller.js +4 -1
  12. package/dist/src/daemon/manual-codex.js +10 -3
  13. package/dist/src/daemon/preparation-lease.js +142 -0
  14. package/dist/src/daemon/runtime.js +65 -23
  15. package/dist/src/daemon/terminal-recovery.js +29 -5
  16. package/dist/src/gateway-client.js +120 -22
  17. package/dist/src/platform/macos/launchd.js +13 -2
  18. package/dist/src/process/node-process-group-host.js +89 -34
  19. package/dist/src/process/owned-process-guardian.js +140 -0
  20. package/dist/src/publication/candidate.js +95 -0
  21. package/dist/src/publication/execution-host.js +55 -0
  22. package/dist/src/publication/git-credential-broker.js +150 -0
  23. package/dist/src/publication/git-credential-helper.js +29 -0
  24. package/dist/src/publication/git-snapshot.js +321 -0
  25. package/dist/src/publication/git-transport.js +156 -0
  26. package/dist/src/publication/installed-host.js +120 -0
  27. package/dist/src/publication/isolated-verifier.js +67 -0
  28. package/dist/src/publication/journal.js +87 -0
  29. package/dist/src/publication/production-host.js +155 -0
  30. package/dist/src/publication/project-registration.js +49 -0
  31. package/dist/src/publication/publisher.js +166 -0
  32. package/dist/src/publication/recovery.js +53 -0
  33. package/dist/src/publication/runtime-release.js +91 -0
  34. package/dist/src/publication/server-ci-error.js +15 -0
  35. package/dist/src/publication/server-ci-host.js +160 -0
  36. package/dist/src/runner.js +33 -4
  37. package/dist/src/safety/codex-daemon-host.js +2 -2
  38. package/dist/src/safety/command-policy-core.js +124 -0
  39. package/dist/src/safety/command-policy.js +1 -124
  40. package/dist/src/safety/execution-failure.js +4 -0
  41. package/dist/src/safety/job-tool-policy.js +9 -0
  42. package/dist/src/safety/local-tool-policy.js +1 -1
  43. package/dist/src/safety/terminal-evidence.js +67 -16
  44. package/dist/src/safety/unsupported-supervisor-command.js +12 -0
  45. package/dist/src/state-store.js +50 -7
  46. package/dist/src/workspace/workspace-manager.js +14 -0
  47. package/package.json +3 -2
  48. package/verification/README.md +63 -0
  49. package/verification/runner-policy.cjs +24 -0
@@ -1,9 +1,13 @@
1
+ import { PublicationOutcomeUnknownError } from '../publication/publisher.js';
1
2
  import { executionError, executionFailure } from '../safety/execution-failure.js';
3
+ import { isUnsupportedSupervisorCommand } from '../safety/unsupported-supervisor-command.js';
2
4
  import { requireSecurityRequirements } from '../safety/security-requirements.js';
3
5
  import { GatewayToolError } from "../gateway-client.js";
4
6
  import { createHash } from "node:crypto";
5
7
  import { completionReportDiagnostic } from "../adapters/completion-report.js";
6
8
  import { PROTOCOL_VERSION, isTerminalEventType, validateEvent, MAX_EVENT_SUMMARY_CHARS } from "../protocol.js";
9
+ import { DAEMON_BLOCKED_REASONS, DAEMON_CANCELLED_REASONS } from "../state-store.js";
10
+ import { PreparationDeadlineError, PreparationLease, validateJobLease } from './preparation-lease.js';
7
11
  function canonical(value) {
8
12
  if (Array.isArray(value))
9
13
  return value.map(canonical);
@@ -40,7 +44,18 @@ function immutable(value) {
40
44
  }
41
45
  return value;
42
46
  }
43
- class PreparationDeadlineError extends Error {
47
+ class LocalFixHostRequired extends Error {
48
+ }
49
+ function daemonOutcome(event) {
50
+ const source = event.eventType;
51
+ const suppliedReason = event.payload?.reason;
52
+ const reason = source === 'blocked' ? 'executor_blocked'
53
+ : source === 'question' ? 'executor_question' : source === 'scope_conflict' ? 'scope_conflict'
54
+ : source === 'cancelled' ? (DAEMON_CANCELLED_REASONS.includes(suppliedReason) ? suppliedReason : 'cancelled')
55
+ : source === 'failed' && DAEMON_BLOCKED_REASONS.includes(suppliedReason) ? suppliedReason : undefined;
56
+ const eventType = ['blocked', 'question', 'scope_conflict'].includes(source) ? 'failed' : source;
57
+ return { event: { ...event, eventType, summary: event.summary.slice(0, MAX_EVENT_SUMMARY_CHARS), ...(reason ? { payload: { ...event.payload, reason } } : {}) },
58
+ status: eventType === 'completed' ? 'completed' : eventType === 'cancelled' ? 'cancelled' : reason ? 'blocked' : 'failed', ...(reason ? { reason } : {}) };
44
59
  }
45
60
  export class ExecutionCoordinator {
46
61
  deps;
@@ -53,24 +68,20 @@ export class ExecutionCoordinator {
53
68
  let workspace;
54
69
  let adapter;
55
70
  let binding;
71
+ let fixSession;
72
+ let localFixPublication;
73
+ const isLocalFix = job.taskPackage.local_fix_scope !== undefined;
74
+ const lifecycle = { frozen: false, current: true };
56
75
  const deadline = Date.now() + (this.deps.preparationTimeoutMs ?? 900_000);
57
- const step = async (operation) => {
58
- const remaining = deadline - Date.now();
59
- if (remaining <= 0)
60
- throw new PreparationDeadlineError();
61
- let timer;
62
- try {
63
- return await Promise.race([operation(), new Promise((_, reject) => {
64
- timer = setTimeout(() => reject(new PreparationDeadlineError()), remaining);
65
- })]);
66
- }
67
- finally {
68
- if (timer)
69
- clearTimeout(timer);
70
- }
71
- };
76
+ let leaseExpiresAt = job.leaseExpiresAt;
77
+ const preparation = new PreparationLease(job, deadline, signal => this.deps.gateway.renewJob(job, signal), expiry => { this.deps.gateway.confirmJobLease?.(job, expiry); leaseExpiresAt = expiry; });
78
+ const step = (operation) => preparation.step(operation);
79
+ let bindingAttempted = false;
72
80
  try {
81
+ await preparation.start();
73
82
  const pkg = job.taskPackage;
83
+ if (isLocalFix && (!this.deps.localFixHost || pkg.repository?.access !== "write"))
84
+ throw new LocalFixHostRequired();
74
85
  const repository = pkg.repository;
75
86
  const adapterId = text(pkg.runtime_adapter, "runtime adapter");
76
87
  const adapterVersion = text(pkg.runtime_adapter_version, "runtime adapter version");
@@ -78,29 +89,48 @@ export class ExecutionCoordinator {
78
89
  requireSecurityRequirements(pkg.security_requirements, { adapterId, adapterVersion });
79
90
  if (!/^[a-f0-9]{64}$/.test(job.launchFingerprint ?? '') || job.gatewayContractFingerprint !== expectedContractFingerprint
80
91
  || job.gatewayTaskPackageFingerprint !== fingerprint(pkg))
81
- throw new Error("Claimed launch fingerprints differ");
92
+ throw new GatewayToolError("Claimed launch fingerprints differ", 'local_agent_jobs_wait', 'JOB_LAUNCH_FENCE_INVALID');
82
93
  stage = "workspace";
83
94
  workspace = await step(() => this.deps.workspaces.create({
84
95
  jobId: job.id,
85
96
  repositoryKey: text(repository?.key, "repository key"),
86
97
  providerUrl: text(repository?.provider_url, "repository provider URL"),
87
98
  baseCommitSha: text(repository?.base_commit_sha, "repository SHA"),
99
+ ...(isLocalFix ? { workingBranch: text(repository?.working_branch, "approved fix branch") } : {})
88
100
  }));
89
101
  stage = "adapter";
90
- adapter = await step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }));
91
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest });
102
+ adapter = await preparation.step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }), late => late.cancel());
103
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest }));
92
104
  stage = "shadow";
93
- const gatewayShadow = await step(() => this.deps.gateway.shadow(job));
105
+ const gatewayShadow = await step(() => this.deps.gateway.shadow(job, preparation.signal));
94
106
  stage = "fingerprint";
95
107
  if (contractFingerprint(gatewayShadow.contractPayload) !== expectedContractFingerprint || gatewayShadow.contractFingerprint !== expectedContractFingerprint || gatewayShadow.taskPackageFingerprint !== fingerprint(pkg)) {
96
- throw new Error("contract or task-package fingerprint mismatch before binding");
108
+ throw new GatewayToolError("contract or task-package fingerprint mismatch before binding", 'work_execution_contract_show', 'JOB_SHADOW_FENCE_INVALID');
97
109
  }
98
110
  stage = "binding";
99
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" });
100
- binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }));
111
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" }));
112
+ preparation.beginBinding();
113
+ bindingAttempted = true;
114
+ binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }, preparation.signal));
115
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest }));
116
+ await preparation.stop();
117
+ preparation.assert();
101
118
  }
102
119
  catch (error) {
120
+ if (stage === 'fingerprint')
121
+ preparation.fail(error);
122
+ if (error instanceof GatewayToolError && !['GATEWAY_TRANSPORT_ERROR', 'GATEWAY_HTTP_500', 'GATEWAY_HTTP_502', 'GATEWAY_HTTP_503', 'GATEWAY_HTTP_504'].includes(error.code))
123
+ preparation.fail(error);
124
+ try {
125
+ await preparation.stop();
126
+ }
127
+ catch { /* The original error and latched fence remain authoritative. */ }
128
+ preparation.dispose();
129
+ if (adapter)
130
+ await adapter.cancel();
103
131
  const diagnostic = { stage, code: "PREPARATION_FAILED" };
132
+ if (error instanceof LocalFixHostRequired)
133
+ diagnostic.code = "LOCAL_FIX_HOST_REQUIRED";
104
134
  if (error instanceof PreparationDeadlineError)
105
135
  diagnostic.code = "PREPARATION_DEADLINE_EXCEEDED";
106
136
  if (error instanceof GatewayToolError) {
@@ -108,49 +138,137 @@ export class ExecutionCoordinator {
108
138
  diagnostic.code = error.code;
109
139
  }
110
140
  this.deps.reportPreparationFailure?.(diagnostic);
111
- if (stage === "binding" && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
141
+ if (bindingAttempted && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
112
142
  // The server may have committed the binding. Never fabricate a blocked result
113
143
  // or repeat the mutation before the deterministic binding reference is reconciled.
114
144
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
115
145
  phase: "binding_uncertain", preparationFailure: diagnostic });
116
146
  throw new Error("Binding outcome requires reconciliation");
117
147
  }
148
+ if (!preparation.hasLiveFence || (error instanceof PreparationDeadlineError && stage === 'adapter'))
149
+ throw error;
118
150
  // Retain this exact safe terminal payload until Gateway acknowledges it.
119
151
  const completionRequest = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
120
152
  status: "blocked", result: { preparation_failure: { ...diagnostic } } };
121
153
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
122
154
  phase: "terminal_pending", preparationFailure: diagnostic, completionRequest: structuredClone(completionRequest) });
155
+ if (!preparation.hasLiveFence) {
156
+ preparation.dispose();
157
+ throw error;
158
+ }
123
159
  await this.deps.gateway.complete(completionRequest);
124
160
  await this.deps.state.clear();
161
+ preparation.dispose();
125
162
  return;
126
163
  }
127
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest });
164
+ const executor = adapter;
128
165
  let heartbeatFailure;
166
+ const evidenceAbort = new AbortController();
167
+ const controlAbort = new AbortController();
168
+ let leaseBoundary;
169
+ let rejectFence;
170
+ const lostFence = new Promise((_, reject) => { rejectFence = reject; });
171
+ void lostFence.catch(() => { });
129
172
  let executionStage = 'binding_model';
130
173
  let requestedTerminal;
174
+ let rejectControlStop;
175
+ const controlStopped = new Promise((_, reject) => { rejectControlStop = reject; });
176
+ void controlStopped.catch(() => { });
177
+ let publicationPending = false;
131
178
  let commandCursor = 0;
132
179
  let controlsStopped = false;
133
180
  let controlsInFlight;
181
+ let executorStop;
182
+ let stopUnconfirmed = false;
183
+ let rejectUnconfirmedStop;
184
+ const unconfirmedStop = new Promise((_, reject) => { rejectUnconfirmedStop = reject; });
185
+ void unconfirmedStop.catch(() => { });
186
+ const stopExecutor = () => {
187
+ if (!executorStop) {
188
+ try {
189
+ executorStop = executor.cancel();
190
+ }
191
+ catch (error) {
192
+ executorStop = Promise.reject(error);
193
+ }
194
+ void executorStop.catch(error => { stopUnconfirmed = true; rejectUnconfirmedStop(error); failFence(error); });
195
+ }
196
+ return executorStop;
197
+ };
198
+ const failFence = (error) => {
199
+ if (heartbeatFailure)
200
+ return;
201
+ lifecycle.current = false;
202
+ heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
203
+ controlAbort.abort();
204
+ evidenceAbort.abort();
205
+ rejectFence(heartbeatFailure);
206
+ // Cancellation cannot queue behind transport or generator cleanup.
207
+ void stopExecutor().catch(() => { });
208
+ };
209
+ const requireControlFence = () => {
210
+ if (!heartbeatFailure && (leaseExpiresAt === undefined || Date.now() >= leaseExpiresAt * 1000))
211
+ failFence(new GatewayToolError('Job lease expired', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID'));
212
+ if (heartbeatFailure)
213
+ throw heartbeatFailure;
214
+ };
215
+ const armLeaseBoundary = () => {
216
+ if (leaseBoundary)
217
+ clearTimeout(leaseBoundary);
218
+ leaseBoundary = setTimeout(() => { try {
219
+ requireControlFence();
220
+ }
221
+ catch { /* The latched fence rejects all pending operations. */ } }, Math.max(0, (leaseExpiresAt ?? 0) * 1000 - Date.now()));
222
+ leaseBoundary.unref?.();
223
+ };
224
+ const fenced = (work) => Promise.race([work, lostFence]);
225
+ armLeaseBoundary();
134
226
  const readControls = async () => {
227
+ requireControlFence();
135
228
  if (!this.deps.gateway.readCommands || controlsStopped || requestedTerminal)
136
229
  return;
137
230
  if (controlsInFlight)
138
231
  return controlsInFlight;
139
232
  controlsInFlight = (async () => {
140
- for (const command of await this.deps.gateway.readCommands(binding.bindingId, commandCursor)) {
233
+ const commands = await fenced(this.deps.gateway.readCommands(binding.bindingId, commandCursor, controlAbort.signal));
234
+ requireControlFence();
235
+ for (const command of commands) {
236
+ requireControlFence();
141
237
  if (controlsStopped)
142
238
  return;
239
+ let commandTerminal;
143
240
  if (["cancel", "pause", "request_replan"].includes(command.command)) {
144
- requestedTerminal = { eventType: command.command === "cancel" ? "cancelled" : "blocked", summary: `supervisor ${command.command} acknowledged` };
145
- await adapter.cancel();
241
+ lifecycle.current = false;
242
+ commandTerminal = { eventType: "cancelled", summary: `supervisor ${command.command} acknowledged`,
243
+ payload: { reason: command.command === 'pause' ? 'paused' : command.command === 'request_replan' ? 'replan_requested' : 'cancelled' } };
244
+ await stopExecutor();
146
245
  }
147
246
  else {
148
- await adapter.command(command);
247
+ try {
248
+ await fenced(adapter.command(command));
249
+ }
250
+ catch (error) {
251
+ requireControlFence();
252
+ // Classify the host capability outcome before executionError wraps it.
253
+ if (!isUnsupportedSupervisorCommand(error))
254
+ throw error;
255
+ lifecycle.current = false;
256
+ await stopExecutor();
257
+ requireControlFence();
258
+ commandTerminal = { eventType: 'failed', summary: 'supervisor command is unsupported for the approved launch',
259
+ payload: { reason: 'unsupported_supervisor_command' } };
260
+ }
149
261
  }
262
+ requireControlFence();
150
263
  if (!this.deps.gateway.acknowledgeCommand)
151
264
  throw new Error("daemon command acknowledgement is unavailable");
152
- await this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence);
265
+ await fenced(this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence, controlAbort.signal));
266
+ requireControlFence();
153
267
  commandCursor = command.sequence;
268
+ if (commandTerminal) {
269
+ requestedTerminal = commandTerminal;
270
+ rejectControlStop(new Error('Supervisor stopped execution'));
271
+ }
154
272
  if (requestedTerminal)
155
273
  break;
156
274
  }
@@ -165,30 +283,70 @@ export class ExecutionCoordinator {
165
283
  controlsInFlight = undefined;
166
284
  }
167
285
  };
286
+ let heartbeatInFlight;
168
287
  const heartbeatTimer = this.deps.gateway.heartbeat || this.deps.gateway.readCommands ? setInterval(() => {
169
- void (async () => { await this.deps.gateway.heartbeat?.(binding.bindingId, job); await readControls(); })().catch(async (error) => {
170
- heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
171
- await adapter.cancel().catch(() => undefined);
172
- });
288
+ if (requestedTerminal || heartbeatInFlight)
289
+ return;
290
+ heartbeatInFlight = (async () => {
291
+ await readControls();
292
+ requireControlFence();
293
+ if (!requestedTerminal && this.deps.gateway.heartbeat) {
294
+ const renewed = await fenced(this.deps.gateway.heartbeat(binding.bindingId, job, controlAbort.signal));
295
+ requireControlFence();
296
+ const expiry = validateJobLease(renewed, job, true);
297
+ if (!['running', 'review_pending'].includes(renewed.job.status))
298
+ throw new GatewayToolError('Running Job lease status is invalid', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID');
299
+ this.deps.gateway.confirmJobLease?.(job, expiry);
300
+ leaseExpiresAt = expiry;
301
+ armLeaseBoundary();
302
+ }
303
+ })().catch((error) => {
304
+ failFence(error);
305
+ }).finally(() => { heartbeatInFlight = undefined; });
173
306
  }, 30_000) : undefined;
174
307
  heartbeatTimer?.unref();
175
308
  let terminal;
176
309
  let failureDiagnostic;
177
310
  let sequence = 0;
178
311
  let checkpointSeen = false;
312
+ let eventUncertain = false;
313
+ const stop = async (ignoreControlFailure = false) => {
314
+ controlsStopped = true;
315
+ const cancelled = stopExecutor();
316
+ if (stopUnconfirmed)
317
+ await cancelled;
318
+ let controlFailure;
319
+ try {
320
+ if (controlsInFlight)
321
+ await controlsInFlight;
322
+ }
323
+ catch (error) {
324
+ controlFailure = error;
325
+ }
326
+ if (heartbeatInFlight)
327
+ await heartbeatInFlight;
328
+ await cancelled;
329
+ if (!ignoreControlFailure && (controlFailure || heartbeatFailure))
330
+ throw controlFailure || heartbeatFailure;
331
+ };
179
332
  const append = async (event) => {
180
333
  const previousStage = executionStage;
181
334
  let payload = event.payload;
182
335
  if (isTerminalEventType(event.eventType)) {
183
336
  executionStage = 'terminal_evidence';
184
- const evidence = this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha });
337
+ requireControlFence();
338
+ const evidence = localFixPublication && typeof localFixPublication.diff_hash === 'string' && typeof localFixPublication.changed_file_manifest_hash === 'string' ? { diffHash: localFixPublication.diff_hash, changedFileManifestHash: localFixPublication.changed_file_manifest_hash } : await fenced(this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha, signal: evidenceAbort.signal }));
339
+ requireControlFence();
185
340
  payload = { ...payload, diff_hash: evidence.diffHash, changed_file_manifest_hash: evidence.changedFileManifestHash };
186
341
  }
187
342
  // A vendor report cannot supply host observations, even when its shape looks valid.
188
343
  if (payload) {
189
344
  payload = { ...payload };
190
345
  delete payload.local_read_observations;
346
+ delete payload.local_fix_publication;
191
347
  }
348
+ if (event.eventType === "completed" && localFixPublication)
349
+ payload = { ...payload, local_fix_publication: structuredClone(localFixPublication) };
192
350
  if (isTerminalEventType(event.eventType) || event.eventType === 'blocked' || event.eventType === 'scope_conflict') {
193
351
  const reads = safeLocalReadObservations(adapter.localReadObservations?.());
194
352
  if (reads)
@@ -205,91 +363,207 @@ export class ExecutionCoordinator {
205
363
  validateEvent(normalized);
206
364
  if (!this.deps.gateway.appendEvent)
207
365
  throw new Error("daemon event reporting is unavailable");
208
- await this.deps.gateway.appendEvent(binding.bindingId, normalized);
366
+ // Keep lease renewal responsive during evidence collection, then settle
367
+ // its last fence before the terminal event consumes the execution lease.
368
+ if (isTerminalEventType(event.eventType)) {
369
+ if (heartbeatTimer)
370
+ clearInterval(heartbeatTimer);
371
+ if (heartbeatInFlight)
372
+ await heartbeatInFlight;
373
+ }
374
+ requireControlFence();
375
+ eventUncertain = true;
376
+ await fenced(this.deps.gateway.appendEvent(binding.bindingId, normalized));
377
+ requireControlFence();
209
378
  sequence = normalized.sequence;
210
379
  executionStage = 'state_save';
211
380
  if (event.eventType === "checkpoint_ready")
212
381
  checkpointSeen = true;
213
382
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, eventCursor: sequence, ...(isTerminalEventType(event.eventType) ? { terminalEventType: event.eventType } : {}), workspaceManifest: workspace.manifest });
383
+ eventUncertain = false;
214
384
  executionStage = previousStage;
215
385
  };
386
+ const publicationStep = async (operation) => {
387
+ requireControlFence();
388
+ if (requestedTerminal)
389
+ throw new Error('Supervisor stopped execution');
390
+ const value = await fenced(Promise.race([operation(), controlStopped]));
391
+ requireControlFence();
392
+ if (requestedTerminal)
393
+ throw new Error('Supervisor stopped execution');
394
+ return value;
395
+ };
216
396
  try {
217
397
  if (!job.taskPackage.resolved_model_id
218
398
  || binding.resolvedModelId !== job.taskPackage.resolved_model_id)
219
399
  throw new Error("Binding model differs from approved launch");
220
400
  executionStage = 'adapter_start';
221
- await adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId });
401
+ preparation.assert();
402
+ requireControlFence();
403
+ if (isLocalFix) {
404
+ await readControls();
405
+ if (!requestedTerminal)
406
+ fixSession = await publicationStep(() => this.deps.localFixHost.prepare({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, lifecycle: {
407
+ isFrozen: () => lifecycle.frozen,
408
+ isCurrent: () => lifecycle.current && !requestedTerminal && !heartbeatFailure && leaseExpiresAt !== undefined && Date.now() < leaseExpiresAt * 1000,
409
+ securityReceipt: () => adapter.securityReceipt?.(),
410
+ } }).then(async (session) => {
411
+ if (!lifecycle.current || requestedTerminal || heartbeatFailure) {
412
+ await session.close();
413
+ throw new Error('Local fix preparation lost its lease');
414
+ }
415
+ return session;
416
+ }));
417
+ }
418
+ if (!requestedTerminal) {
419
+ await fenced(Promise.race([adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId }), unconfirmedStop]));
420
+ }
222
421
  executionStage = 'commands';
223
422
  await readControls();
224
423
  executionStage = 'event_stream';
225
424
  if (adapter.events && !requestedTerminal) {
226
- for await (const event of adapter.events()) {
227
- if (heartbeatFailure)
228
- throw heartbeatFailure;
229
- if (requestedTerminal)
230
- break;
231
- if (event.eventType === "completed" && !checkpointSeen) {
232
- await append({ eventType: "checkpoint_ready", summary: "final checkpoint ready" });
233
- }
234
- if (isTerminalEventType(event.eventType)) {
235
- if (controlsInFlight)
236
- await controlsInFlight;
425
+ const iterator = adapter.events()[Symbol.asyncIterator]();
426
+ try {
427
+ while (true) {
428
+ const next = await fenced(Promise.race([iterator.next(), unconfirmedStop]));
429
+ if (next.done)
430
+ break;
431
+ const event = next.value;
432
+ if (heartbeatFailure)
433
+ throw heartbeatFailure;
237
434
  if (requestedTerminal)
238
435
  break;
239
- controlsStopped = true;
436
+ if (isTerminalEventType(event.eventType)) {
437
+ if (event.eventType === "completed" && fixSession) {
438
+ // Stop only this owned model before running verified host publication.
439
+ // Controls/heartbeat remain active throughout the publication phase.
440
+ if (this.deps.localFixHost?.requiresOwnedFreeze) {
441
+ if (!adapter.freezeForPublication)
442
+ throw new Error('LOCAL_FIX_OWNED_FREEZE_UNAVAILABLE');
443
+ await publicationStep(() => adapter.freezeForPublication());
444
+ }
445
+ else
446
+ await publicationStep(() => stopExecutor());
447
+ lifecycle.frozen = true;
448
+ await readControls();
449
+ if (requestedTerminal || heartbeatFailure)
450
+ break;
451
+ executionStage = 'local_fix_publication';
452
+ if (!checkpointSeen)
453
+ await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
454
+ await publicationStep(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } }));
455
+ publicationPending = true;
456
+ localFixPublication = await publicationStep(() => fixSession.finalize());
457
+ if (this.deps.gateway.syncPublicationCursor) {
458
+ try {
459
+ const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence));
460
+ if (!Number.isSafeInteger(cursor) || cursor < sequence)
461
+ throw new Error('Invalid event cursor');
462
+ sequence = cursor;
463
+ }
464
+ catch {
465
+ throw new PublicationOutcomeUnknownError();
466
+ }
467
+ }
468
+ if (controlsInFlight)
469
+ await controlsInFlight;
470
+ requireControlFence();
471
+ if (requestedTerminal)
472
+ throw new PublicationOutcomeUnknownError();
473
+ executionStage = 'event_stream';
474
+ }
475
+ }
476
+ if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict", "question"].includes(event.eventType)) {
477
+ terminal = event;
478
+ void stopExecutor().catch(() => { });
479
+ // Resolve already-issued controls before choosing the terminal receipt,
480
+ // and stop writes before the generator's deferred cleanup or evidence.
481
+ if (controlsInFlight)
482
+ await Promise.race([controlsInFlight, unconfirmedStop]);
483
+ await stop();
484
+ break;
485
+ }
486
+ await append(event);
487
+ await readControls();
240
488
  }
241
- await append(event);
242
- if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict"].includes(event.eventType)) {
243
- controlsStopped = true;
244
- terminal = event;
245
- break;
489
+ }
490
+ finally {
491
+ // A failed owned stop cannot wait behind a blocked generator next().
492
+ // Keep disposal attached, but return to durable suspension promptly.
493
+ if (stopUnconfirmed || heartbeatFailure) {
494
+ try {
495
+ void Promise.resolve(iterator.return?.()).catch(() => { });
496
+ }
497
+ catch { /* Unconfirmed cleanup already fences reporting. */ }
246
498
  }
247
- await readControls();
499
+ else
500
+ await iterator.return?.();
248
501
  }
249
502
  }
250
- if (controlsInFlight)
251
- await controlsInFlight;
252
- if (requestedTerminal) {
503
+ await stop();
504
+ if (publicationPending && requestedTerminal)
505
+ throw new PublicationOutcomeUnknownError();
506
+ if (requestedTerminal)
253
507
  terminal = requestedTerminal;
254
- await append(terminal);
255
- }
256
- if (heartbeatFailure)
257
- throw heartbeatFailure;
258
508
  if (!terminal) {
259
509
  terminal = { eventType: "failed", summary: "adapter ended without a terminal event" };
260
- await append(terminal);
261
510
  }
511
+ terminal = daemonOutcome(terminal).event;
512
+ if (terminal.eventType === 'completed' && !checkpointSeen)
513
+ await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
514
+ await append(terminal);
262
515
  }
263
516
  catch (error) {
264
- await adapter.cancel().catch(() => undefined);
265
- // Do not complete a job whose terminal evidence was not acknowledged.
266
- // A safe bounded reason prevents raw adapter/transport errors leaking.
267
- const diagnostic = executionFailure(error, executionStage);
268
- failureDiagnostic = diagnostic;
269
- // Log before reporting so an unavailable Gateway does not erase the cause.
270
- try {
271
- this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
517
+ await stop(true);
518
+ if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
519
+ // The provider may already have committed/pushed/opened a PR. Do not
520
+ // release this run as failed or let restart begin another model session.
521
+ throw new Error('Local publication outcome requires reconciliation');
522
+ }
523
+ if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
524
+ terminal = daemonOutcome(requestedTerminal).event;
525
+ }
526
+ else {
527
+ // Do not complete a job whose terminal evidence was not acknowledged.
528
+ // A safe bounded reason prevents raw adapter/transport errors leaking.
529
+ const diagnostic = executionFailure(error, executionStage);
530
+ failureDiagnostic = diagnostic;
531
+ // Log before reporting so an unavailable Gateway does not erase the cause.
532
+ try {
533
+ this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
534
+ }
535
+ catch { /* logging must not change terminal reconciliation */ }
536
+ if (heartbeatFailure)
537
+ throw error;
538
+ if (eventUncertain || ['GATEWAY_TOOL_REJECTED', 'GATEWAY_TRANSPORT_ERROR', 'WORK_LOOP_CANCELLED'].includes(diagnostic.code)
539
+ || diagnostic.stage === 'terminal_evidence')
540
+ throw error;
541
+ terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
272
542
  }
273
- catch { /* logging must not change terminal reconciliation */ }
274
- terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
275
543
  await append(terminal);
276
544
  }
277
545
  finally {
278
546
  controlsStopped = true;
279
547
  if (heartbeatTimer)
280
548
  clearInterval(heartbeatTimer);
549
+ lifecycle.current = false;
550
+ if (leaseBoundary)
551
+ clearTimeout(leaseBoundary);
552
+ await fixSession?.close();
281
553
  }
282
- const status = terminal.eventType === "completed" ? "completed" : terminal.eventType === "cancelled" ? "cancelled" : terminal.eventType === "blocked" || terminal.eventType === "scope_conflict" ? "blocked" : "failed";
283
- const diagnostic = status === "failed" || terminal.eventType === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
284
- const reportDiagnostic = diagnostic && (terminal.eventType === "blocked"
554
+ const { status, reason } = daemonOutcome(terminal);
555
+ const diagnostic = status === "failed" || status === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
556
+ const reportDiagnostic = diagnostic && (status === "blocked"
285
557
  ? diagnostic.code === "CODEX_COMPLETION_REPORT_BLOCKED" : diagnostic.code !== "CODEX_COMPLETION_REPORT_BLOCKED") ? diagnostic : undefined;
286
558
  const securityReceipt = adapter.securityReceipt?.();
287
559
  const reads = safeLocalReadObservations(adapter.localReadObservations?.());
288
- const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
560
+ const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reason ? { reason } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
289
561
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
290
562
  phase: "terminal_pending", bindingId: binding.bindingId, terminalEventType: terminal.eventType,
291
563
  completionRequest: structuredClone(completionRequest) });
564
+ requireControlFence();
292
565
  await this.deps.gateway.complete(completionRequest);
566
+ requireControlFence();
293
567
  await this.deps.state.clear();
294
568
  }
295
569
  }
@@ -20,7 +20,10 @@ export class JobPoller {
20
20
  this.failures = 0;
21
21
  }
22
22
  catch {
23
- this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
23
+ try {
24
+ this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
25
+ }
26
+ catch { /* Observational logging cannot bypass stop/backoff or discard recovery state. */ }
24
27
  // Unknown execution/terminal outcomes must reconcile before any new claim.
25
28
  if (stage === "execute") {
26
29
  this.stop();