mtech-ai-local-runner 0.1.31 → 0.1.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,13 @@
1
1
  import { PublicationOutcomeUnknownError } from '../publication/publisher.js';
2
2
  import { executionError, executionFailure } from '../safety/execution-failure.js';
3
+ import { isUnsupportedSupervisorCommand } from '../safety/unsupported-supervisor-command.js';
3
4
  import { requireSecurityRequirements } from '../safety/security-requirements.js';
4
5
  import { GatewayToolError } from "../gateway-client.js";
5
6
  import { createHash } from "node:crypto";
6
7
  import { completionReportDiagnostic } from "../adapters/completion-report.js";
7
8
  import { PROTOCOL_VERSION, isTerminalEventType, validateEvent, MAX_EVENT_SUMMARY_CHARS } from "../protocol.js";
9
+ import { DAEMON_BLOCKED_REASONS, DAEMON_CANCELLED_REASONS } from "../state-store.js";
10
+ import { PreparationDeadlineError, PreparationLease, validateJobLease } from './preparation-lease.js';
8
11
  function canonical(value) {
9
12
  if (Array.isArray(value))
10
13
  return value.map(canonical);
@@ -41,10 +44,19 @@ function immutable(value) {
41
44
  }
42
45
  return value;
43
46
  }
44
- class PreparationDeadlineError extends Error {
45
- }
46
47
  class LocalFixHostRequired extends Error {
47
48
  }
49
+ function daemonOutcome(event) {
50
+ const source = event.eventType;
51
+ const suppliedReason = event.payload?.reason;
52
+ const reason = source === 'blocked' ? 'executor_blocked'
53
+ : source === 'question' ? 'executor_question' : source === 'scope_conflict' ? 'scope_conflict'
54
+ : source === 'cancelled' ? (DAEMON_CANCELLED_REASONS.includes(suppliedReason) ? suppliedReason : 'cancelled')
55
+ : source === 'failed' && DAEMON_BLOCKED_REASONS.includes(suppliedReason) ? suppliedReason : undefined;
56
+ const eventType = ['blocked', 'question', 'scope_conflict'].includes(source) ? 'failed' : source;
57
+ return { event: { ...event, eventType, summary: event.summary.slice(0, MAX_EVENT_SUMMARY_CHARS), ...(reason ? { payload: { ...event.payload, reason } } : {}) },
58
+ status: eventType === 'completed' ? 'completed' : eventType === 'cancelled' ? 'cancelled' : reason ? 'blocked' : 'failed', ...(reason ? { reason } : {}) };
59
+ }
48
60
  export class ExecutionCoordinator {
49
61
  deps;
50
62
  constructor(deps) {
@@ -61,22 +73,12 @@ export class ExecutionCoordinator {
61
73
  const isLocalFix = job.taskPackage.local_fix_scope !== undefined;
62
74
  const lifecycle = { frozen: false, current: true };
63
75
  const deadline = Date.now() + (this.deps.preparationTimeoutMs ?? 900_000);
64
- const step = async (operation) => {
65
- const remaining = deadline - Date.now();
66
- if (remaining <= 0)
67
- throw new PreparationDeadlineError();
68
- let timer;
69
- try {
70
- return await Promise.race([operation(), new Promise((_, reject) => {
71
- timer = setTimeout(() => reject(new PreparationDeadlineError()), remaining);
72
- })]);
73
- }
74
- finally {
75
- if (timer)
76
- clearTimeout(timer);
77
- }
78
- };
76
+ let leaseExpiresAt = job.leaseExpiresAt;
77
+ const preparation = new PreparationLease(job, deadline, signal => this.deps.gateway.renewJob(job, signal), expiry => { this.deps.gateway.confirmJobLease?.(job, expiry); leaseExpiresAt = expiry; });
78
+ const step = (operation) => preparation.step(operation);
79
+ let bindingAttempted = false;
79
80
  try {
81
+ await preparation.start();
80
82
  const pkg = job.taskPackage;
81
83
  if (isLocalFix && (!this.deps.localFixHost || pkg.repository?.access !== "write"))
82
84
  throw new LocalFixHostRequired();
@@ -87,7 +89,7 @@ export class ExecutionCoordinator {
87
89
  requireSecurityRequirements(pkg.security_requirements, { adapterId, adapterVersion });
88
90
  if (!/^[a-f0-9]{64}$/.test(job.launchFingerprint ?? '') || job.gatewayContractFingerprint !== expectedContractFingerprint
89
91
  || job.gatewayTaskPackageFingerprint !== fingerprint(pkg))
90
- throw new Error("Claimed launch fingerprints differ");
92
+ throw new GatewayToolError("Claimed launch fingerprints differ", 'local_agent_jobs_wait', 'JOB_LAUNCH_FENCE_INVALID');
91
93
  stage = "workspace";
92
94
  workspace = await step(() => this.deps.workspaces.create({
93
95
  jobId: job.id,
@@ -97,21 +99,35 @@ export class ExecutionCoordinator {
97
99
  ...(isLocalFix ? { workingBranch: text(repository?.working_branch, "approved fix branch") } : {})
98
100
  }));
99
101
  stage = "adapter";
100
- adapter = await step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }));
101
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest });
102
+ adapter = await preparation.step(() => this.deps.registry.select(adapterId, adapterVersion, { worktreePath: workspace.path, jobId: job.id }), late => late.cancel());
103
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "shadow", workspaceManifest: workspace.manifest }));
102
104
  stage = "shadow";
103
- const gatewayShadow = await step(() => this.deps.gateway.shadow(job));
105
+ const gatewayShadow = await step(() => this.deps.gateway.shadow(job, preparation.signal));
104
106
  stage = "fingerprint";
105
107
  if (contractFingerprint(gatewayShadow.contractPayload) !== expectedContractFingerprint || gatewayShadow.contractFingerprint !== expectedContractFingerprint || gatewayShadow.taskPackageFingerprint !== fingerprint(pkg)) {
106
- throw new Error("contract or task-package fingerprint mismatch before binding");
108
+ throw new GatewayToolError("contract or task-package fingerprint mismatch before binding", 'work_execution_contract_show', 'JOB_SHADOW_FENCE_INVALID');
107
109
  }
108
110
  stage = "binding";
109
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" });
110
- binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }));
111
- if (isLocalFix)
112
- fixSession = await step(() => this.deps.localFixHost.prepare({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, lifecycle: { isFrozen: () => lifecycle.frozen, isCurrent: () => lifecycle.current, securityReceipt: () => adapter.securityReceipt?.() } }));
111
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "binding" }));
112
+ preparation.beginBinding();
113
+ bindingAttempted = true;
114
+ binding = await step(() => this.deps.gateway.createBinding({ ...job, checkoutPath: workspace.path }, preparation.signal));
115
+ await step(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest }));
116
+ await preparation.stop();
117
+ preparation.assert();
113
118
  }
114
119
  catch (error) {
120
+ if (stage === 'fingerprint')
121
+ preparation.fail(error);
122
+ if (error instanceof GatewayToolError && !['GATEWAY_TRANSPORT_ERROR', 'GATEWAY_HTTP_500', 'GATEWAY_HTTP_502', 'GATEWAY_HTTP_503', 'GATEWAY_HTTP_504'].includes(error.code))
123
+ preparation.fail(error);
124
+ try {
125
+ await preparation.stop();
126
+ }
127
+ catch { /* The original error and latched fence remain authoritative. */ }
128
+ preparation.dispose();
129
+ if (adapter)
130
+ await adapter.cancel();
115
131
  const diagnostic = { stage, code: "PREPARATION_FAILED" };
116
132
  if (error instanceof LocalFixHostRequired)
117
133
  diagnostic.code = "LOCAL_FIX_HOST_REQUIRED";
@@ -122,50 +138,137 @@ export class ExecutionCoordinator {
122
138
  diagnostic.code = error.code;
123
139
  }
124
140
  this.deps.reportPreparationFailure?.(diagnostic);
125
- if (stage === "binding" && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
141
+ if (bindingAttempted && !(error instanceof GatewayToolError && error.code === "GATEWAY_TOOL_REJECTED")) {
126
142
  // The server may have committed the binding. Never fabricate a blocked result
127
143
  // or repeat the mutation before the deterministic binding reference is reconciled.
128
144
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
129
145
  phase: "binding_uncertain", preparationFailure: diagnostic });
130
146
  throw new Error("Binding outcome requires reconciliation");
131
147
  }
148
+ if (!preparation.hasLiveFence || (error instanceof PreparationDeadlineError && stage === 'adapter'))
149
+ throw error;
132
150
  // Retain this exact safe terminal payload until Gateway acknowledges it.
133
151
  const completionRequest = { jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
134
152
  status: "blocked", result: { preparation_failure: { ...diagnostic } } };
135
153
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
136
154
  phase: "terminal_pending", preparationFailure: diagnostic, completionRequest: structuredClone(completionRequest) });
155
+ if (!preparation.hasLiveFence) {
156
+ preparation.dispose();
157
+ throw error;
158
+ }
137
159
  await this.deps.gateway.complete(completionRequest);
138
160
  await this.deps.state.clear();
161
+ preparation.dispose();
139
162
  return;
140
163
  }
141
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, workspaceManifest: workspace.manifest });
164
+ const executor = adapter;
142
165
  let heartbeatFailure;
166
+ const evidenceAbort = new AbortController();
167
+ const controlAbort = new AbortController();
168
+ let leaseBoundary;
169
+ let rejectFence;
170
+ const lostFence = new Promise((_, reject) => { rejectFence = reject; });
171
+ void lostFence.catch(() => { });
143
172
  let executionStage = 'binding_model';
144
173
  let requestedTerminal;
174
+ let rejectControlStop;
175
+ const controlStopped = new Promise((_, reject) => { rejectControlStop = reject; });
176
+ void controlStopped.catch(() => { });
177
+ let publicationPending = false;
145
178
  let commandCursor = 0;
146
179
  let controlsStopped = false;
147
180
  let controlsInFlight;
181
+ let executorStop;
182
+ let stopUnconfirmed = false;
183
+ let rejectUnconfirmedStop;
184
+ const unconfirmedStop = new Promise((_, reject) => { rejectUnconfirmedStop = reject; });
185
+ void unconfirmedStop.catch(() => { });
186
+ const stopExecutor = () => {
187
+ if (!executorStop) {
188
+ try {
189
+ executorStop = executor.cancel();
190
+ }
191
+ catch (error) {
192
+ executorStop = Promise.reject(error);
193
+ }
194
+ void executorStop.catch(error => { stopUnconfirmed = true; rejectUnconfirmedStop(error); failFence(error); });
195
+ }
196
+ return executorStop;
197
+ };
198
+ const failFence = (error) => {
199
+ if (heartbeatFailure)
200
+ return;
201
+ lifecycle.current = false;
202
+ heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
203
+ controlAbort.abort();
204
+ evidenceAbort.abort();
205
+ rejectFence(heartbeatFailure);
206
+ // Cancellation cannot queue behind transport or generator cleanup.
207
+ void stopExecutor().catch(() => { });
208
+ };
209
+ const requireControlFence = () => {
210
+ if (!heartbeatFailure && (leaseExpiresAt === undefined || Date.now() >= leaseExpiresAt * 1000))
211
+ failFence(new GatewayToolError('Job lease expired', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID'));
212
+ if (heartbeatFailure)
213
+ throw heartbeatFailure;
214
+ };
215
+ const armLeaseBoundary = () => {
216
+ if (leaseBoundary)
217
+ clearTimeout(leaseBoundary);
218
+ leaseBoundary = setTimeout(() => { try {
219
+ requireControlFence();
220
+ }
221
+ catch { /* The latched fence rejects all pending operations. */ } }, Math.max(0, (leaseExpiresAt ?? 0) * 1000 - Date.now()));
222
+ leaseBoundary.unref?.();
223
+ };
224
+ const fenced = (work) => Promise.race([work, lostFence]);
225
+ armLeaseBoundary();
148
226
  const readControls = async () => {
227
+ requireControlFence();
149
228
  if (!this.deps.gateway.readCommands || controlsStopped || requestedTerminal)
150
229
  return;
151
230
  if (controlsInFlight)
152
231
  return controlsInFlight;
153
232
  controlsInFlight = (async () => {
154
- for (const command of await this.deps.gateway.readCommands(binding.bindingId, commandCursor)) {
233
+ const commands = await fenced(this.deps.gateway.readCommands(binding.bindingId, commandCursor, controlAbort.signal));
234
+ requireControlFence();
235
+ for (const command of commands) {
236
+ requireControlFence();
155
237
  if (controlsStopped)
156
238
  return;
239
+ let commandTerminal;
157
240
  if (["cancel", "pause", "request_replan"].includes(command.command)) {
158
241
  lifecycle.current = false;
159
- requestedTerminal = { eventType: command.command === "cancel" ? "cancelled" : "blocked", summary: `supervisor ${command.command} acknowledged` };
160
- await adapter.cancel();
242
+ commandTerminal = { eventType: "cancelled", summary: `supervisor ${command.command} acknowledged`,
243
+ payload: { reason: command.command === 'pause' ? 'paused' : command.command === 'request_replan' ? 'replan_requested' : 'cancelled' } };
244
+ await stopExecutor();
161
245
  }
162
246
  else {
163
- await adapter.command(command);
247
+ try {
248
+ await fenced(adapter.command(command));
249
+ }
250
+ catch (error) {
251
+ requireControlFence();
252
+ // Classify the host capability outcome before executionError wraps it.
253
+ if (!isUnsupportedSupervisorCommand(error))
254
+ throw error;
255
+ lifecycle.current = false;
256
+ await stopExecutor();
257
+ requireControlFence();
258
+ commandTerminal = { eventType: 'failed', summary: 'supervisor command is unsupported for the approved launch',
259
+ payload: { reason: 'unsupported_supervisor_command' } };
260
+ }
164
261
  }
262
+ requireControlFence();
165
263
  if (!this.deps.gateway.acknowledgeCommand)
166
264
  throw new Error("daemon command acknowledgement is unavailable");
167
- await this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence);
265
+ await fenced(this.deps.gateway.acknowledgeCommand(binding.bindingId, command.sequence, controlAbort.signal));
266
+ requireControlFence();
168
267
  commandCursor = command.sequence;
268
+ if (commandTerminal) {
269
+ requestedTerminal = commandTerminal;
270
+ rejectControlStop(new Error('Supervisor stopped execution'));
271
+ }
169
272
  if (requestedTerminal)
170
273
  break;
171
274
  }
@@ -180,24 +283,60 @@ export class ExecutionCoordinator {
180
283
  controlsInFlight = undefined;
181
284
  }
182
285
  };
286
+ let heartbeatInFlight;
183
287
  const heartbeatTimer = this.deps.gateway.heartbeat || this.deps.gateway.readCommands ? setInterval(() => {
184
- void (async () => { await this.deps.gateway.heartbeat?.(binding.bindingId, job); await readControls(); })().catch(async (error) => {
185
- lifecycle.current = false;
186
- heartbeatFailure = executionError(error, 'heartbeat', 'EXECUTION_FAILED');
187
- await adapter.cancel().catch(() => undefined);
188
- });
288
+ if (requestedTerminal || heartbeatInFlight)
289
+ return;
290
+ heartbeatInFlight = (async () => {
291
+ await readControls();
292
+ requireControlFence();
293
+ if (!requestedTerminal && this.deps.gateway.heartbeat) {
294
+ const renewed = await fenced(this.deps.gateway.heartbeat(binding.bindingId, job, controlAbort.signal));
295
+ requireControlFence();
296
+ const expiry = validateJobLease(renewed, job, true);
297
+ if (!['running', 'review_pending'].includes(renewed.job.status))
298
+ throw new GatewayToolError('Running Job lease status is invalid', 'local_agent_job_heartbeat', 'JOB_LEASE_FENCE_INVALID');
299
+ this.deps.gateway.confirmJobLease?.(job, expiry);
300
+ leaseExpiresAt = expiry;
301
+ armLeaseBoundary();
302
+ }
303
+ })().catch((error) => {
304
+ failFence(error);
305
+ }).finally(() => { heartbeatInFlight = undefined; });
189
306
  }, 30_000) : undefined;
190
307
  heartbeatTimer?.unref();
191
308
  let terminal;
192
309
  let failureDiagnostic;
193
310
  let sequence = 0;
194
311
  let checkpointSeen = false;
312
+ let eventUncertain = false;
313
+ const stop = async (ignoreControlFailure = false) => {
314
+ controlsStopped = true;
315
+ const cancelled = stopExecutor();
316
+ if (stopUnconfirmed)
317
+ await cancelled;
318
+ let controlFailure;
319
+ try {
320
+ if (controlsInFlight)
321
+ await controlsInFlight;
322
+ }
323
+ catch (error) {
324
+ controlFailure = error;
325
+ }
326
+ if (heartbeatInFlight)
327
+ await heartbeatInFlight;
328
+ await cancelled;
329
+ if (!ignoreControlFailure && (controlFailure || heartbeatFailure))
330
+ throw controlFailure || heartbeatFailure;
331
+ };
195
332
  const append = async (event) => {
196
333
  const previousStage = executionStage;
197
334
  let payload = event.payload;
198
335
  if (isTerminalEventType(event.eventType)) {
199
336
  executionStage = 'terminal_evidence';
200
- const evidence = localFixPublication && typeof localFixPublication.diff_hash === 'string' && typeof localFixPublication.changed_file_manifest_hash === 'string' ? { diffHash: localFixPublication.diff_hash, changedFileManifestHash: localFixPublication.changed_file_manifest_hash } : this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha });
337
+ requireControlFence();
338
+ const evidence = localFixPublication && typeof localFixPublication.diff_hash === 'string' && typeof localFixPublication.changed_file_manifest_hash === 'string' ? { diffHash: localFixPublication.diff_hash, changedFileManifestHash: localFixPublication.changed_file_manifest_hash } : await fenced(this.deps.terminalEvidenceCollector.collect({ checkoutPath: workspace.path, baseSha: workspace.baseCommitSha, signal: evidenceAbort.signal }));
339
+ requireControlFence();
201
340
  payload = { ...payload, diff_hash: evidence.diffHash, changed_file_manifest_hash: evidence.changedFileManifestHash };
202
341
  }
203
342
  // A vendor report cannot supply host observations, even when its shape looks valid.
@@ -224,112 +363,183 @@ export class ExecutionCoordinator {
224
363
  validateEvent(normalized);
225
364
  if (!this.deps.gateway.appendEvent)
226
365
  throw new Error("daemon event reporting is unavailable");
227
- await this.deps.gateway.appendEvent(binding.bindingId, normalized);
366
+ // Keep lease renewal responsive during evidence collection, then settle
367
+ // its last fence before the terminal event consumes the execution lease.
368
+ if (isTerminalEventType(event.eventType)) {
369
+ if (heartbeatTimer)
370
+ clearInterval(heartbeatTimer);
371
+ if (heartbeatInFlight)
372
+ await heartbeatInFlight;
373
+ }
374
+ requireControlFence();
375
+ eventUncertain = true;
376
+ await fenced(this.deps.gateway.appendEvent(binding.bindingId, normalized));
377
+ requireControlFence();
228
378
  sequence = normalized.sequence;
229
379
  executionStage = 'state_save';
230
380
  if (event.eventType === "checkpoint_ready")
231
381
  checkpointSeen = true;
232
382
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: "running", bindingId: binding.bindingId, eventCursor: sequence, ...(isTerminalEventType(event.eventType) ? { terminalEventType: event.eventType } : {}), workspaceManifest: workspace.manifest });
383
+ eventUncertain = false;
233
384
  executionStage = previousStage;
234
385
  };
386
+ const publicationStep = async (operation) => {
387
+ requireControlFence();
388
+ if (requestedTerminal)
389
+ throw new Error('Supervisor stopped execution');
390
+ const value = await fenced(Promise.race([operation(), controlStopped]));
391
+ requireControlFence();
392
+ if (requestedTerminal)
393
+ throw new Error('Supervisor stopped execution');
394
+ return value;
395
+ };
235
396
  try {
236
397
  if (!job.taskPackage.resolved_model_id
237
398
  || binding.resolvedModelId !== job.taskPackage.resolved_model_id)
238
399
  throw new Error("Binding model differs from approved launch");
239
400
  executionStage = 'adapter_start';
240
- await adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId });
401
+ preparation.assert();
402
+ requireControlFence();
403
+ if (isLocalFix) {
404
+ await readControls();
405
+ if (!requestedTerminal)
406
+ fixSession = await publicationStep(() => this.deps.localFixHost.prepare({ job, checkoutPath: workspace.path, bindingId: binding.bindingId, lifecycle: {
407
+ isFrozen: () => lifecycle.frozen,
408
+ isCurrent: () => lifecycle.current && !requestedTerminal && !heartbeatFailure && leaseExpiresAt !== undefined && Date.now() < leaseExpiresAt * 1000,
409
+ securityReceipt: () => adapter.securityReceipt?.(),
410
+ } }).then(async (session) => {
411
+ if (!lifecycle.current || requestedTerminal || heartbeatFailure) {
412
+ await session.close();
413
+ throw new Error('Local fix preparation lost its lease');
414
+ }
415
+ return session;
416
+ }));
417
+ }
418
+ if (!requestedTerminal) {
419
+ await fenced(Promise.race([adapter.start({ securityBinding: { launchFingerprint: job.launchFingerprint, contractFingerprint: job.gatewayContractFingerprint, taskPackageFingerprint: job.gatewayTaskPackageFingerprint }, generation: job.generation, approvedTaskPackage: job.taskPackage, bindingId: binding.bindingId, sessionId: binding.bindingId, taskPackage: binding.taskPackage, resolvedModelId: binding.resolvedModelId }), unconfirmedStop]));
420
+ }
241
421
  executionStage = 'commands';
242
422
  await readControls();
243
423
  executionStage = 'event_stream';
244
424
  if (adapter.events && !requestedTerminal) {
245
- for await (const event of adapter.events()) {
246
- if (heartbeatFailure)
247
- throw heartbeatFailure;
248
- if (requestedTerminal)
249
- break;
250
- if (event.eventType === "completed" && !checkpointSeen) {
251
- await append({ eventType: "checkpoint_ready", summary: "final checkpoint ready" });
252
- }
253
- if (isTerminalEventType(event.eventType)) {
254
- if (controlsInFlight)
255
- await controlsInFlight;
425
+ const iterator = adapter.events()[Symbol.asyncIterator]();
426
+ try {
427
+ while (true) {
428
+ const next = await fenced(Promise.race([iterator.next(), unconfirmedStop]));
429
+ if (next.done)
430
+ break;
431
+ const event = next.value;
432
+ if (heartbeatFailure)
433
+ throw heartbeatFailure;
256
434
  if (requestedTerminal)
257
435
  break;
258
- if (event.eventType === "completed" && fixSession) {
259
- // Stop only this owned model before running verified host publication.
260
- // Controls/heartbeat remain active throughout the publication phase.
261
- if (this.deps.localFixHost?.requiresOwnedFreeze) {
262
- if (!adapter.freezeForPublication)
263
- throw new Error('LOCAL_FIX_OWNED_FREEZE_UNAVAILABLE');
264
- await adapter.freezeForPublication();
265
- }
266
- else
267
- await adapter.cancel();
268
- lifecycle.frozen = true;
269
- await readControls();
270
- if (requestedTerminal || heartbeatFailure)
271
- break;
272
- executionStage = 'local_fix_publication';
273
- await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } });
274
- localFixPublication = await fixSession.finalize();
275
- if (this.deps.gateway.syncPublicationCursor) {
276
- try {
277
- const cursor = await this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence);
278
- if (!Number.isSafeInteger(cursor) || cursor < sequence)
279
- throw new Error('Invalid event cursor');
280
- sequence = cursor;
436
+ if (isTerminalEventType(event.eventType)) {
437
+ if (event.eventType === "completed" && fixSession) {
438
+ // Stop only this owned model before running verified host publication.
439
+ // Controls/heartbeat remain active throughout the publication phase.
440
+ if (this.deps.localFixHost?.requiresOwnedFreeze) {
441
+ if (!adapter.freezeForPublication)
442
+ throw new Error('LOCAL_FIX_OWNED_FREEZE_UNAVAILABLE');
443
+ await publicationStep(() => adapter.freezeForPublication());
281
444
  }
282
- catch {
283
- throw new PublicationOutcomeUnknownError();
445
+ else
446
+ await publicationStep(() => stopExecutor());
447
+ lifecycle.frozen = true;
448
+ await readControls();
449
+ if (requestedTerminal || heartbeatFailure)
450
+ break;
451
+ executionStage = 'local_fix_publication';
452
+ if (!checkpointSeen)
453
+ await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
454
+ await publicationStep(() => this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, phase: 'publication_pending', bindingId: binding.bindingId, eventCursor: sequence, workspaceManifest: workspace.manifest, publicationRecovery: { version: 1, job, checkoutPath: workspace.path, bindingId: binding.bindingId, completedEvent: event, gatewayContext: this.deps.gateway.publicationRecoveryContext?.(binding.bindingId), securityReceipt: adapter.securityReceipt?.(), freezeConfirmed: lifecycle.frozen } }));
455
+ publicationPending = true;
456
+ localFixPublication = await publicationStep(() => fixSession.finalize());
457
+ if (this.deps.gateway.syncPublicationCursor) {
458
+ try {
459
+ const cursor = await publicationStep(() => this.deps.gateway.syncPublicationCursor(binding.bindingId, sequence));
460
+ if (!Number.isSafeInteger(cursor) || cursor < sequence)
461
+ throw new Error('Invalid event cursor');
462
+ sequence = cursor;
463
+ }
464
+ catch {
465
+ throw new PublicationOutcomeUnknownError();
466
+ }
284
467
  }
468
+ if (controlsInFlight)
469
+ await controlsInFlight;
470
+ requireControlFence();
471
+ if (requestedTerminal)
472
+ throw new PublicationOutcomeUnknownError();
473
+ executionStage = 'event_stream';
285
474
  }
475
+ }
476
+ if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict", "question"].includes(event.eventType)) {
477
+ terminal = event;
478
+ void stopExecutor().catch(() => { });
479
+ // Resolve already-issued controls before choosing the terminal receipt,
480
+ // and stop writes before the generator's deferred cleanup or evidence.
286
481
  if (controlsInFlight)
287
- await controlsInFlight;
288
- if (requestedTerminal || heartbeatFailure)
289
- break;
290
- executionStage = 'event_stream';
482
+ await Promise.race([controlsInFlight, unconfirmedStop]);
483
+ await stop();
484
+ break;
291
485
  }
292
- controlsStopped = true;
486
+ await append(event);
487
+ await readControls();
293
488
  }
294
- await append(event);
295
- if (isTerminalEventType(event.eventType) || ["blocked", "scope_conflict"].includes(event.eventType)) {
296
- controlsStopped = true;
297
- terminal = event;
298
- break;
489
+ }
490
+ finally {
491
+ // A failed owned stop cannot wait behind a blocked generator next().
492
+ // Keep disposal attached, but return to durable suspension promptly.
493
+ if (stopUnconfirmed || heartbeatFailure) {
494
+ try {
495
+ void Promise.resolve(iterator.return?.()).catch(() => { });
496
+ }
497
+ catch { /* Unconfirmed cleanup already fences reporting. */ }
299
498
  }
300
- await readControls();
499
+ else
500
+ await iterator.return?.();
301
501
  }
302
502
  }
303
- if (controlsInFlight)
304
- await controlsInFlight;
305
- if (requestedTerminal) {
503
+ await stop();
504
+ if (publicationPending && requestedTerminal)
505
+ throw new PublicationOutcomeUnknownError();
506
+ if (requestedTerminal)
306
507
  terminal = requestedTerminal;
307
- await append(terminal);
308
- }
309
- if (heartbeatFailure)
310
- throw heartbeatFailure;
311
508
  if (!terminal) {
312
509
  terminal = { eventType: "failed", summary: "adapter ended without a terminal event" };
313
- await append(terminal);
314
510
  }
511
+ terminal = daemonOutcome(terminal).event;
512
+ if (terminal.eventType === 'completed' && !checkpointSeen)
513
+ await append({ eventType: 'checkpoint_ready', summary: 'final checkpoint ready' });
514
+ await append(terminal);
315
515
  }
316
516
  catch (error) {
317
- if (error instanceof PublicationOutcomeUnknownError) {
517
+ await stop(true);
518
+ if (error instanceof PublicationOutcomeUnknownError || (publicationPending && (heartbeatFailure || requestedTerminal))) {
318
519
  // The provider may already have committed/pushed/opened a PR. Do not
319
520
  // release this run as failed or let restart begin another model session.
320
521
  throw new Error('Local publication outcome requires reconciliation');
321
522
  }
322
- await adapter.cancel().catch(() => undefined);
323
- // Do not complete a job whose terminal evidence was not acknowledged.
324
- // A safe bounded reason prevents raw adapter/transport errors leaking.
325
- const diagnostic = executionFailure(error, executionStage);
326
- failureDiagnostic = diagnostic;
327
- // Log before reporting so an unavailable Gateway does not erase the cause.
328
- try {
329
- this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
523
+ if (requestedTerminal && !heartbeatFailure && !eventUncertain) {
524
+ terminal = daemonOutcome(requestedTerminal).event;
525
+ }
526
+ else {
527
+ // Do not complete a job whose terminal evidence was not acknowledged.
528
+ // A safe bounded reason prevents raw adapter/transport errors leaking.
529
+ const diagnostic = executionFailure(error, executionStage);
530
+ failureDiagnostic = diagnostic;
531
+ // Log before reporting so an unavailable Gateway does not erase the cause.
532
+ try {
533
+ this.deps.reportExecutionFailure?.(structuredClone(diagnostic));
534
+ }
535
+ catch { /* logging must not change terminal reconciliation */ }
536
+ if (heartbeatFailure)
537
+ throw error;
538
+ if (eventUncertain || ['GATEWAY_TOOL_REJECTED', 'GATEWAY_TRANSPORT_ERROR', 'WORK_LOOP_CANCELLED'].includes(diagnostic.code)
539
+ || diagnostic.stage === 'terminal_evidence')
540
+ throw error;
541
+ terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
330
542
  }
331
- catch { /* logging must not change terminal reconciliation */ }
332
- terminal = { eventType: "failed", summary: "daemon execution or event reporting failed", payload: { execution_failure: { ...diagnostic } } };
333
543
  await append(terminal);
334
544
  }
335
545
  finally {
@@ -337,19 +547,23 @@ export class ExecutionCoordinator {
337
547
  if (heartbeatTimer)
338
548
  clearInterval(heartbeatTimer);
339
549
  lifecycle.current = false;
550
+ if (leaseBoundary)
551
+ clearTimeout(leaseBoundary);
340
552
  await fixSession?.close();
341
553
  }
342
- const status = terminal.eventType === "completed" ? "completed" : terminal.eventType === "cancelled" ? "cancelled" : terminal.eventType === "blocked" || terminal.eventType === "scope_conflict" ? "blocked" : "failed";
343
- const diagnostic = status === "failed" || terminal.eventType === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
344
- const reportDiagnostic = diagnostic && (terminal.eventType === "blocked"
554
+ const { status, reason } = daemonOutcome(terminal);
555
+ const diagnostic = status === "failed" || status === "blocked" ? completionReportDiagnostic(terminal.payload?.completion_report) : undefined;
556
+ const reportDiagnostic = diagnostic && (status === "blocked"
345
557
  ? diagnostic.code === "CODEX_COMPLETION_REPORT_BLOCKED" : diagnostic.code !== "CODEX_COMPLETION_REPORT_BLOCKED") ? diagnostic : undefined;
346
558
  const securityReceipt = adapter.securityReceipt?.();
347
559
  const reads = safeLocalReadObservations(adapter.localReadObservations?.());
348
- const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
560
+ const completionRequest = structuredClone({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken, status, result: { binding_id: binding.bindingId, event_type: terminal.eventType, summary: terminal.summary, ...(localFixPublication ? { local_fix_publication: structuredClone(localFixPublication) } : {}), ...(reason ? { reason } : {}), ...(reads ? { local_read_observations: reads } : {}), ...(failureDiagnostic ? { execution_failure: { ...failureDiagnostic } } : {}), ...(reportDiagnostic ? { completion_report: reportDiagnostic } : {}), ...(securityReceipt ? { security_receipt: securityReceipt } : {}) } });
349
561
  await this.deps.state.save({ jobId: job.id, generation: job.generation, leaseToken: job.leaseToken,
350
562
  phase: "terminal_pending", bindingId: binding.bindingId, terminalEventType: terminal.eventType,
351
563
  completionRequest: structuredClone(completionRequest) });
564
+ requireControlFence();
352
565
  await this.deps.gateway.complete(completionRequest);
566
+ requireControlFence();
353
567
  await this.deps.state.clear();
354
568
  }
355
569
  }
@@ -20,7 +20,10 @@ export class JobPoller {
20
20
  this.failures = 0;
21
21
  }
22
22
  catch {
23
- this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
23
+ try {
24
+ this.deps.reportError?.({ stage, code: stage === "execute" ? "EXECUTION_UNACKNOWLEDGED" : "GATEWAY_UNAVAILABLE" });
25
+ }
26
+ catch { /* Observational logging cannot bypass stop/backoff or discard recovery state. */ }
24
27
  // Unknown execution/terminal outcomes must reconcile before any new claim.
25
28
  if (stage === "execute") {
26
29
  this.stop();