@the-open-engine/zeroshot 6.36.0 → 6.37.0

package/cli/index.js

@@ -73,6 +73,11 @@ const {
   resolveEffectiveRunPlan,
 } = require('../lib/start-cluster');
 const { requirePreflight } = require('../src/preflight');
+const {
+  exitCodeForResult,
+  isForegroundStatusSettled,
+  writeForegroundResult,
+} = require('../src/foreground-benchmark-run');
 const {
   createUnsupportedProviderCapabilityError,
   serializeTaskStartupError,
@@ -251,6 +256,17 @@ function shouldRunDetached(options) {
   return options.detach && !process.env.ZEROSHOT_DAEMON;
 }
 
+function requireForegroundResultMode(options) {
+  if (!options.resultFile) return;
+  if (shouldRunDetached(options) || process.env.ZEROSHOT_DAEMON) {
+    throw new Error('--result-file requires foreground execution without --detach');
+  }
+  if (options.pr || options.ship) {
+    throw new Error('--result-file is incompatible with --pr and --ship delivery');
+  }
+  process.env.ZEROSHOT_TASK_EXECUTION_CONTEXT = 'benchmark';
+}
+
 function printDetachedClusterStart(plan, clusterId, logPath) {
   const runMode = runModeFromPlan(plan);
   console.log(runMode ? `Started ${clusterId} (${runMode})` : `Started ${clusterId}`);
@@ -605,65 +621,83 @@ function createForegroundCleanup({
   return { stop, stopWithFlush };
 }
 
-function setupForegroundSigintHandler({ orchestrator, clusterId, cleanup, stopChecking }) {
-  const handler = async () => {
+function setupForegroundSignalHandler({ orchestrator, clusterId, cleanup, settle, handleSigterm }) {
+  let handling = false;
+  const handler = async (signal) => {
+    if (handling) return;
+    handling = true;
     cleanup.stop();
-    stopChecking();
-    console.log(chalk.dim('\n\n--- Interrupted ---'));
+    console.log(chalk.dim(`\n\n--- Interrupted (${signal}) ---`));
 
     try {
       console.log(chalk.dim(`Stopping cluster ${clusterId}...`));
       await orchestrator.stop(clusterId);
       console.log(chalk.dim(`Cluster ${clusterId} stopped.`));
+      settle({ cancelled: true });
     } catch (stopErr) {
-      console.error(chalk.red(`Failed to stop cluster: ${stopErr.message}`));
+      settle(null, stopErr);
     }
-
-    process.exit(0);
   };
 
-  process.on('SIGINT', handler);
+  const onSigint = () => handler('SIGINT');
+  const onSigterm = () => handler('SIGTERM');
+  process.on('SIGINT', onSigint);
+  if (handleSigterm) process.on('SIGTERM', onSigterm);
   return () => {
-    process.off('SIGINT', handler);
+    process.off('SIGINT', onSigint);
+    if (handleSigterm) process.off('SIGTERM', onSigterm);
   };
 }
 
-function waitForClusterCompletion(orchestrator, clusterId, cleanup) {
-  return new Promise((resolve) => {
+function waitForClusterCompletion(orchestrator, clusterId, cleanup, handleSigterm) {
+  return new Promise((resolve, reject) => {
     let checkInterval;
+    let settled = false;
+    let terminalObservedAt = null;
+    let removeSignals = () => {};
     const stopChecking = () => {
       if (checkInterval) {
         clearInterval(checkInterval);
       }
     };
-    const removeSigint = setupForegroundSigintHandler({
+    const settle = (result, error) => {
+      if (settled) return;
+      settled = true;
+      stopChecking();
+      removeSignals();
+      if (error) reject(error);
+      else resolve(result);
+    };
+    removeSignals = setupForegroundSignalHandler({
       orchestrator,
       clusterId,
       cleanup,
-      stopChecking,
+      settle,
+      handleSigterm,
     });
 
-    const finish = (finalizer) => {
-      stopChecking();
-      removeSigint();
-      finalizer();
-      resolve();
-    };
-
     checkInterval = setInterval(() => {
       try {
         const status = orchestrator.getStatus(clusterId);
-        if (status.state !== 'running') {
-          finish(cleanup.stopWithFlush);
+        if (isForegroundStatusSettled(status)) {
+          cleanup.stopWithFlush();
+          settle({ cancelled: false });
+        } else if (['stopped', 'killed'].includes(status.state)) {
+          terminalObservedAt ??= Date.now();
+          if (Date.now() - terminalObservedAt > 30_000) {
+            cleanup.stop();
+            settle(null, new Error('foreground agent processes did not settle after terminal'));
+          }
         }
-      } catch {
-        finish(cleanup.stop);
+      } catch (error) {
+        cleanup.stop();
+        settle(null, error);
       }
     }, 500);
   });
 }
 
-async function streamClusterInForeground(cluster, orchestrator, clusterId, plan) {
+async function streamClusterInForeground(cluster, orchestrator, clusterId, plan, handleSigterm) {
   const sendersWithOutput = new Set();
   const processedMessageIds = new Set();
 
@@ -696,8 +730,34 @@ async function streamClusterInForeground(cluster, orchestrator, clusterId, plan)
     sendersWithOutput,
   });
 
-  await waitForClusterCompletion(orchestrator, clusterId, cleanup);
-  console.log(chalk.dim(`\nCluster ${clusterId} completed.`));
+  const result = await waitForClusterCompletion(orchestrator, clusterId, cleanup, handleSigterm);
+  const label = result.cancelled ? 'cancelled' : 'finished';
+  console.log(chalk.dim(`\nCluster ${clusterId} ${label}.`));
+  return result;
+}
+
+async function finishForegroundRun({ cluster, orchestrator, clusterId, plan, resultPath }) {
+  try {
+    const foreground = await streamClusterInForeground(
+      cluster,
+      orchestrator,
+      clusterId,
+      plan,
+      Boolean(resultPath)
+    );
+    if (!resultPath) return;
+    const receipt = writeForegroundResult({
+      orchestrator,
+      cluster,
+      clusterId,
+      resultPath,
+      cancelled: foreground.cancelled,
+    });
+    process.exitCode = exitCodeForResult(receipt);
+    console.log(chalk.dim(`Result ${receipt.outcome} committed to ${resultPath}`));
+  } finally {
+    orchestrator.close();
+  }
 }
 
 function setupDaemonCleanup(orchestrator, clusterId) {
@@ -2662,6 +2722,10 @@ program
     'fast'
   )
   .option('-d, --detach', 'Run in background (default: attach to first agent)')
+  .option(
+    '--result-file <path>',
+    'Atomically write a closed foreground result receipt (incompatible with --detach)'
+  )
   .addHelpText(
     'after',
     `
@@ -2768,6 +2832,7 @@ Force provider flags: -G (GitHub), -L (GitLab), -J (Jira), -D (DevOps), -N (Line
       }
 
       const { generateName } = require('../src/name-generator');
+      requireForegroundResultMode(effectiveOptions);
       if (shouldRunDetached(effectiveOptions)) {
         const clusterId = generateName('cluster');
         await spawnDetachedCluster(effectiveOptions, effectiveRunPlan, clusterId, stdinText);
@@ -2811,8 +2876,13 @@ Force provider flags: -G (GitHub), -L (GitLab), -J (Jira), -D (DevOps), -N (Line
       }
 
       if (!process.env.ZEROSHOT_DAEMON) {
-        await streamClusterInForeground(cluster, orchestrator, clusterId, effectiveRunPlan);
-        orchestrator.close();
+        await finishForegroundRun({
+          cluster,
+          orchestrator,
+          clusterId,
+          plan: effectiveRunPlan,
+          resultPath: effectiveOptions.resultFile,
+        });
       }
       setupDaemonCleanup(orchestrator, clusterId);
     } catch (error) {

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@the-open-engine/zeroshot",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@the-open-engine/zeroshot",
-      "version": "6.36.0",
+      "version": "6.37.0",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@the-open-engine/zeroshot",
-  "version": "6.36.0",
+  "version": "6.37.0",
   "description": "Independent executor–verifier orchestration for software changes.",
   "main": "src/orchestrator.js",
   "bin": {

package/src/agent/agent-lifecycle.js

@@ -17,7 +17,7 @@ const { executeHook } = require('./agent-hook-executor');
 const IsolationManager = require('../isolation-manager');
 const crypto = require('crypto');
 const { bufferMessage, scheduleDrain, drainBufferedMessages } = require('../message-buffer');
-const { isPlatformSupported } = require('./agent-stuck-detector');
+const { createLivenessPoll } = require('./agent-liveness-poll');
 const { normalizeProviderName, getDefaultProviderId } = require('../../lib/provider-names');
 const { loadSettings } = require('../../lib/settings');
 const { findPlatformMismatchReason } = require('./validation-platform');
@@ -223,7 +223,7 @@ async function stop(agent) {
   stopLivenessCheck(agent);
 
   const hasNestedExecutions = agent.nestedExecutions?.hasActive === true;
-  if (!agent.running && !agent.currentTask && !hasNestedExecutions) {
+  if (!agent.running && !agent.currentTask && !hasNestedExecutions && !agent._currentExecution) {
     return;
   }
 
@@ -242,26 +242,26 @@ async function stop(agent) {
       throw new Error(`Task shutdown could not confirm termination: ${termination.reason}`);
     }
   }
-
-  // Wait for in-flight execution to complete (up to 5 seconds)
-  // This prevents write-after-close race conditions
-  if (agent._currentExecution) {
+  const currentExecution = agent._currentExecution;
+  if (currentExecution) {
     let executionTimeout = null;
     try {
-      await Promise.race([
-        agent._currentExecution,
+      const outcome = await Promise.race([
+        Promise.resolve(currentExecution).then(
+          () => 'settled',
+          () => 'settled'
+        ),
         new Promise((resolve) => {
-          executionTimeout = setTimeout(resolve, 5000);
+          executionTimeout = setTimeout(() => resolve('timeout'), 5000);
         }),
       ]);
-    } catch {
-      // Ignore errors from cancelled execution
-    } finally {
-      if (executionTimeout) {
-        clearTimeout(executionTimeout);
+      if (outcome === 'timeout') {
+        throw new Error(`Agent ${agent.id} execution did not settle after task termination`);
       }
-      agent._currentExecution = null;
+    } finally {
+      if (executionTimeout) clearTimeout(executionTimeout);
     }
+    if (agent._currentExecution === currentExecution) agent._currentExecution = null;
   }
 
   agent._log(`Agent ${agent.id} stopped`);
@@ -1268,67 +1268,16 @@ function startLivenessCheck(agent) {
   agent.livenessTerminationAttempts = 0;
   agent.livenessTerminationRetryAt = 0;
 
-  agent.livenessCheckInterval = setInterval(() => {
-    const hasRecoverableTask =
-      Boolean(agent.currentTask) ||
-      Boolean(agent.isolation?.enabled && agent.currentTaskId) ||
-      agent.nestedExecutions?.hasActive === true;
-    if (!hasRecoverableTask || agent.livenessTerminationStarted) {
-      return;
-    }
-
-    const now = Date.now();
-    if (agent.livenessTerminationContext) {
-      if (now >= agent.livenessTerminationRetryAt) {
-        attemptLivenessTermination(agent, settings);
-      }
-      return;
-    }
-
-    const taskStartedAt = agent.taskStartedAt || agent.lastOutputTime || now;
-    const lastOutputTime = agent.lastOutputTime || taskStartedAt;
-    const taskRuntime = now - taskStartedAt;
-    const timeSinceLastOutput = now - lastOutputTime;
-
-    if (configuredTimeout && taskRuntime >= configuredTimeout) {
-      const reason = `Task timed out after ${configuredTimeout}ms`;
-      beginLivenessTermination(agent, settings, reason, 'AGENT_TASK_TIMEOUT', {
-        taskId: agent.currentTaskId,
-        taskRuntime,
-        timeout: configuredTimeout,
-      });
-      return;
-    }
-
-    if (timeSinceLastOutput < staleDuration) {
-      agent.consecutiveStaleWarnings = 0;
-      return;
-    }
-
-    agent.consecutiveStaleWarnings += 1;
-    agent._publishLifecycle('AGENT_STALE_WARNING', {
-      taskId: agent.currentTaskId,
-      timeSinceLastOutput,
-      staleDuration,
-      lastOutputTime,
-      consecutiveWarnings: agent.consecutiveStaleWarnings,
-      warningsBeforeKill,
-      processDiagnosticsAvailable: isPlatformSupported(),
-      analysis: `Provider produced no output for ${timeSinceLastOutput}ms`,
-    });
-
-    if (agent.consecutiveStaleWarnings < warningsBeforeKill) {
-      return;
-    }
-
-    const reason = `Provider produced no output for ${timeSinceLastOutput}ms`;
-    beginLivenessTermination(agent, settings, reason, 'PROVIDER_INACTIVITY_TIMEOUT', {
-      taskId: agent.currentTaskId,
-      timeSinceLastOutput,
-      staleDuration,
-      consecutiveWarnings: agent.consecutiveStaleWarnings,
-    });
-  }, checkIntervalMs);
+  const poll = createLivenessPoll({
+    agent,
+    settings,
+    configuredTimeout,
+    staleDuration,
+    warningsBeforeKill,
+    attemptTermination: attemptLivenessTermination,
+    beginTermination: beginLivenessTermination,
+  });
+  agent.livenessCheckInterval = setInterval(poll, checkIntervalMs);
 }
 
 const MAX_LIVENESS_TERMINATION_ATTEMPTS = 3;

package/src/agent/agent-liveness-poll.js

@@ -0,0 +1,93 @@
+const { isPlatformSupported } = require('./agent-stuck-detector');
+
+function hasRecoverableTask(agent) {
+  return (
+    Boolean(agent.currentTask) ||
+    Boolean(agent.isolation?.enabled && agent.currentTaskId) ||
+    agent.nestedExecutions?.hasActive === true
+  );
+}
+
+function handlePendingTermination(agent, settings, now, attemptTermination) {
+  if (!agent.livenessTerminationContext) return false;
+  if (now >= agent.livenessTerminationRetryAt) attemptTermination(agent, settings);
+  return true;
+}
+
+function taskTiming(agent, now) {
+  const taskStartedAt = agent.taskStartedAt || agent.lastOutputTime || now;
+  const lastOutputTime = agent.lastOutputTime || taskStartedAt;
+  return {
+    taskRuntime: now - taskStartedAt,
+    timeSinceLastOutput: now - lastOutputTime,
+    lastOutputTime,
+  };
+}
+
+function handleTaskTimeout(context, timing) {
+  const { agent, settings, configuredTimeout, beginTermination } = context;
+  if (!configuredTimeout || timing.taskRuntime < configuredTimeout) return false;
+  beginTermination(
+    agent,
+    settings,
+    `Task timed out after ${configuredTimeout}ms`,
+    'AGENT_TASK_TIMEOUT',
+    {
+      taskId: agent.currentTaskId,
+      taskRuntime: timing.taskRuntime,
+      timeout: configuredTimeout,
+    }
+  );
+  return true;
+}
+
+function publishStaleWarning(context, timing) {
+  const { agent, staleDuration, warningsBeforeKill } = context;
+  agent.consecutiveStaleWarnings += 1;
+  agent._publishLifecycle('AGENT_STALE_WARNING', {
+    taskId: agent.currentTaskId,
+    timeSinceLastOutput: timing.timeSinceLastOutput,
+    staleDuration,
+    lastOutputTime: timing.lastOutputTime,
+    consecutiveWarnings: agent.consecutiveStaleWarnings,
+    warningsBeforeKill,
+    processDiagnosticsAvailable: isPlatformSupported(),
+    analysis: `Provider produced no output for ${timing.timeSinceLastOutput}ms`,
+  });
+}
+
+function terminateForInactivity(context, timing) {
+  const { agent, settings, staleDuration, beginTermination } = context;
+  beginTermination(
+    agent,
+    settings,
+    `Provider produced no output for ${timing.timeSinceLastOutput}ms`,
+    'PROVIDER_INACTIVITY_TIMEOUT',
+    {
+      taskId: agent.currentTaskId,
+      timeSinceLastOutput: timing.timeSinceLastOutput,
+      staleDuration,
+      consecutiveWarnings: agent.consecutiveStaleWarnings,
+    }
+  );
+}
+
+function createLivenessPoll(context) {
+  const { agent, settings, staleDuration, warningsBeforeKill, attemptTermination } = context;
+  return () => {
+    if (!hasRecoverableTask(agent) || agent.livenessTerminationStarted) return;
+    const now = Date.now();
+    if (handlePendingTermination(agent, settings, now, attemptTermination)) return;
+    const timing = taskTiming(agent, now);
+    if (handleTaskTimeout(context, timing)) return;
+    if (timing.timeSinceLastOutput < staleDuration) {
+      agent.consecutiveStaleWarnings = 0;
+      return;
+    }
+    publishStaleWarning(context, timing);
+    if (agent.consecutiveStaleWarnings < warningsBeforeKill) return;
+    terminateForInactivity(context, timing);
+  };
+}
+
+module.exports = { createLivenessPoll };

package/src/agent/agent-task-executor.js

@@ -1866,6 +1866,19 @@ function finalizeLogFollow(agent, state) {
   }
 }
 
+function finishHostLogCapture(agent, state, pollLogFile, processNewContent, broadcastLine) {
+  // A task status can become terminal before the watcher has flushed its final
+  // filesystem write (observed on Modal). Stop the timers first, then take one
+  // authoritative catch-up read and finish any final UTF-8/non-newline record.
+  finalizeLogFollow(agent, state);
+  pollLogFile();
+  const decoderTail = state.logDecoder.end();
+  if (decoderTail) processNewContent(decoderTail);
+  if (state.lineBuffer.byteLength > 0) {
+    completeLogRecord(state.lineBuffer, broadcastLine, true);
+  }
+}
+
 function settleHostStatusFailure({ agent, providerName, state, resolve, text, data, error }) {
   if (state.resolved) return;
   state.resolved = true;
@@ -1986,6 +1999,7 @@ function handleStatusCompletion({
   state,
   stdout,
   pollLogFile,
+  finishLogCapture,
   resolve,
   reject,
 }) {
@@ -2012,7 +2026,7 @@ function handleStatusCompletion({
     if (state.resolved) return;
     state.resolved = true;
 
-    finalizeLogFollow(agent, state);
+    finishLogCapture();
     flushAgentOutput(agent, providerName, state);
 
     buildCompletionResult({
@@ -2066,14 +2080,12 @@ function createLogFollower({
     const state = createLogFollowState();
     state.skipStructuredResultCheck = skipStructuredResultCheck;
     state.nested = nested;
-
     state.logFilePath = lookupLogFilePath(ctPath, taskId);
     if (state.logFilePath) {
       agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
     } else {
       agent._log(`⏳ Agent ${agent.id}: Waiting for log file...`);
     }
-
     const broadcastLine = (line) => broadcastAgentLine({ agent, providerName, state, line });
     const processNewContent = (content) => appendContentToBuffer(state, content, broadcastLine);
     const pollLogFile = () =>
@@ -2085,6 +2097,8 @@ function createLogFollower({
         state,
         onNewContent: processNewContent,
       });
+    const finishLogCapture = () =>
+      finishHostLogCapture(agent, state, pollLogFile, processNewContent, broadcastLine);
 
     state.pollInterval = setInterval(pollLogFile, 300);
 
@@ -2120,6 +2134,7 @@ function createLogFollower({
             state,
             stdout,
             pollLogFile,
+            finishLogCapture,
             resolve,
             reject,
           });
@@ -3546,6 +3561,7 @@ module.exports = {
   consumeIsolatedTailChunk,
   flushAgentOutput,
   flushIsolatedOutput,
+  finishHostLogCapture,
   CONTROL_PLANE_OUTPUT_LIMITS: Object.freeze({
     maxBytes: MAX_CONTROL_PLANE_OUTPUT_BYTES,
     maxRecords: MAX_CONTROL_PLANE_OUTPUT_RECORDS,

package/src/foreground-benchmark-files.js

@@ -0,0 +1,81 @@
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { buildTelemetry } = require('./foreground-benchmark-result');
+
+function serialized(value) {
+  return Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
+}
+
+function atomicWriteNew(targetPath, content) {
+  const directory = path.dirname(targetPath);
+  const temporary = path.join(
+    directory,
+    `.${path.basename(targetPath)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`
+  );
+  let descriptor;
+  let published = false;
+  try {
+    descriptor = fs.openSync(temporary, 'wx', 0o600);
+    fs.writeFileSync(descriptor, content);
+    fs.fsyncSync(descriptor);
+    fs.closeSync(descriptor);
+    descriptor = undefined;
+    fs.linkSync(temporary, targetPath);
+    published = true;
+    fs.unlinkSync(temporary);
+    const directoryDescriptor = fs.openSync(directory, fs.constants.O_RDONLY);
+    try {
+      fs.fsyncSync(directoryDescriptor);
+    } finally {
+      fs.closeSync(directoryDescriptor);
+    }
+  } catch (error) {
+    error.atomicTargetPublished = published;
+    throw error;
+  } finally {
+    if (descriptor !== undefined) fs.closeSync(descriptor);
+    try {
+      fs.unlinkSync(temporary);
+    } catch {
+      // Preserve the primary write error; a leftover randomized temp is non-authoritative.
+    }
+  }
+}
+
+function removeOrphanTelemetry(telemetryPath, primaryError) {
+  try {
+    fs.unlinkSync(telemetryPath);
+  } catch (cleanupError) {
+    if (cleanupError.code !== 'ENOENT') primaryError.cleanupError = cleanupError;
+  }
+}
+
+function writeBenchmarkResultBundle(resultPath, result, snapshot) {
+  if (typeof resultPath !== 'string' || resultPath.length === 0) {
+    throw new Error('result path must be non-empty text');
+  }
+  const resolvedResult = path.resolve(resultPath);
+  const telemetryPath = `${resolvedResult}.telemetry.json`;
+  const telemetry = buildTelemetry(result.runId, snapshot);
+  const telemetryBytes = serialized(telemetry);
+  atomicWriteNew(telemetryPath, telemetryBytes);
+  const receipt = {
+    ...result,
+    telemetry: {
+      artifact: path.basename(telemetryPath),
+      byteLength: telemetryBytes.length,
+      sha256: crypto.createHash('sha256').update(telemetryBytes).digest('hex'),
+    },
+  };
+  try {
+    atomicWriteNew(resolvedResult, serialized(receipt));
+  } catch (error) {
+    if (!error.atomicTargetPublished) removeOrphanTelemetry(telemetryPath, error);
+    throw error;
+  }
+  return receipt;
+}
+
+module.exports = { writeBenchmarkResultBundle };

package/src/foreground-benchmark-result.js

@@ -0,0 +1,233 @@
+const crypto = require('node:crypto');
+const { VALID_PROVIDERS } = require('../lib/provider-names');
+
+const RESULT_SCHEMA = 'zeroshot-benchmark-result/v1';
+const TELEMETRY_SCHEMA = 'zeroshot-benchmark-telemetry/v1';
+const EMPTY_DIAGNOSTIC = Object.freeze({
+  byteLength: 0,
+  sha256: crypto.createHash('sha256').update('').digest('hex'),
+});
+const TASK_FAILURE_REASONS = new Set(['max_iterations', 'structured_output_invalid']);
+const PROVIDER_CODES = new Set(['crash', 'refusal']);
+const PROVIDERS = new Set(VALID_PROVIDERS);
+const PROVIDER_EVENTS = new Set(['terminal_error', 'turn.failed']);
+const PROVIDER_CATEGORIES = new Set([
+  'authentication',
+  'permanent',
+  'quota',
+  'transient',
+  'unknown',
+]);
+const PROVIDER_KINDS = new Set([
+  'permanent-pattern',
+  'rate-limit',
+  'retryable-pattern',
+  'status-permanent',
+  'status-retryable',
+  'code-retryable',
+  'unknown-retryable',
+]);
+const TOKEN_FIELDS = [
+  'inputTokens',
+  'outputTokens',
+  'cacheReadInputTokens',
+  'cacheCreationInputTokens',
+  'totalCostUsd',
+  'count',
+];
+
+function requireObject(value, label) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(`${label} must be an object`);
+  }
+  return value;
+}
+
+function requireClosedText(value, allowed, label) {
+  if (typeof value !== 'string' || !allowed.has(value)) {
+    throw new Error(`${label} is outside the closed result contract`);
+  }
+  return value;
+}
+
+function requireDiagnostic(value) {
+  if (
+    value &&
+    Number.isSafeInteger(value.byteLength) &&
+    value.byteLength >= 0 &&
+    typeof value.sha256 === 'string' &&
+    /^[a-f0-9]{64}$/.test(value.sha256)
+  ) {
+    return { byteLength: value.byteLength, sha256: value.sha256 };
+  }
+  throw new Error('provider diagnostic is outside the closed result contract');
+}
+
+function terminalData(message) {
+  return requireObject(requireObject(message.content, 'terminal content').data, 'terminal data');
+}
+
+function isExplicitTaskFailure(message, data) {
+  if (message.sender === 'orchestrator' || !TASK_FAILURE_REASONS.has(data.reason)) return false;
+  if (data.reason === 'max_iterations') return message.receiver === 'system';
+  return message.receiver === 'broadcast' && data.code === 'STRUCTURED_OUTPUT_INVALID';
+}
+
+function classifyTerminal(message) {
+  if (message.topic === 'CLUSTER_COMPLETE') {
+    return {
+      outcome: 'completed',
+      terminalOwner: 'task',
+      code: 'ok',
+      kind: 'workflow_complete',
+      retryable: false,
+      diagnostic: { ...EMPTY_DIAGNOSTIC },
+      provider: null,
+      event: null,
+      category: null,
+    };
+  }
+
+  if (message.topic !== 'CLUSTER_FAILED') {
+    throw new Error(`unsupported terminal topic: ${message.topic}`);
+  }
+  const data = terminalData(message);
+  if (isExplicitTaskFailure(message, data)) {
+    return {
+      outcome: 'task_failure',
+      terminalOwner: 'task',
+      code: data.reason,
+      kind: 'declared_failure',
+      retryable: false,
+      diagnostic: { ...EMPTY_DIAGNOSTIC },
+      provider: null,
+      event: null,
+      category: null,
+    };
+  }
+  if (data.reason === 'provider_execution_failed') {
+    if (typeof data.retryable !== 'boolean') {
+      throw new Error('provider retryable must be boolean');
+    }
+    return {
+      outcome: 'provider_failure',
+      terminalOwner: 'provider',
+      code: requireClosedText(data.code, PROVIDER_CODES, 'provider code'),
+      kind: requireClosedText(data.kind, PROVIDER_KINDS, 'provider kind'),
+      retryable: data.retryable,
+      diagnostic: requireDiagnostic(data.diagnostic),
+      provider: requireClosedText(data.provider, PROVIDERS, 'provider'),
+      event: requireClosedText(data.event, PROVIDER_EVENTS, 'provider event'),
+      category: requireClosedText(data.category, PROVIDER_CATEGORIES, 'provider category'),
+    };
+  }
+  return {
+    outcome: 'engine_failure',
+    terminalOwner: 'engine',
+    code: 'engine_failed',
+    kind: 'declared_failure',
+    retryable: false,
+    diagnostic: { ...EMPTY_DIAGNOSTIC },
+    provider: null,
+    event: null,
+    category: null,
+  };
+}
+
+function validateStoppedAgents(agents) {
+  if (!Array.isArray(agents)) throw new Error('agents must be an array');
+  for (const agent of agents) {
+    const state = requireObject(agent, 'agent state');
+    if (state.pid !== null && state.pid !== undefined) {
+      throw new Error(`agent ${String(state.id)} still has a live process identity`);
+    }
+  }
+}
+
+function validateRunId(runId) {
+  const parts = typeof runId === 'string' ? runId.split('-') : [];
+  const valid =
+    parts.length >= 2 &&
+    parts.every(
+      (part) => part.length > 0 && [...part].every((character) => /[a-z0-9]/.test(character))
+    );
+  if (!valid) {
+    throw new Error('runId must be a canonical cluster id');
+  }
+}
+
+function buildBenchmarkResult({ runId, terminalMessages, agents }) {
+  validateRunId(runId);
+  if (!Array.isArray(terminalMessages) || terminalMessages.length !== 1) {
+    throw new Error('foreground run must have exactly one terminal event');
+  }
+  validateStoppedAgents(agents);
+  return {
+    schema: RESULT_SCHEMA,
+    runId,
+    ...classifyTerminal(terminalMessages[0]),
+  };
+}
+
+function buildCancelledResult({ runId, agents }) {
+  validateRunId(runId);
+  validateStoppedAgents(agents);
+  return {
+    schema: RESULT_SCHEMA,
+    runId,
+    outcome: 'cancelled',
+    terminalOwner: 'controller',
+    code: 'cancelled',
+    kind: 'controlled_cancellation',
+    retryable: false,
+    diagnostic: { ...EMPTY_DIAGNOSTIC },
+    provider: null,
+    event: null,
+    category: null,
+  };
+}
+
+function normalizeTokenEntry(value, label) {
+  const source = requireObject(value, label);
+  const entry = {};
+  for (const field of TOKEN_FIELDS) {
+    const amount = source[field] ?? 0;
+    if (typeof amount !== 'number' || !Number.isFinite(amount) || amount < 0) {
+      throw new Error(`${label}.${field} must be a finite non-negative number`);
+    }
+    entry[field] = amount;
+  }
+  return entry;
+}
+
+function buildTelemetry(runId, snapshot) {
+  validateRunId(runId);
+  const source = requireObject(snapshot, 'telemetry snapshot');
+  if (!Number.isSafeInteger(source.messageCount) || source.messageCount < 0) {
+    throw new Error('telemetry messageCount must be a non-negative safe integer');
+  }
+  const roles = requireObject(source.tokensByRole, 'tokensByRole');
+  const names = Object.keys(roles).sort();
+  if (names.length > 64) throw new Error('telemetry role count exceeds 64');
+  const tokensByRole = {};
+  for (const name of names) {
+    const validRoleName =
+      name === '_total' ||
+      (name.length <= 128 &&
+        /^[a-zA-Z0-9]$/.test(name[0] || '') &&
+        [...name].every((character) => /[a-zA-Z0-9_.-]/.test(character)));
+    if (!validRoleName) {
+      throw new Error('telemetry role name is invalid');
+    }
+    tokensByRole[name] = normalizeTokenEntry(roles[name], `tokensByRole.${name}`);
+  }
+  return { schema: TELEMETRY_SCHEMA, runId, messageCount: source.messageCount, tokensByRole };
+}
+
+module.exports = {
+  RESULT_SCHEMA,
+  TELEMETRY_SCHEMA,
+  buildBenchmarkResult,
+  buildCancelledResult,
+  buildTelemetry,
+};

package/src/foreground-benchmark-run.js

@@ -0,0 +1,72 @@
+const { buildBenchmarkResult, buildCancelledResult } = require('./foreground-benchmark-result');
+const { writeBenchmarkResultBundle } = require('./foreground-benchmark-files');
+
+const TERMINAL_TOPICS = ['CLUSTER_COMPLETE', 'CLUSTER_FAILED'];
+const VERIFIER_ELIGIBLE = new Set(['completed', 'task_failure']);
+const EXIT_CODES = Object.freeze({
+  provider_failure: 20,
+  engine_failure: 21,
+  cancelled: 22,
+});
+
+function isForegroundStatusSettled(status) {
+  return (
+    status &&
+    ['stopped', 'killed'].includes(status.state) &&
+    status.isZombie === false &&
+    Array.isArray(status.agents) &&
+    status.agents.every((agent) => agent && (agent.pid === null || agent.pid === undefined))
+  );
+}
+
+function terminalMessages(cluster, clusterId) {
+  const messages = TERMINAL_TOPICS.flatMap((topic) =>
+    cluster.messageBus.query({ cluster_id: clusterId, topic })
+  );
+  return messages.sort((left, right) => {
+    const a = BigInt(left.sequence);
+    const b = BigInt(right.sequence);
+    if (a < b) return -1;
+    if (a > b) return 1;
+    return 0;
+  });
+}
+
+function requireSettledStatus(orchestrator, clusterId) {
+  const status = orchestrator.getStatus(clusterId);
+  if (!isForegroundStatusSettled(status)) {
+    throw new Error(`foreground cluster is not settled: ${String(status?.state || 'unavailable')}`);
+  }
+  return status;
+}
+
+function writeForegroundResult({ orchestrator, cluster, clusterId, resultPath, cancelled }) {
+  const status = requireSettledStatus(orchestrator, clusterId);
+  const terminals = terminalMessages(cluster, clusterId);
+  let result;
+  if (cancelled && terminals.length === 0) {
+    result = buildCancelledResult({ runId: clusterId, agents: status.agents });
+  } else {
+    result = buildBenchmarkResult({
+      runId: clusterId,
+      terminalMessages: terminals,
+      agents: status.agents,
+    });
+  }
+  const snapshot = cluster.messageBus.readSnapshot(clusterId);
+  return writeBenchmarkResultBundle(resultPath, result, snapshot);
+}
+
+function exitCodeForResult(result) {
+  if (VERIFIER_ELIGIBLE.has(result.outcome)) return 0;
+  const exitCode = EXIT_CODES[result.outcome];
+  if (exitCode === undefined) throw new Error(`unsupported result outcome: ${result.outcome}`);
+  return exitCode;
+}
+
+module.exports = {
+  exitCodeForResult,
+  isForegroundStatusSettled,
+  terminalMessages,
+  writeForegroundResult,
+};

package/task-lib/runner.js

@@ -625,6 +625,13 @@ export function shouldUseAttachableWatcher(options, providerName) {
     return false;
   }
 
+  // Benchmark runs are non-interactive and keep the cluster process in the foreground. Use the
+  // pipe watcher so task completion is observed only after stdout/stderr close; a PTY exit can
+  // race its final buffered output on remote runtimes and lose the terminal structured result.
+  if (resolveTaskExecutionContext() === 'benchmark') {
+    return false;
+  }
+
   // The rpc-stdio lane owns bidirectional correlated RPC over stdio itself (see
   // omp-rpc-driver.ts) and always uses rpc-watcher.js instead of the attachable PTY watcher.
   if (getProviderRegistryEntry(providerName).invoke.lane === 'rpc-stdio') {