@the-open-engine/zeroshot 6.36.0 → 6.38.0

package/src/foreground-benchmark-result.js

@@ -0,0 +1,233 @@
+const crypto = require('node:crypto');
+const { VALID_PROVIDERS } = require('../lib/provider-names');
+
+const RESULT_SCHEMA = 'zeroshot-benchmark-result/v1';
+const TELEMETRY_SCHEMA = 'zeroshot-benchmark-telemetry/v1';
+const EMPTY_DIAGNOSTIC = Object.freeze({
+  byteLength: 0,
+  sha256: crypto.createHash('sha256').update('').digest('hex'),
+});
+const TASK_FAILURE_REASONS = new Set(['max_iterations', 'structured_output_invalid']);
+const PROVIDER_CODES = new Set(['crash', 'refusal']);
+const PROVIDERS = new Set(VALID_PROVIDERS);
+const PROVIDER_EVENTS = new Set(['terminal_error', 'turn.failed']);
+const PROVIDER_CATEGORIES = new Set([
+  'authentication',
+  'permanent',
+  'quota',
+  'transient',
+  'unknown',
+]);
+const PROVIDER_KINDS = new Set([
+  'permanent-pattern',
+  'rate-limit',
+  'retryable-pattern',
+  'status-permanent',
+  'status-retryable',
+  'code-retryable',
+  'unknown-retryable',
+]);
+const TOKEN_FIELDS = [
+  'inputTokens',
+  'outputTokens',
+  'cacheReadInputTokens',
+  'cacheCreationInputTokens',
+  'totalCostUsd',
+  'count',
+];
+
+function requireObject(value, label) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(`${label} must be an object`);
+  }
+  return value;
+}
+
+function requireClosedText(value, allowed, label) {
+  if (typeof value !== 'string' || !allowed.has(value)) {
+    throw new Error(`${label} is outside the closed result contract`);
+  }
+  return value;
+}
+
+function requireDiagnostic(value) {
+  if (
+    value &&
+    Number.isSafeInteger(value.byteLength) &&
+    value.byteLength >= 0 &&
+    typeof value.sha256 === 'string' &&
+    /^[a-f0-9]{64}$/.test(value.sha256)
+  ) {
+    return { byteLength: value.byteLength, sha256: value.sha256 };
+  }
+  throw new Error('provider diagnostic is outside the closed result contract');
+}
+
+function terminalData(message) {
+  return requireObject(requireObject(message.content, 'terminal content').data, 'terminal data');
+}
+
+function isExplicitTaskFailure(message, data) {
+  if (message.sender === 'orchestrator' || !TASK_FAILURE_REASONS.has(data.reason)) return false;
+  if (data.reason === 'max_iterations') return message.receiver === 'system';
+  return message.receiver === 'broadcast' && data.code === 'STRUCTURED_OUTPUT_INVALID';
+}
+
+function classifyTerminal(message) {
+  if (message.topic === 'CLUSTER_COMPLETE') {
+    return {
+      outcome: 'completed',
+      terminalOwner: 'task',
+      code: 'ok',
+      kind: 'workflow_complete',
+      retryable: false,
+      diagnostic: { ...EMPTY_DIAGNOSTIC },
+      provider: null,
+      event: null,
+      category: null,
+    };
+  }
+
+  if (message.topic !== 'CLUSTER_FAILED') {
+    throw new Error(`unsupported terminal topic: ${message.topic}`);
+  }
+  const data = terminalData(message);
+  if (isExplicitTaskFailure(message, data)) {
+    return {
+      outcome: 'task_failure',
+      terminalOwner: 'task',
+      code: data.reason,
+      kind: 'declared_failure',
+      retryable: false,
+      diagnostic: { ...EMPTY_DIAGNOSTIC },
+      provider: null,
+      event: null,
+      category: null,
+    };
+  }
+  if (data.reason === 'provider_execution_failed') {
+    if (typeof data.retryable !== 'boolean') {
+      throw new Error('provider retryable must be boolean');
+    }
+    return {
+      outcome: 'provider_failure',
+      terminalOwner: 'provider',
+      code: requireClosedText(data.code, PROVIDER_CODES, 'provider code'),
+      kind: requireClosedText(data.kind, PROVIDER_KINDS, 'provider kind'),
+      retryable: data.retryable,
+      diagnostic: requireDiagnostic(data.diagnostic),
+      provider: requireClosedText(data.provider, PROVIDERS, 'provider'),
+      event: requireClosedText(data.event, PROVIDER_EVENTS, 'provider event'),
+      category: requireClosedText(data.category, PROVIDER_CATEGORIES, 'provider category'),
+    };
+  }
+  return {
+    outcome: 'engine_failure',
+    terminalOwner: 'engine',
+    code: 'engine_failed',
+    kind: 'declared_failure',
+    retryable: false,
+    diagnostic: { ...EMPTY_DIAGNOSTIC },
+    provider: null,
+    event: null,
+    category: null,
+  };
+}
+
+function validateStoppedAgents(agents) {
+  if (!Array.isArray(agents)) throw new Error('agents must be an array');
+  for (const agent of agents) {
+    const state = requireObject(agent, 'agent state');
+    if (state.pid !== null && state.pid !== undefined) {
+      throw new Error(`agent ${String(state.id)} still has a live process identity`);
+    }
+  }
+}
+
+function validateRunId(runId) {
+  const parts = typeof runId === 'string' ? runId.split('-') : [];
+  const valid =
+    parts.length >= 2 &&
+    parts.every(
+      (part) => part.length > 0 && [...part].every((character) => /[a-z0-9]/.test(character))
+    );
+  if (!valid) {
+    throw new Error('runId must be a canonical cluster id');
+  }
+}
+
+function buildBenchmarkResult({ runId, terminalMessages, agents }) {
+  validateRunId(runId);
+  if (!Array.isArray(terminalMessages) || terminalMessages.length !== 1) {
+    throw new Error('foreground run must have exactly one terminal event');
+  }
+  validateStoppedAgents(agents);
+  return {
+    schema: RESULT_SCHEMA,
+    runId,
+    ...classifyTerminal(terminalMessages[0]),
+  };
+}
+
+function buildCancelledResult({ runId, agents }) {
+  validateRunId(runId);
+  validateStoppedAgents(agents);
+  return {
+    schema: RESULT_SCHEMA,
+    runId,
+    outcome: 'cancelled',
+    terminalOwner: 'controller',
+    code: 'cancelled',
+    kind: 'controlled_cancellation',
+    retryable: false,
+    diagnostic: { ...EMPTY_DIAGNOSTIC },
+    provider: null,
+    event: null,
+    category: null,
+  };
+}
+
+function normalizeTokenEntry(value, label) {
+  const source = requireObject(value, label);
+  const entry = {};
+  for (const field of TOKEN_FIELDS) {
+    const amount = source[field] ?? 0;
+    if (typeof amount !== 'number' || !Number.isFinite(amount) || amount < 0) {
+      throw new Error(`${label}.${field} must be a finite non-negative number`);
+    }
+    entry[field] = amount;
+  }
+  return entry;
+}
+
+function buildTelemetry(runId, snapshot) {
+  validateRunId(runId);
+  const source = requireObject(snapshot, 'telemetry snapshot');
+  if (!Number.isSafeInteger(source.messageCount) || source.messageCount < 0) {
+    throw new Error('telemetry messageCount must be a non-negative safe integer');
+  }
+  const roles = requireObject(source.tokensByRole, 'tokensByRole');
+  const names = Object.keys(roles).sort();
+  if (names.length > 64) throw new Error('telemetry role count exceeds 64');
+  const tokensByRole = {};
+  for (const name of names) {
+    const validRoleName =
+      name === '_total' ||
+      (name.length <= 128 &&
+        /^[a-zA-Z0-9]$/.test(name[0] || '') &&
+        [...name].every((character) => /[a-zA-Z0-9_.-]/.test(character)));
+    if (!validRoleName) {
+      throw new Error('telemetry role name is invalid');
+    }
+    tokensByRole[name] = normalizeTokenEntry(roles[name], `tokensByRole.${name}`);
+  }
+  return { schema: TELEMETRY_SCHEMA, runId, messageCount: source.messageCount, tokensByRole };
+}
+
+module.exports = {
+  RESULT_SCHEMA,
+  TELEMETRY_SCHEMA,
+  buildBenchmarkResult,
+  buildCancelledResult,
+  buildTelemetry,
+};

package/src/foreground-benchmark-run.js

@@ -0,0 +1,72 @@
+const { buildBenchmarkResult, buildCancelledResult } = require('./foreground-benchmark-result');
+const { writeBenchmarkResultBundle } = require('./foreground-benchmark-files');
+
+const TERMINAL_TOPICS = ['CLUSTER_COMPLETE', 'CLUSTER_FAILED'];
+const VERIFIER_ELIGIBLE = new Set(['completed', 'task_failure']);
+const EXIT_CODES = Object.freeze({
+  provider_failure: 20,
+  engine_failure: 21,
+  cancelled: 22,
+});
+
+function isForegroundStatusSettled(status) {
+  return (
+    status &&
+    ['stopped', 'killed'].includes(status.state) &&
+    status.isZombie === false &&
+    Array.isArray(status.agents) &&
+    status.agents.every((agent) => agent && (agent.pid === null || agent.pid === undefined))
+  );
+}
+
+function terminalMessages(cluster, clusterId) {
+  const messages = TERMINAL_TOPICS.flatMap((topic) =>
+    cluster.messageBus.query({ cluster_id: clusterId, topic })
+  );
+  return messages.sort((left, right) => {
+    const a = BigInt(left.sequence);
+    const b = BigInt(right.sequence);
+    if (a < b) return -1;
+    if (a > b) return 1;
+    return 0;
+  });
+}
+
+function requireSettledStatus(orchestrator, clusterId) {
+  const status = orchestrator.getStatus(clusterId);
+  if (!isForegroundStatusSettled(status)) {
+    throw new Error(`foreground cluster is not settled: ${String(status?.state || 'unavailable')}`);
+  }
+  return status;
+}
+
+function writeForegroundResult({ orchestrator, cluster, clusterId, resultPath, cancelled }) {
+  const status = requireSettledStatus(orchestrator, clusterId);
+  const terminals = terminalMessages(cluster, clusterId);
+  let result;
+  if (cancelled && terminals.length === 0) {
+    result = buildCancelledResult({ runId: clusterId, agents: status.agents });
+  } else {
+    result = buildBenchmarkResult({
+      runId: clusterId,
+      terminalMessages: terminals,
+      agents: status.agents,
+    });
+  }
+  const snapshot = cluster.messageBus.readSnapshot(clusterId);
+  return writeBenchmarkResultBundle(resultPath, result, snapshot);
+}
+
+function exitCodeForResult(result) {
+  if (VERIFIER_ELIGIBLE.has(result.outcome)) return 0;
+  const exitCode = EXIT_CODES[result.outcome];
+  if (exitCode === undefined) throw new Error(`unsupported result outcome: ${result.outcome}`);
+  return exitCode;
+}
+
+module.exports = {
+  exitCodeForResult,
+  isForegroundStatusSettled,
+  terminalMessages,
+  writeForegroundResult,
+};

package/src/isolation-manager.js

@@ -30,6 +30,7 @@ const {
   isUsableEnvValue,
   validateProviderEnvAuth,
 } = require('../lib/docker-config');
+const { resolveProviderCredentialPaths } = require('../lib/provider-credential-path');
 const { getProvider } = require('./providers');
 const { readRepoSettings } = require('../lib/repo-settings');
 const { provisionClaudeCredentials } = require('./claude-credentials');
@@ -151,6 +152,34 @@ function expandHomePath(value) {
   return value.replace(/^~(?=\/|$)/, os.homedir());
 }
 
+function appendCredentialMount(plan, mount, explicitContainerPaths, claudeContainerPath) {
+  if (mount.container === claudeContainerPath) {
+    console.warn(
+      `[IsolationManager] Skipping mount for ${mount.host} -> ${mount.container} ` +
+        '(Claude config is managed by zeroshot).'
+    );
+    return;
+  }
+
+  const hostPath = expandHomePath(mount.host);
+  let stat;
+  try {
+    stat = fs.statSync(hostPath);
+  } catch {
+    return;
+  }
+  if (hostPath.endsWith('config') && !stat.isFile()) return;
+
+  plan.mountedHosts.push(hostPath);
+  const mountSpec = mount.readonly
+    ? `${hostPath}:${mount.container}:ro`
+    : `${hostPath}:${mount.container}`;
+  plan.args.push('-v', mountSpec);
+  if (explicitContainerPaths.has(mount.container)) {
+    plan.explicitMountContainerPaths.push(mount.container);
+  }
+}
+
 function pathContains(base, target) {
   const resolvedBase = path.resolve(base);
   const resolvedTarget = path.resolve(target);
@@ -325,7 +354,6 @@ class IsolationManager {
     this._removeContainerByName(containerName);
 
     workDir = await this._prepareIsolatedWorkspace(clusterId, workDir, reuseExisting);
-
     // The cluster config dir carries Claude credentials (via provisionClaudeCredentials) and the
     // Claude-specific AskUserQuestion-blocking hook (~/.claude/settings.json PreToolUse), which
     // only the `claude` CLI reads. Creating and mounting it for every provider — regardless of
@@ -534,33 +562,7 @@ class IsolationManager {
     const claudeContainerPath = path.posix.join(containerHome, '.claude');
 
     for (const mount of mounts) {
-      if (mount.container === claudeContainerPath) {
-        console.warn(
-          `[IsolationManager] Skipping mount for ${mount.host} -> ${mount.container} ` +
-            '(Claude config is managed by zeroshot).'
-        );
-        continue;
-      }
-
-      const hostPath = expandHomePath(mount.host);
-
-      try {
-        const stat = fs.statSync(hostPath);
-        if (hostPath.endsWith('config') && !stat.isFile()) {
-          continue;
-        }
-      } catch {
-        continue;
-      }
-
-      const mountSpec = mount.readonly
-        ? `${hostPath}:${mount.container}:ro`
-        : `${hostPath}:${mount.container}`;
-      plan.args.push('-v', mountSpec);
-      plan.mountedHosts.push(hostPath);
-      if (explicitContainerPaths.has(mount.container)) {
-        plan.explicitMountContainerPaths.push(mount.container);
-      }
+      appendCredentialMount(plan, mount, explicitContainerPaths, claudeContainerPath);
     }
 
     const { envToPass, explicitNames } = this._collectDockerEnvVars(
@@ -568,6 +570,16 @@ class IsolationManager {
       userMountConfig,
       settings
     );
+    const hostOnlyEnvs = new Set(
+      mountConfig
+        .filter((preset) => typeof preset === 'string')
+        .map((preset) => MOUNT_PRESETS[preset]?.hostEnv)
+        .filter((name) => typeof name === 'string')
+    );
+    for (const hostOnlyEnv of hostOnlyEnvs) {
+      delete envToPass[hostOnlyEnv];
+      explicitNames.delete(hostOnlyEnv);
+    }
     for (const [key, value] of Object.entries(envToPass)) {
       plan.args.push('-e', `${key}=${value}`);
       plan.forwardedEnv[key] = value;
@@ -680,7 +692,9 @@ class IsolationManager {
 
     // A mount only counts if it carries the secret (credentialInMount !== false).
     const credentialPaths = provider.getCredentialPaths ? provider.getCredentialPaths() : [];
-    const expandedCreds = credentialPaths.map((cred) => expandHomePath(cred));
+    const expandedCreds = resolveProviderCredentialPaths({ ...metadata, credentialPaths }).filter(
+      (credPath) => fs.existsSync(credPath)
+    );
     const hasCredentialMount =
       docker.credentialInMount !== false &&
       mountedHosts.some((hostPath) =>

package/src/orchestrator.js

@@ -111,6 +111,35 @@ function applyModelOverride(agentConfig, modelOverride) {
  */
 const VALID_OPERATIONS = ['add_agents', 'remove_agents', 'update_agent', 'publish', 'load_config'];
 
+function isValidPreparedWorktreeDescriptor(prepared, cwd) {
+  if (!prepared) return false;
+  if (typeof prepared !== 'object') return false;
+  if (path.resolve(String(prepared.path)) !== cwd) return false;
+  if (path.resolve(String(prepared.repoRoot)) !== cwd) return false;
+  if (typeof prepared.branch !== 'string') return false;
+  if (prepared.branch.length === 0) return false;
+  if (prepared.branch.length > 255) return false;
+  return /^[0-9a-f]{40}$/.test(String(prepared.baseSha));
+}
+
+function resolvePreparedWorktree(options) {
+  const prepared = options.preparedWorktree;
+  if (!prepared) return null;
+  const cwd = path.resolve(options.cwd || process.cwd());
+  if (options.worktree !== true || options.isolation) {
+    throw new Error('Invalid prepared worktree authority');
+  }
+  if (!isValidPreparedWorktreeDescriptor(prepared, cwd)) {
+    throw new Error('Invalid prepared worktree authority');
+  }
+  return Object.freeze({
+    path: cwd,
+    repoRoot: cwd,
+    branch: prepared.branch,
+    baseSha: prepared.baseSha,
+  });
+}
+
 /**
  * Workflow-triggering topics that indicate cluster state progression
  * These are the topics that MATTER for resume - not AGENT_OUTPUT noise
@@ -1854,7 +1883,10 @@ class Orchestrator {
     let worktreeInfo = null;
     let isolationImage = null;
 
-    if (options.isolation) {
+    if (options.preparedWorktree) {
+      worktreeInfo = resolvePreparedWorktree(options);
+      this._log(`[Orchestrator] Using prepared worktree: ${worktreeInfo.path}`);
+    } else if (options.isolation) {
       // Resolve the provider first so the capability gate and the cluster's image variant (which
       // installs the provider CLI as a Docker-cached layer) both see the same provider. Providers
       // baked into the base image (e.g. Claude) resolve back to the base image unchanged.

package/task-lib/runner.js

@@ -625,6 +625,13 @@ export function shouldUseAttachableWatcher(options, providerName) {
     return false;
   }
 
+  // Benchmark runs are non-interactive and keep the cluster process in the foreground. Use the
+  // pipe watcher so task completion is observed only after stdout/stderr close; a PTY exit can
+  // race its final buffered output on remote runtimes and lose the terminal structured result.
+  if (resolveTaskExecutionContext() === 'benchmark') {
+    return false;
+  }
+
   // The rpc-stdio lane owns bidirectional correlated RPC over stdio itself (see
   // omp-rpc-driver.ts) and always uses rpc-watcher.js instead of the attachable PTY watcher.
   if (getProviderRegistryEntry(providerName).invoke.lane === 'rpc-stdio') {

package/task-lib/watcher-output-runtime.js

@@ -103,7 +103,12 @@ function createCodexOutputPassthrough({ log, captureProviderSession }) {
   };
 }
 
-function createBoundedLinePassthrough({ log, handleLine, deferRawUntilOverflow = false }) {
+function createBoundedLinePassthrough({
+  log,
+  handleLine,
+  deferRawUntilOverflow = false,
+  linePrefix = '',
+}) {
   const decoder = new StringDecoder('utf8');
   let atLineStart = true;
   let lineTimestamp = null;
@@ -160,7 +165,7 @@ function createBoundedLinePassthrough({ log, handleLine, deferRawUntilOverflow =
     while (offset < text.length) {
       if (atLineStart) {
         lineTimestamp = Date.now();
-        appendRaw(logged, `[${lineTimestamp}]`);
+        appendRaw(logged, `[${lineTimestamp}]${linePrefix}`);
         atLineStart = false;
       }
       const newline = text.indexOf('\n', offset);
@@ -361,6 +366,10 @@ export function createWatcherOutputRuntime({
       });
   const stderrPassthrough = createBoundedLinePassthrough({
     log,
+    // Pi reserves stdout for its JSON protocol and deliberately routes ordinary writes to
+    // stderr. Keep those diagnostics in the task log with explicit provenance so followers can
+    // exclude them from strict JSON validation.
+    linePrefix: providerName === 'pi' ? '[ZEROSHOT][PROVIDER_STDERR] ' : '',
     handleLine: (line, timestamp) => maybeHandleFatalError(line, timestamp),
   });
 
@@ -370,7 +379,7 @@ export function createWatcherOutputRuntime({
     if (!detected) return false;
     fatalError = detected;
     if (silentJsonMode) log(`[${timestamp}]${line}\n`);
-    log(`[${timestamp}][FATAL] ${detected}\n`);
+    log(`[${timestamp}][ZEROSHOT][FATAL] ${detected}\n`);
     stopProvider(timestamp);
     return true;
   }
@@ -396,13 +405,15 @@ export function createWatcherOutputRuntime({
       fatalError =
         `Provider structured output exceeded the ${MAX_WATCHER_CONTROL_RECORD_BYTES}-byte ` +
         'watcher inspection limit; complete output remains in the task log';
-      log(`[${timestamp}][FATAL] ${fatalError}\n`);
+      log(`[${timestamp}][ZEROSHOT][FATAL] ${fatalError}\n`);
       stopProvider(timestamp);
       return;
     }
     captureProviderSession(line);
     if (silentJsonMode && !line.trim()) return;
-    maybeHandleFatalError(line, timestamp);
+    // Pi reserves stdout for JSON lifecycle events. Error text inside an assistant message may be
+    // followed by an automatic retry, so only Pi stderr can prove a pre-agent startup failure.
+    if (providerName !== 'pi') maybeHandleFatalError(line, timestamp);
     if (captureStreamingError(line, timestamp)) return;
     if (silentJsonMode) {
       maybeCaptureStructuredOutput(line);