@the-open-engine/zeroshot 6.34.2 → 6.35.0

package/src/agent/provider-terminal-failure.js

@@ -0,0 +1,186 @@
+// @ts-nocheck
+
+const { getProvider } = require('../providers');
+const { extractCliFailure } = require('./output-extraction');
+
+function categoryForProviderFailure(error, classification) {
+  const isPermanent = classification.retryable === false;
+  const authenticationPattern =
+    /(?:invalid[_ -]?api[_ -]?key|api[_ -]?key.*invalid|unauthori[sz]ed|forbidden|authentication|permission denied)/i;
+  if (isPermanent && authenticationPattern.test(error)) return 'authentication';
+
+  const quotaPattern = /(?:insufficient[_ -]?quota|quota exceeded|resource_exhausted)/i;
+  if (isPermanent && quotaPattern.test(error)) return 'quota';
+  if (isPermanent) return 'permanent';
+  return classification.kind === 'unknown-retryable' ? 'unknown' : 'transient';
+}
+
+function classifyProviderFailure(providerName, error) {
+  let rawClassification = { retryable: true, kind: 'unknown-retryable' };
+  try {
+    rawClassification = getProvider(providerName).adapter.classifyError(new Error(error));
+  } catch {
+    // Extraction remains available if a provider adapter cannot be loaded.
+  }
+  return {
+    retryable: rawClassification?.retryable !== false,
+    kind:
+      typeof rawClassification?.kind === 'string' ? rawClassification.kind : 'unknown-retryable',
+  };
+}
+
+function extractProviderFailure(output, providerName) {
+  const cliError = extractCliFailure(output, providerName);
+  if (!cliError) return null;
+
+  const classification = classifyProviderFailure(providerName, cliError.error);
+  const category = categoryForProviderFailure(cliError.error, classification);
+  return {
+    error: `Provider ${cliError.provider} failed (${category}; ${classification.kind})`,
+    provider: cliError.provider,
+    event: cliError.provider === 'codex' ? 'turn.failed' : 'terminal_error',
+    category,
+    classification,
+    diagnostic: cliError.diagnostic,
+  };
+}
+
+function redactTerminalFailureForControlPlane(state, providerName, content) {
+  const failure = extractProviderFailure(content, providerName);
+  if (!failure) return content;
+
+  state.providerFailure = failure;
+  let eventType = 'provider.failure';
+  try {
+    const parsed = JSON.parse(content);
+    if (typeof parsed?.type === 'string') eventType = parsed.type;
+  } catch {
+    // extractProviderFailure already proved a supported terminal envelope.
+  }
+  return JSON.stringify({
+    type: eventType,
+    ...(providerName === 'claude' ? { is_error: true } : {}),
+    ...(providerName === 'gemini' ? { status: 'error', severity: 'error' } : {}),
+    error: { message: failure.error },
+    zeroshot_failure: {
+      provider: failure.provider,
+      event: failure.event,
+      category: failure.category,
+      kind: failure.classification.kind,
+      retryable: failure.classification.retryable,
+      diagnostic: failure.diagnostic,
+    },
+  });
+}
+
+function decorateError(error, failure) {
+  if (!failure) return error;
+  error.provider = failure.provider || null;
+  error.providerEvent = failure.event || null;
+  error.providerCategory = failure.category || null;
+  error.classification = failure.classification || null;
+  error.providerDiagnostic = failure.diagnostic || null;
+  if (failure.classification?.retryable === false) error.permanent = true;
+  return error;
+}
+
+function receiptFields(error) {
+  if (!error?.provider) return {};
+  return {
+    provider: error.provider,
+    event: error.providerEvent,
+    category: error.providerCategory,
+    kind: error.classification?.kind,
+    retryable: error.classification?.retryable,
+    diagnostic: error.providerDiagnostic,
+  };
+}
+
+function workerFailure(error) {
+  const authenticationFailure =
+    error?.provider &&
+    error?.classification?.retryable === false &&
+    error?.providerCategory === 'authentication';
+  return authenticationFailure
+    ? { code: 'refusal', reason: 'authentication_required' }
+    : { code: 'crash', reason: 'declared_failure' };
+}
+
+function publishCriticalFailure({
+  agent,
+  error,
+  attempts,
+  worker,
+  unsupportedCapability,
+  structuredOutputInvalid,
+}) {
+  const specific =
+    error?.hookFailure ||
+    structuredOutputInvalid ||
+    unsupportedCapability ||
+    error?.vertexModelError ||
+    error?.terminationExhausted;
+  const critical =
+    agent.role === 'implementation' ||
+    agent.role === 'coordinator' ||
+    agent.id === 'consensus-coordinator';
+  if (!critical || specific) return worker;
+
+  agent._publish({
+    topic: 'CLUSTER_FAILED',
+    receiver: 'broadcast',
+    content: {
+      text: `Critical agent ${agent.id} exhausted its retry budget`,
+      data: {
+        reason: error?.provider ? 'provider_execution_failed' : 'critical_agent_exhausted',
+        agentId: agent.id,
+        role: agent.role,
+        attempts,
+        code: worker.code,
+        workerReason: worker.reason,
+        ...receiptFields(error),
+      },
+    },
+  });
+  return worker;
+}
+
+function buildFinalFailureInfo({
+  agent,
+  error,
+  attempts,
+  worker,
+  unsupportedCapability,
+  structuredOutputInvalid,
+}) {
+  return {
+    ...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
+    agentId: agent.id,
+    taskId: error?.taskId || agent.currentTaskId,
+    iteration: agent.iteration,
+    error: error.message,
+    attempts,
+    ...receiptFields(error),
+    ...(error?.provider ? { code: worker.code, workerReason: worker.reason } : {}),
+    ...(unsupportedCapability
+      ? {
+          code: error.code,
+          permanent: true,
+          provider: error.provider,
+          capability: error.capability,
+        }
+      : {}),
+    ...(structuredOutputInvalid ? { code: error.code, details: error.details ?? null } : {}),
+    timestamp: Date.now(),
+  };
+}
+
+module.exports = {
+  buildFinalFailureInfo,
+  decorateError,
+  extractProviderFailure,
+  publishCriticalFailure,
+  receiptFields,
+  redactTerminalFailureForControlPlane,
+  workerFailure,
+};

package/src/agent-cli-provider/adapters/gateway.ts

@@ -33,6 +33,7 @@ export const gatewaySettingsDefaults: Readonly<Record<string, unknown>> = Object
   protocol: 'openai',
   baseUrl: null,
   apiKey: null,
+  apiKeyEnv: null,
   headers: null,
   model: null,
   maxTokens: null,

package/src/agent-cli-provider/gateway-tools.ts

@@ -52,6 +52,7 @@ export function normalizeGatewayBuildOptions(
   const protocol = optionalGatewayProtocol(value.protocol, `${field}.protocol`);
   const baseUrl = optionalString(value.baseUrl, `${field}.baseUrl`);
   const apiKey = optionalString(value.apiKey, `${field}.apiKey`);
+  const apiKeyEnv = optionalEnvironmentName(value.apiKeyEnv, `${field}.apiKeyEnv`);
   const model = optionalNullableString(value.model, `${field}.model`);
   const maxTokens = optionalNullablePositiveInteger(value.maxTokens, `${field}.maxTokens`);
   const headers = optionalStringRecord(value.headers, `${field}.headers`);
@@ -60,6 +61,13 @@ export function normalizeGatewayBuildOptions(
   if (protocol !== undefined) result.protocol = protocol;
   if (baseUrl !== undefined) result.baseUrl = baseUrl;
   if (typeof apiKey === 'string') result.apiKey = apiKey;
+  if (apiKeyEnv !== undefined) result.apiKeyEnv = apiKeyEnv;
+  if (apiKey !== undefined && apiKeyEnv !== undefined) {
+    invalidField(
+      `${field}.apiKeyEnv`,
+      `${field}.apiKey and ${field}.apiKeyEnv are mutually exclusive.`
+    );
+  }
   if (model !== undefined) result.model = model;
   if (maxTokens !== undefined) result.maxTokens = maxTokens;
   if (headers !== undefined) result.headers = headers;
@@ -115,6 +123,18 @@ export function validateGatewaySettings(settings: Record<string, unknown>): stri
     );
     optionalString(settings.baseUrl, 'providerSettings.gateway.baseUrl');
     optionalString(settings.apiKey, 'providerSettings.gateway.apiKey');
+    optionalEnvironmentName(settings.apiKeyEnv, 'providerSettings.gateway.apiKeyEnv');
+    if (
+      settings.apiKey !== null &&
+      settings.apiKey !== undefined &&
+      settings.apiKeyEnv !== null &&
+      settings.apiKeyEnv !== undefined
+    ) {
+      invalidField(
+        'providerSettings.gateway.apiKeyEnv',
+        'providerSettings.gateway.apiKey and providerSettings.gateway.apiKeyEnv are mutually exclusive.'
+      );
+    }
     optionalNullableString(settings.model, 'providerSettings.gateway.model');
     const maxTokens = optionalNullablePositiveInteger(
       settings.maxTokens,
@@ -138,6 +158,14 @@ export function validateGatewaySettings(settings: Record<string, unknown>): stri
   }
 }
 
+function optionalEnvironmentName(value: unknown, field: string): string | undefined {
+  const name = optionalString(value, field);
+  if (name !== undefined && !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+    invalidField(field, `${field} must name an environment variable.`);
+  }
+  return name;
+}
+
 export function normalizeGatewayToolPolicy(
   value: GatewayToolPolicy,
   field: string,
@@ -212,7 +240,11 @@ function requiredGatewayToolPolicy(
   return normalizeGatewayToolPolicy(value, field, cwd);
 }
 
-function normalizeGatewayRoots(value: readonly string[], field: string, cwd: string): readonly string[] {
+function normalizeGatewayRoots(
+  value: readonly string[],
+  field: string,
+  cwd: string
+): readonly string[] {
   if (value.length === 0) {
     invalidField(field, `${field} must contain at least one root path.`);
   }
@@ -368,7 +400,11 @@ async function readGatewayFile(
   input: unknown,
   policy: GatewayToolPolicy
 ): Promise<Record<string, unknown>> {
-  const targetPath = await assertWithinRoots(readGatewayFileInput(input), policy.roots, 'read_file.path');
+  const targetPath = await assertWithinRoots(
+    readGatewayFileInput(input),
+    policy.roots,
+    'read_file.path'
+  );
   const content = await readFile(targetPath, 'utf8');
   return { path: targetPath, content };
 }
@@ -420,7 +456,9 @@ async function applyGatewayPatch(
   if (!current.includes(search)) {
     throw new Error('apply_patch.search did not match the target file.');
   }
-  const next = request.replaceAll ? current.split(search).join(replace) : current.replace(search, replace);
+  const next = request.replaceAll
+    ? current.split(search).join(replace)
+    : current.replace(search, replace);
   await writeFile(targetPath, next, 'utf8');
   return {
     path: targetPath,

package/src/agent-cli-provider/provider-registry.ts

@@ -316,13 +316,15 @@ export const providerRegistry = [
     invoke: SPAWN_INVOKE,
     installInstructions: 'Bundled with Zeroshot; no external provider CLI install is required.',
     authInstructions:
-      'Configure providerSettings.gateway.protocol, baseUrl, apiKey, model, maxTokens when required, and toolPolicy in Zeroshot settings.',
+      'Configure providerSettings.gateway protocol, base URL, apiKey or apiKeyEnv, model, ' +
+      'maxTokens, and toolPolicy in Zeroshot settings.',
     credentialPaths: [],
     credentialEnvKeys: gatewayAdapter.credentialEnvKeys,
     settingsFields: [
       'protocol',
       'baseUrl',
       'apiKey',
+      'apiKeyEnv',
       'headers',
       'model',
       'maxTokens',

package/src/agent-cli-provider/single-agent-runtime.ts

@@ -766,6 +766,7 @@ function resolveRuntimeGatewayOptions(
   const cwd = baseOptions.cwd ?? process.cwd();
   const settingsGateway = providerSettings.gateway ?? {};
   const requestGateway = baseOptions.gateway ?? {};
+  const apiKey = resolveGatewayApiKey(requestGateway, settingsGateway);
   const mergedHeaders =
     requestGateway.headers === undefined
       ? settingsGateway.headers
@@ -777,9 +778,7 @@ function resolveRuntimeGatewayOptions(
     ...((requestGateway.baseUrl ?? settingsGateway.baseUrl)
       ? { baseUrl: requestGateway.baseUrl ?? settingsGateway.baseUrl }
       : {}),
-    ...((requestGateway.apiKey ?? settingsGateway.apiKey)
-      ? { apiKey: requestGateway.apiKey ?? settingsGateway.apiKey }
-      : {}),
+    ...(apiKey === undefined ? {} : { apiKey }),
     ...(mergedHeaders === undefined ? {} : { headers: mergedHeaders }),
     model: requestGateway.model ?? modelSpec.model ?? settingsGateway.model ?? null,
     ...((requestGateway.maxTokens ?? settingsGateway.maxTokens)
@@ -792,6 +791,34 @@ function resolveRuntimeGatewayOptions(
   return resolveGatewayConfiguration(mergedGateway, 'options.gateway', cwd);
 }
 
+function resolveGatewayApiKey(
+  request: GatewayBuildOptions,
+  settings: GatewayBuildOptions
+): string | undefined {
+  if (request.apiKeyEnv !== undefined) {
+    throw new Error('options.gateway.apiKeyEnv requires trusted provider settings');
+  }
+  const requestOwnsTransport =
+    request.protocol !== undefined ||
+    request.baseUrl !== undefined ||
+    request.headers !== undefined;
+  if ((settings.apiKeyEnv !== undefined || settings.apiKey !== undefined) && requestOwnsTransport) {
+    throw new Error('options.gateway transport cannot override credential settings');
+  }
+  const direct = request.apiKey ?? settings.apiKey;
+  const environmentName = settings.apiKeyEnv;
+  if (direct !== undefined && environmentName !== undefined) {
+    throw new Error('gateway apiKey and apiKeyEnv are mutually exclusive');
+  }
+  if (direct !== undefined) return direct;
+  if (environmentName === undefined) return undefined;
+  const value = process.env[environmentName];
+  if (typeof value !== 'string' || !value.trim()) {
+    throw new Error(`gateway apiKeyEnv requires environment variable ${environmentName}`);
+  }
+  return value;
+}
+
 function shouldIncludeAuthEnv(
   baseOptions: BuildProviderCommandOptions,
   authEnv: Readonly<Record<string, string>>

package/src/agent-cli-provider/types.ts

@@ -74,6 +74,7 @@ export interface GatewayBuildOptions {
   readonly protocol?: GatewayProtocol;
   readonly baseUrl?: string;
   readonly apiKey?: string;
+  readonly apiKeyEnv?: string;
   readonly headers?: Readonly<Record<string, string>>;
   readonly model?: string | null;
   readonly maxTokens?: number;

package/src/orchestrator.js

@@ -275,6 +275,7 @@ class Orchestrator {
     // Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
     this.closed = false;
     this._conductorWatchdogs = new Set();
+    this._clusterRunBoundaries = new Map();
 
     // Track if clusters are loaded (for lazy loading pattern)
     this._clustersLoaded = options.skipLoad === true;
@@ -1575,11 +1576,17 @@ class Orchestrator {
   }
 
   _registerAgentErrorHandler(messageBus, clusterId) {
+    this._recordClusterRunBoundary(messageBus, clusterId);
     this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
       const agentRole = message.content?.data?.role;
       const attempts = message.content?.data?.attempts || 1;
       const hookFailure = message.content?.data?.hookFailure === true;
       const restartExhausted = message.content?.data?.restartExhausted === true;
+      const durableClusterFailure = this._findCurrentRunClusterFailure(
+        messageBus,
+        clusterId,
+        message.sequence
+      );
 
       await this._saveClusters();
 
@@ -1587,7 +1594,10 @@ class Orchestrator {
         agentRole === 'implementation' ||
         agentRole === 'coordinator' ||
         message.sender === 'consensus-coordinator';
-      const shouldStop = shouldStopForRole && (hookFailure || restartExhausted || attempts >= 3);
+      const shouldStop =
+        !durableClusterFailure &&
+        shouldStopForRole &&
+        (hookFailure || restartExhausted || attempts >= 3);
 
       if (shouldStop) {
         this._log(`\n${'='.repeat(80)}`);
@@ -1610,6 +1620,25 @@ class Orchestrator {
     });
   }
 
+  _recordClusterRunBoundary(messageBus, clusterId) {
+    const latest = messageBus.findLast({ cluster_id: clusterId, orderBySequence: true });
+    this._clusterRunBoundaries.set(clusterId, latest?.sequence ?? null);
+  }
+
+  _findCurrentRunClusterFailure(messageBus, clusterId, throughId) {
+    const boundary = this._clusterRunBoundaries.get(clusterId);
+    return (
+      messageBus.query({
+        cluster_id: clusterId,
+        topic: 'CLUSTER_FAILED',
+        order: 'desc',
+        limit: 1,
+        ...(boundary === null ? {} : { afterId: boundary }),
+        ...(throughId === undefined ? {} : { throughId }),
+      })[0] || null
+    );
+  }
+
   _registerPushBlockedHandler(messageBus, clusterId) {
     this._subscribeToClusterTopic(messageBus, clusterId, 'PUSH_BLOCKED', async (message) => {
       const reason = message.content?.data?.blocked_reason || 'unknown';
@@ -2425,6 +2454,7 @@ class Orchestrator {
 
     // Now remove from memory after persisting
     this.clusters.delete(clusterId);
+    this._clusterRunBoundaries.delete(clusterId);
   }
 
   /**
@@ -2462,6 +2492,7 @@ class Orchestrator {
       watchdog.dispose();
     }
     this._conductorWatchdogs.clear();
+    this._clusterRunBoundaries.clear();
 
     for (const cluster of this.clusters.values()) {
       if (typeof cluster.snapshotter?.stop === 'function') {
@@ -2902,6 +2933,7 @@ class Orchestrator {
   }
 
   async _restartClusterAgents(cluster) {
+    this._recordClusterRunBoundary(cluster.messageBus, cluster.id);
     cluster.state = 'running';
     cluster.pid = process.pid;
     for (const agent of cluster.agents) {