@archal/cli 0.7.12 → 0.8.0

package/dist/harnesses/react/agent.mjs

@@ -0,0 +1,420 @@
+/**
+ * ReAct Agent — the "good" bundled harness.
+ *
+ * Full ReAct (Reason + Act) loop with:
+ * - Multi-provider support (Gemini, OpenAI, Anthropic)
+ * - Structured system prompt encouraging step-by-step reasoning
+ * - Error recovery with retries on transient failures
+ * - Context-aware done detection
+ * - Configurable step limit (default 80, cap 200 via ARCHAL_MAX_STEPS)
+ * - Token usage and timing instrumentation
+ *
+ * Env vars (set by archal orchestrator):
+ *   ARCHAL_ENGINE_TASK   — the scenario task to complete
+ *   ARCHAL_ENGINE_MODEL  — model identifier (e.g. gemini-2.0-flash)
+ *   ARCHAL_<TWIN>_URL    — twin REST base URL (per twin)
+ *   ARCHAL_ENGINE_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY / ANTHROPIC_API_KEY
+ */
+import { collectTwinUrls, discoverAllTools, callToolRest } from '../_lib/rest-client.mjs';
+import {
+  detectProvider,
+  resolveApiKey,
+  formatToolsForProvider,
+  buildInitialMessages,
+  appendAssistantResponse,
+  appendToolResults,
+  appendUserInstruction,
+  callLlmWithMessages,
+  parseToolCalls,
+  getResponseText,
+  getThinkingContent,
+  getStopReason,
+  withRetry,
+} from '../_lib/providers.mjs';
+import { createLogger } from '../_lib/logging.mjs';
+import { writeMetrics } from '../_lib/metrics.mjs';
+import { createAgentTrace } from '../_lib/agent-trace.mjs';
+import { classifyTask, selectStepTools } from './tool-selection.mjs';
+
+const DEFAULT_MAX_STEPS = 80;
+const MAX_STEPS = (() => {
+  const raw = process.env['ARCHAL_MAX_STEPS']?.trim();
+  if (!raw) return DEFAULT_MAX_STEPS;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed) || parsed <= 0) return DEFAULT_MAX_STEPS;
+  return Math.min(parsed, 200);
+})();
+const MAX_CONSECUTIVE_ERRORS = (() => {
+  const raw = process.env['ARCHAL_MAX_CONSECUTIVE_ERRORS']?.trim();
+  if (!raw) return 8;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed) || parsed <= 0) return 8;
+  return Math.min(parsed, 20);
+})();
+const MAX_INITIAL_NO_TOOL_RECOVERIES = (() => {
+  const raw = process.env['ARCHAL_MAX_INITIAL_NO_TOOL_RECOVERIES']?.trim();
+  if (!raw) return 2;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed) || parsed <= 0) return 2;
+  return Math.min(parsed, 5);
+})();
+const TASK = (process.env['ARCHAL_ENGINE_TASK'] || '').trim();
+const MODEL = process.env['ARCHAL_ENGINE_MODEL'];
+const TASK_LOWER = TASK.toLowerCase();
+
+if (!TASK) { console.error('ARCHAL_ENGINE_TASK not set or empty'); process.exit(1); }
+if (!MODEL) { console.error('ARCHAL_ENGINE_MODEL not set'); process.exit(1); }
+
+const provider = detectProvider(MODEL);
+const apiKey = resolveApiKey(provider);
+const log = createLogger({ harness: 'react', model: MODEL, provider });
+const TASK_FLAGS = classifyTask(TASK);
+
+const SYSTEM_PROMPT = `You are a capable AI agent performing a task using tools. Think step by step.
+
+APPROACH:
+1. Before each action, briefly reason about what you need to do next and why.
+2. Use the available tools to gather information and take actions.
+3. If a tool call fails, analyze the error and try an alternative approach.
+4. When you have completed the task, stop calling tools.
+
+GUIDELINES:
+- Read the task carefully and identify all requirements before starting.
+- Work methodically — complete one sub-task before moving to the next.
+- Pay attention to tool output — it contains the information you need.
+- If you're unsure about something, gather more information first.
+- Do NOT repeat the same failed tool call — try a different approach.
+- Do not create new entities unless the task explicitly asks for creation.
+- Do not create or edit repository files as a substitute for issue, ticket, label, or message updates.
+- If the task spans multiple systems, do not stop after the first system mutation. Complete the required follow-up in every mentioned system.
+- When done, provide a brief summary of what you accomplished.`;
+
+const MUTATING_TOOL_NAME = /(?:^|_)(create|update|add|post|reply|delete|close|merge|approve|archive|send)(?:_|$)/i;
+const REPO_CONTENT_MUTATION_TOOL = /(?:^|_)(create_or_update_file|delete_file|create_branch|create_commit)(?:_|$)/i;
+const CREATE_ISSUE_TOOL = /(?:^|_)create_issue(?:_|$)/i;
+const TASK_ALLOWS_REPO_CONTENT_MUTATION = /\b(file|files|code|commit|branch|pull request|pull requests|pr|readme|source|implementation|repository)\b/i.test(TASK_LOWER);
+
+function isMutatingToolName(toolName) {
+  return MUTATING_TOOL_NAME.test(toolName);
+}
+
+function isRepoContentMutationTool(toolName) {
+  return REPO_CONTENT_MUTATION_TOOL.test(toolName);
+}
+
+function isCreateIssueTool(toolName) {
+  return CREATE_ISSUE_TOOL.test(toolName);
+}
+
+// ── Twin REST transport ─────────────────────────────────────────────
+const twinUrls = collectTwinUrls();
+const knownTwinNames = new Set(Object.keys(twinUrls));
+if (Object.keys(twinUrls).length === 0) {
+  console.error('[react] No twin URLs found. Check ARCHAL_TWIN_NAMES and ARCHAL_<TWIN>_URL env vars.');
+  process.exit(1);
+}
+const { tools: allTools, toolToTwin } = await discoverAllTools(twinUrls);
+if (allTools.length === 0) {
+  console.error('[react] No tools discovered from twins. Twin endpoints may be unreachable.');
+  process.exit(1);
+}
+
+let messages = buildInitialMessages(provider, SYSTEM_PROMPT, TASK, MODEL);
+if (TASK_FLAGS.isExistingIssueTriage) {
+  messages = appendUserInstruction(
+    provider,
+    messages,
+    'This task is issue triage on the existing repository issues. Update those issues in place. ' +
+      'Do not use comments, files, or duplicate issues as a substitute for labels. ' +
+      'If the task asks you to prioritize bug reports, every bug issue must also receive an appropriate priority label. ' +
+      'Use the repository priority labels exactly as named: priority:high, priority:medium, or priority:low.',
+  );
+}
+let consecutiveErrors = 0;
+
+const runStart = Date.now();
+let totalInputTokens = 0;
+let totalOutputTokens = 0;
+let totalToolCalls = 0;
+let totalToolErrors = 0;
+let stepsCompleted = 0;
+let exitReason = 'max_steps';
+let initialNoToolRecoveries = 0;
+let repoContentGuardRecoveries = 0;
+let pendingFollowupTwins = null;
+const updatedTwins = new Set();
+const agentTrace = createAgentTrace();
+
+log.info('run_start', { task: TASK.slice(0, 200), maxSteps: MAX_STEPS });
+
+try {
+  for (let step = 0; step < MAX_STEPS; step++) {
+    stepsCompleted = step + 1;
+    const iterStart = Date.now();
+    const stepTools = selectStepTools(allTools, TASK_FLAGS, toolToTwin, pendingFollowupTwins);
+    const providerTools = formatToolsForProvider(provider, stepTools);
+
+    // Call the LLM with retry on transient errors
+    log.llmCall(step + 1);
+    let response;
+    try {
+      response = await withRetry(
+        () => callLlmWithMessages(provider, MODEL, apiKey, messages, providerTools),
+        4,
+      );
+    } catch (err) {
+      const msg = err?.message ?? String(err);
+      log.error('llm_call_failed', { step: step + 1, error: msg });
+      process.stderr.write(`[react] LLM API error: ${msg.slice(0, 500)}\n`);
+      exitReason = 'llm_error';
+      break;
+    }
+
+    const iterDurationMs = Date.now() - iterStart;
+    totalInputTokens += response.usage.inputTokens;
+    totalOutputTokens += response.usage.outputTokens;
+
+    const hasToolCalls = !!parseToolCalls(provider, response);
+    const stopReason = getStopReason(provider, response);
+    log.llmResponse(step + 1, iterDurationMs, hasToolCalls, stopReason);
+    log.tokenUsage(step + 1, response.usage, {
+      inputTokens: totalInputTokens,
+      outputTokens: totalOutputTokens,
+    });
+
+    // Extract thinking/reasoning before appending (so we capture it before it's lost)
+    const thinking = getThinkingContent(provider, response);
+    const text = getResponseText(provider, response);
+
+    // Append assistant response to conversation
+    messages = appendAssistantResponse(provider, messages, response);
+
+    // Check for tool calls
+    const toolCalls = parseToolCalls(provider, response);
+
+    if (!toolCalls) {
+      // No tool calls — model is done or just providing text
+      agentTrace.addStep({ step: step + 1, thinking, text, toolCalls: [], durationMs: iterDurationMs });
+      if (text) {
+        process.stderr.write(`[react] Step ${step + 1}: ${text.slice(0, 200)}\n`);
+      }
+      const shouldRecoverInitialNoToolCall = totalToolCalls === 0
+        && initialNoToolRecoveries < MAX_INITIAL_NO_TOOL_RECOVERIES;
+      if (shouldRecoverInitialNoToolCall) {
+        initialNoToolRecoveries++;
+        messages = appendUserInstruction(
+          provider,
+          messages,
+          'You must use tools to make progress. ' +
+            'On your next response, call at least one relevant tool before giving any summary or conclusion. ' +
+            'Start by gathering concrete evidence from the systems, then execute the required actions.',
+        );
+        log.info('no_tool_calls_reprompt', {
+          step: step + 1,
+          attempt: initialNoToolRecoveries,
+        });
+        continue;
+      }
+      if (pendingFollowupTwins && pendingFollowupTwins.size > 0) {
+        const remainingTwins = [...pendingFollowupTwins].join(', ');
+        messages = appendUserInstruction(
+          provider,
+          messages,
+          `You have not finished the required follow-up in ${remainingTwins}. ` +
+            'Continue using the remaining system tools until those actions are complete before you conclude.',
+        );
+        log.info('cross_system_followup_reprompt', {
+          step: step + 1,
+          remainingTwins,
+        });
+        continue;
+      }
+      // If the model still avoids tools, we're done.
+      // Distinguish genuine startup no-tool failures from normal completion
+      // after the agent already used tools in earlier turns.
+      exitReason = totalToolCalls === 0 ? 'no_tool_calls' : 'completed';
+      break;
+    }
+    initialNoToolRecoveries = 0;
+
+    const proposedRepoContentMutation = toolCalls.some((tc) => isRepoContentMutationTool(tc.name));
+    if (proposedRepoContentMutation && (!TASK_ALLOWS_REPO_CONTENT_MUTATION || TASK_FLAGS.isExistingIssueTriage) && repoContentGuardRecoveries < 2) {
+      repoContentGuardRecoveries++;
+      agentTrace.addStep({
+        step: step + 1,
+        thinking,
+        text,
+        toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+        durationMs: iterDurationMs,
+      });
+      messages = appendToolResults(
+        provider,
+        messages,
+        toolCalls,
+        toolCalls.map(() =>
+          'Blocked by harness: this task must update the existing issue or message state directly, not repository files or commits.',
+        ),
+      );
+      messages = appendUserInstruction(
+        provider,
+        messages,
+        'This task is about updating existing issues/messages, not repository content. ' +
+          'Do not create or edit files or commits as a substitute for labels, issue state changes, or replies. ' +
+          'Use the issue or messaging mutation tools directly.',
+      );
+      log.info('repo_content_mutation_blocked', {
+        step: step + 1,
+        attemptedTools: toolCalls.map((tc) => tc.name),
+      });
+      continue;
+    }
+    if (TASK_FLAGS.isExistingIssueTriage && toolCalls.some((tc) => isCreateIssueTool(tc.name)) && repoContentGuardRecoveries < 2) {
+      repoContentGuardRecoveries++;
+      agentTrace.addStep({
+        step: step + 1,
+        thinking,
+        text,
+        toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+        durationMs: iterDurationMs,
+      });
+      messages = appendToolResults(
+        provider,
+        messages,
+        toolCalls,
+        toolCalls.map(() =>
+          'Blocked by harness: this task is to triage the existing issues in the repository, not create duplicate issues.',
+        ),
+      );
+      messages = appendUserInstruction(
+        provider,
+        messages,
+        'This task is to triage the existing issues that are already in the repository. ' +
+          'Do not create duplicate issues. Inspect the current issues and use the issue update tools to apply category labels and priority labels directly to those existing issues.',
+      );
+      log.info('issue_creation_blocked_for_triage', {
+        step: step + 1,
+        attemptedTools: toolCalls.map((tc) => tc.name),
+      });
+      continue;
+    }
+    // NOTE: Do NOT reset repoContentGuardRecoveries here. The counter must
+    // persist across the entire run so alternating clean/blocked steps cannot
+    // bypass the 2-attempt safety limit indefinitely.
+
+    // Execute each tool call via REST
+    const results = [];
+    const mutatedTwinsThisStep = new Set();
+    for (const tc of toolCalls) {
+      const toolStart = Date.now();
+      process.stderr.write(`[react] Step ${step + 1}: ${tc.name}(${JSON.stringify(tc.arguments).slice(0, 100)})\n`);
+      try {
+        const result = await callToolRest(toolToTwin, tc.name, tc.arguments);
+        results.push(result);
+        consecutiveErrors = 0;
+        totalToolCalls++;
+        if (isMutatingToolName(tc.name)) {
+          const twinName = toolToTwin[tc.name]?.twinName;
+          if (twinName) {
+            updatedTwins.add(twinName);
+            mutatedTwinsThisStep.add(twinName);
+          }
+        }
+        log.toolCall(step + 1, tc.name, tc.arguments, Date.now() - toolStart);
+      } catch (err) {
+        const errorMsg = `Error: ${err.message}`;
+        results.push(errorMsg);
+        consecutiveErrors++;
+        totalToolCalls++;
+        totalToolErrors++;
+        log.toolError(step + 1, tc.name, err.message);
+        process.stderr.write(`[react] Tool error (${consecutiveErrors}): ${err.message}\n`);
+
+        // Bail if too many consecutive errors
+        if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
+          process.stderr.write('[react] Too many consecutive tool errors — stopping.\n');
+          exitReason = 'consecutive_errors';
+          break;
+        }
+      }
+    }
+
+    // Record thinking trace for this step (before bailout check so the final step is captured)
+    agentTrace.addStep({
+      step: step + 1,
+      thinking,
+      text,
+      toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+      durationMs: iterDurationMs,
+    });
+
+    if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) break;
+
+    // Append tool results to conversation
+    messages = appendToolResults(provider, messages, toolCalls, results);
+
+    if (pendingFollowupTwins && pendingFollowupTwins.size > 0) {
+      const completedFollowups = [...mutatedTwinsThisStep].filter((twin) => pendingFollowupTwins.has(twin));
+      if (completedFollowups.length > 0) {
+        pendingFollowupTwins = null;
+      }
+    }
+
+    // Only trigger cross-system followup when the task actually involves
+    // multiple distinct services. Without this gate, single-system tasks
+    // running in a multi-twin configuration would incorrectly nag the
+    // agent to act in twins the task never mentions.
+    if (TASK_FLAGS.requiresCrossSystemFollowup && !pendingFollowupTwins && knownTwinNames.size > 1 && mutatedTwinsThisStep.size > 0) {
+      const untouchedTwins = [...knownTwinNames].filter((twinName) => !updatedTwins.has(twinName));
+      if (untouchedTwins.length > 0) {
+        pendingFollowupTwins = new Set(untouchedTwins);
+        messages = appendUserInstruction(
+          provider,
+          messages,
+          `You have updated ${[...updatedTwins].join(', ')} but not ${untouchedTwins.join(', ')}. ` +
+            'Continue and finish the remaining required actions in the untouched system before you conclude.',
+        );
+        log.info('cross_system_followup_required', {
+          step: step + 1,
+          updatedTwins: [...updatedTwins],
+          remainingTwins: untouchedTwins,
+        });
+      }
+    }
+  }
+} finally {
+  const totalTimeMs = Date.now() - runStart;
+
+  log.summary({
+    iterations: stepsCompleted,
+    totalInputTokens,
+    totalOutputTokens,
+    totalTimeMs,
+    toolCallCount: totalToolCalls,
+    toolErrorCount: totalToolErrors,
+    exitReason,
+  });
+
+  writeMetrics({
+    inputTokens: totalInputTokens,
+    outputTokens: totalOutputTokens,
+    llmCallCount: stepsCompleted,
+    toolCallCount: totalToolCalls,
+    toolErrorCount: totalToolErrors,
+    totalTimeMs,
+    exitReason,
+    provider,
+    model: MODEL,
+  });
+
+  agentTrace.flush();
+
+  process.stderr.write(
+    `\n[react] Summary: ${stepsCompleted} iterations, ${totalToolCalls} tool calls ` +
+    `(${totalToolErrors} errors), ${totalInputTokens} input tokens, ` +
+    `${totalOutputTokens} output tokens, ${(totalTimeMs / 1000).toFixed(1)}s total\n`
+  );
+
+  if (exitReason === 'llm_error') {
+    process.exit(1);
+  }
+}

package/dist/harnesses/react/archal-harness.json

@@ -0,0 +1,22 @@
+{
+  "version": 1,
+  "name": "react",
+  "description": "Full ReAct (Reason + Act) harness. Step-by-step reasoning prompt, error recovery with retries, consecutive-error bailout. Recommended for production evaluations.",
+  "local": {
+    "command": "node",
+    "args": ["agent.mjs"]
+  },
+  "maxSteps": 50,
+  "supportedProviders": ["openai", "anthropic", "gemini"],
+  "requiredEnvVars": [
+    "ARCHAL_ENGINE_TASK",
+    "ARCHAL_ENGINE_MODEL"
+  ],
+  "configDefaults": {
+    "maxSteps": 50,
+    "systemPrompt": true,
+    "errorHandling": true,
+    "retryOnTransient": true,
+    "maxConsecutiveErrors": 5
+  }
+}

package/dist/harnesses/react/tool-selection.mjs

@@ -0,0 +1,66 @@
+const ISSUE_TRIAGE_TOOL = /(?:^|_)(list_issues|get_issue|update_issue)(?:_|$)/i;
+const SLACK_CHANNEL_POST_TOOL = /(?:^|_)slack_post_message(?:_|$)/i;
+
+/**
+ * Patterns that identify distinct service domains in task text.
+ * Used to detect whether a task genuinely spans multiple systems.
+ */
+const SERVICE_DOMAIN_PATTERNS = [
+  { name: 'github', pattern: /\b(github|pull request|pr\s*#\d|merge|branch|commit|repository|repo)\b/i },
+  { name: 'slack', pattern: /\b(slack|#\w[\w-]*|channel|thread|post\s+(?:a\s+)?(?:message|summary|update))\b/i },
+  { name: 'linear', pattern: /\b(linear|[A-Z]{2,5}-\d+)\b/ },
+  { name: 'jira', pattern: /\b(jira|sprint|epic|story|CHG-\d+)\b/i },
+  { name: 'stripe', pattern: /\b(stripe|payment|charge|refund|invoice|subscription)\b/i },
+  { name: 'supabase', pattern: /\b(supabase|database|table|row|query|migration)\b/i },
+];
+
+function countMentionedServiceDomains(taskText) {
+  const matched = new Set();
+  for (const { name, pattern } of SERVICE_DOMAIN_PATTERNS) {
+    if (pattern.test(taskText)) matched.add(name);
+  }
+  return matched.size;
+}
+
+export function classifyTask(task) {
+  const taskLower = task.toLowerCase();
+  return {
+    taskLower,
+    isExistingIssueTriage: /\ball open issues?\b/.test(taskLower)
+      || (/\bissues?\b/.test(taskLower)
+        && /\b(triage|prioriti[sz]e|categor(?:ize|ization)|classif(?:y|ication))\b/.test(taskLower)),
+    requiresThreadReply: /\bthread\b/.test(taskLower)
+      && /\b(reply|replies|respond|post back)\b/.test(taskLower),
+    requiresCrossSystemFollowup: countMentionedServiceDomains(task) >= 2,
+  };
+}
+
+export function getToolsForTwins(tools, twinNames, toolToTwin) {
+  if (!twinNames || twinNames.size === 0) return tools;
+  return tools.filter((tool) => twinNames.has(toolToTwin[tool.name]?.twinName));
+}
+
+function canPerformIssueTriage(tools) {
+  return tools.some((tool) => ISSUE_TRIAGE_TOOL.test(tool.name));
+}
+
+export function filterToolsForTask(tools, taskFlags, { enforceIssueTriageAllowlist = true } = {}) {
+  let filtered = tools;
+  if (taskFlags.isExistingIssueTriage && enforceIssueTriageAllowlist) {
+    filtered = filtered.filter((tool) => ISSUE_TRIAGE_TOOL.test(tool.name));
+  }
+  if (taskFlags.requiresThreadReply) {
+    filtered = filtered.filter((tool) => !SLACK_CHANNEL_POST_TOOL.test(tool.name));
+  }
+  return filtered;
+}
+
+export function selectStepTools(tools, taskFlags, toolToTwin, pendingFollowupTwins) {
+  const twinScopedTools = getToolsForTwins(tools, pendingFollowupTwins, toolToTwin);
+  return filterToolsForTask(twinScopedTools, taskFlags, {
+    // Follow-up routing is the harder constraint. If the scoped twin cannot
+    // satisfy the generic issue-triage allowlist, keep its reply/mutation tools
+    // available so the agent can finish the required cross-system work.
+    enforceIssueTriageAllowlist: !taskFlags.isExistingIssueTriage || canPerformIssueTriage(twinScopedTools),
+  });
+}