@archal/cli 0.7.12 → 0.8.0

package/dist/harnesses/zero-shot/agent.mjs

@@ -0,0 +1,211 @@
+/**
+ * Zero-Shot Agent — the "medium" bundled harness.
+ *
+ * Sends the full task with all tools in one shot, minimal guidance.
+ * - Multi-provider support (Gemini, OpenAI, Anthropic)
+ * - Minimal system prompt — no reasoning encouragement
+ * - Basic error handling (log and continue, no retry)
+ * - Max 40 steps
+ *
+ * Env vars (set by archal orchestrator):
+ *   ARCHAL_ENGINE_TASK   — the scenario task to complete
+ *   ARCHAL_ENGINE_MODEL  — model identifier
+ *   ARCHAL_<TWIN>_URL    — twin REST base URL (per twin)
+ *   ARCHAL_ENGINE_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY / ANTHROPIC_API_KEY
+ */
+import { collectTwinUrls, discoverAllTools, callToolRest } from '../_lib/rest-client.mjs';
+import {
+  detectProvider,
+  resolveApiKey,
+  formatToolsForProvider,
+  buildInitialMessages,
+  appendAssistantResponse,
+  appendToolResults,
+  appendUserInstruction,
+  callLlmWithMessages,
+  parseToolCalls,
+  getResponseText,
+  getThinkingContent,
+  getStopReason,
+} from '../_lib/providers.mjs';
+import { createLogger } from '../_lib/logging.mjs';
+import { writeMetrics } from '../_lib/metrics.mjs';
+import { createAgentTrace } from '../_lib/agent-trace.mjs';
+
+const MAX_STEPS = 40;
+const MAX_INITIAL_NO_TOOL_RECOVERIES = (() => {
+  const raw = process.env['ARCHAL_MAX_INITIAL_NO_TOOL_RECOVERIES']?.trim();
+  if (!raw) return 2;
+  const parsed = parseInt(raw, 10);
+  if (Number.isNaN(parsed) || parsed <= 0) return 2;
+  return Math.min(parsed, 5);
+})();
+const TASK = (process.env['ARCHAL_ENGINE_TASK'] || '').trim();
+const MODEL = process.env['ARCHAL_ENGINE_MODEL'];
+
+if (!TASK) { console.error('ARCHAL_ENGINE_TASK not set or empty'); process.exit(1); }
+if (!MODEL) { console.error('ARCHAL_ENGINE_MODEL not set'); process.exit(1); }
+
+const provider = detectProvider(MODEL);
+const apiKey = resolveApiKey(provider);
+const log = createLogger({ harness: 'zero-shot', model: MODEL, provider });
+
+// Minimal system prompt — no reasoning guidance
+const SYSTEM_PROMPT = 'Complete the task. Use the tools provided.';
+
+// ── Twin REST transport ─────────────────────────────────────────────
+const twinUrls = collectTwinUrls();
+if (Object.keys(twinUrls).length === 0) {
+  console.error('[zero-shot] No twin URLs found. Check ARCHAL_TWIN_NAMES and ARCHAL_<TWIN>_URL env vars.');
+  process.exit(1);
+}
+const { tools: allTools, toolToTwin } = await discoverAllTools(twinUrls);
+if (allTools.length === 0) {
+  console.error('[zero-shot] No tools discovered from twins. Twin endpoints may be unreachable.');
+  process.exit(1);
+}
+const providerTools = formatToolsForProvider(provider, allTools);
+
+let messages = buildInitialMessages(provider, SYSTEM_PROMPT, TASK, MODEL);
+
+const runStart = Date.now();
+let totalInputTokens = 0;
+let totalOutputTokens = 0;
+let totalToolCalls = 0;
+let totalToolErrors = 0;
+let stepsCompleted = 0;
+let exitReason = 'max_steps';
+let initialNoToolRecoveries = 0;
+const agentTrace = createAgentTrace();
+
+log.info('run_start', { task: TASK.slice(0, 200), maxSteps: MAX_STEPS });
+
+try {
+  for (let step = 0; step < MAX_STEPS; step++) {
+    stepsCompleted = step + 1;
+    const iterStart = Date.now();
+
+    log.llmCall(step + 1);
+    let response;
+    try {
+      response = await callLlmWithMessages(provider, MODEL, apiKey, messages, providerTools);
+    } catch (err) {
+      const msg = err?.message ?? String(err);
+      log.error('llm_call_failed', { step: step + 1, error: msg });
+      process.stderr.write(`[zero-shot] LLM API error: ${msg.slice(0, 500)}\n`);
+      exitReason = 'llm_error';
+      break;
+    }
+
+    const iterDurationMs = Date.now() - iterStart;
+    totalInputTokens += response.usage.inputTokens;
+    totalOutputTokens += response.usage.outputTokens;
+
+    const thinking = getThinkingContent(provider, response);
+    const text = getResponseText(provider, response);
+
+    const hasToolCalls = !!parseToolCalls(provider, response);
+    const stopReason = getStopReason(provider, response);
+    log.llmResponse(step + 1, iterDurationMs, hasToolCalls, stopReason);
+    log.tokenUsage(step + 1, response.usage, {
+      inputTokens: totalInputTokens,
+      outputTokens: totalOutputTokens,
+    });
+
+    messages = appendAssistantResponse(provider, messages, response);
+
+    const toolCalls = parseToolCalls(provider, response);
+
+    if (!toolCalls) {
+      agentTrace.addStep({ step: step + 1, thinking, text, toolCalls: [], durationMs: iterDurationMs });
+      if (text) {
+        process.stderr.write(`[zero-shot] Step ${step + 1}: ${text.slice(0, 200)}\n`);
+      }
+      const shouldRecoverInitialNoToolCall = totalToolCalls === 0
+        && initialNoToolRecoveries < MAX_INITIAL_NO_TOOL_RECOVERIES;
+      if (shouldRecoverInitialNoToolCall) {
+        initialNoToolRecoveries++;
+        messages = appendUserInstruction(
+          provider,
+          messages,
+          'You must use tools to make progress. ' +
+            'On your next response, call at least one relevant tool before giving any summary or conclusion. ' +
+            'Start by gathering concrete evidence from the systems, then execute the required actions.',
+        );
+        log.info('no_tool_calls_reprompt', {
+          step: step + 1,
+          attempt: initialNoToolRecoveries,
+        });
+        continue;
+      }
+      exitReason = totalToolCalls === 0 ? 'no_tool_calls' : 'completed';
+      break;
+    }
+    initialNoToolRecoveries = 0;
+
+    const results = [];
+    for (const tc of toolCalls) {
+      const toolStart = Date.now();
+      process.stderr.write(`[zero-shot] Step ${step + 1}: ${tc.name}\n`);
+      try {
+        const result = await callToolRest(toolToTwin, tc.name, tc.arguments);
+        results.push(result);
+        totalToolCalls++;
+        log.toolCall(step + 1, tc.name, tc.arguments, Date.now() - toolStart);
+      } catch (err) {
+        // Log error and continue with error text — no retry
+        results.push(`Error: ${err.message}`);
+        totalToolCalls++;
+        totalToolErrors++;
+        log.toolError(step + 1, tc.name, err.message);
+        process.stderr.write(`[zero-shot] Tool error: ${err.message}\n`);
+      }
+    }
+
+    agentTrace.addStep({
+      step: step + 1,
+      thinking,
+      text,
+      toolCalls: toolCalls.map((tc) => ({ name: tc.name, arguments: tc.arguments })),
+      durationMs: iterDurationMs,
+    });
+
+    messages = appendToolResults(provider, messages, toolCalls, results);
+  }
+} finally {
+  const totalTimeMs = Date.now() - runStart;
+
+  log.summary({
+    iterations: stepsCompleted,
+    totalInputTokens,
+    totalOutputTokens,
+    totalTimeMs,
+    toolCallCount: totalToolCalls,
+    toolErrorCount: totalToolErrors,
+    exitReason,
+  });
+
+  writeMetrics({
+    inputTokens: totalInputTokens,
+    outputTokens: totalOutputTokens,
+    llmCallCount: stepsCompleted,
+    toolCallCount: totalToolCalls,
+    toolErrorCount: totalToolErrors,
+    totalTimeMs,
+    exitReason,
+    provider,
+    model: MODEL,
+  });
+
+  agentTrace.flush();
+
+  process.stderr.write(
+    `\n[zero-shot] Summary: ${stepsCompleted} iterations, ${totalToolCalls} tool calls ` +
+    `(${totalToolErrors} errors), ${totalInputTokens} input tokens, ` +
+    `${totalOutputTokens} output tokens, ${(totalTimeMs / 1000).toFixed(1)}s total\n`
+  );
+
+  if (exitReason === 'llm_error') {
+    process.exit(1);
+  }
+}

package/dist/harnesses/zero-shot/archal-harness.json

@@ -0,0 +1,21 @@
+{
+  "version": 1,
+  "name": "zero-shot",
+  "description": "Medium-quality harness. Minimal system prompt, basic error handling (log and continue), no retry. Good for testing model raw capability without agent scaffolding.",
+  "local": {
+    "command": "node",
+    "args": ["agent.mjs"]
+  },
+  "maxSteps": 40,
+  "supportedProviders": ["openai", "anthropic", "gemini"],
+  "requiredEnvVars": [
+    "ARCHAL_ENGINE_TASK",
+    "ARCHAL_ENGINE_MODEL"
+  ],
+  "configDefaults": {
+    "maxSteps": 40,
+    "systemPrompt": true,
+    "errorHandling": true,
+    "retryOnTransient": false
+  }
+}