@wix/evalforge-evaluator 0.201.0 → 0.203.0

package/build/types/run-scenario/agents/opencode/build-trace.d.ts

@@ -1,3 +1,17 @@
 import type { LLMTrace } from '@wix/evalforge-types';
 import type { TimestampedOpenCodeEvent } from './types.js';
-export declare function buildLLMTrace(timestampedEvents: TimestampedOpenCodeEvent[], totalDurationMs: number, model: string, provider: string, executionStartTime: Date): LLMTrace;
+/**
+ * Build an LLMTrace from the NDJSON events emitted by `opencode run --format json`.
+ *
+ * Events are grouped into turns delimited by `step_finish` events; each turn
+ * produces THINKING, TOOL_USE, and COMPLETION sub-steps to match the
+ * claude-code trace format.
+ */
+export declare function buildLLMTrace(timestampedEvents: TimestampedOpenCodeEvent[], totalDurationMs: number, model: string, provider: string, executionStartTime: Date, 
+/**
+ * Per-request cost (USD) captured from the gateway, in request order. When an
+ * entry is a finite number it overrides OpenCode's self-reported
+ * `step_finish.cost` (which is ~0 for the Wix gateway); missing entries fall
+ * back to it. @see gateway-cost-interceptor.ts
+ */
+gatewayCosts?: (number | null)[]): LLMTrace;

package/build/types/run-scenario/agents/opencode/execute.d.ts

@@ -21,15 +21,11 @@ export declare function writePromptImages(cwd: string, images: TriggerPromptImag
  */
 export declare function prepareOpenCodeEnvironment(cwd: string, skills: SkillWithLatestVersion[], options: Pick<OpenCodeExecutionOptions, 'mcps' | 'subAgents' | 'rules' | 'systemPrompt'>): Promise<void>;
 /**
- * Execute skills using the OpenCode CLI with automatic retry on idle timeouts.
+ * Execute skills using the OpenCode CLI.
  *
- * When the OpenCode process stalls (no output for 120s), it is killed and
- * retried with a recovery prompt that carries context about work already done.
- * This works around known OpenCode hanging issues where the upstream LLM
- * connection stalls mid-stream.
- *
- * @see https://github.com/anomalyco/opencode/issues/13230 - Memory leak causes OOM
- * @see https://github.com/anomalyco/opencode/issues/13841 - LLM connection stalls
+ * Wraps {@link executeWithOpenCodeInner} with a localhost cost interceptor so
+ * the trace uses the gateway's real per-request cost instead of OpenCode's
+ * self-reported (~$0) cost. The interceptor is always closed when done.
  */
 export declare function executeWithOpenCode(skills: SkillWithLatestVersion[], scenario: TestScenario, options: OpenCodeExecutionOptions): Promise<{
     result: OpenCodeExecutionResult;

package/build/types/run-scenario/agents/opencode/gateway-cost-interceptor.d.ts

@@ -0,0 +1,28 @@
+/**
+ * A localhost pass-through proxy placed between the OpenCode CLI and the real
+ * Wix AI Gateway.
+ *
+ * OpenCode prices the gateway as a free custom provider, so its self-reported
+ * `step_finish.cost` is ~0. The gateway, however, returns the real cost as
+ * `total_cost_usd` in every response (injected by `wix-ai-gateway-proxy`). But
+ * OpenCode owns the HTTP connection and discards that field, so the evaluator
+ * never sees it.
+ *
+ * This interceptor forwards each request to the real gateway untouched, streams
+ * the response straight back to OpenCode, and reads the `total_cost_usd` the
+ * gateway already put there — capturing the true per-request cost without any
+ * pricing tables. Captured costs are returned in request order so each maps to
+ * its OpenCode turn.
+ */
+export interface GatewayCostInterceptor {
+    /** Base URL to hand OpenCode as its gateway (`${url}/proxy/<provider>`). */
+    readonly url: string;
+    /** Per-request `total_cost_usd`, in request order; null when none was found. */
+    getCapturedCosts(): (number | null)[];
+    close(): Promise<void>;
+}
+/**
+ * Start the interceptor, forwarding to `realGatewayUrl` (no `/proxy` suffix —
+ * OpenCode appends that, mirroring `buildOpenCodeEnv`).
+ */
+export declare function startGatewayCostInterceptor(realGatewayUrl: string): Promise<GatewayCostInterceptor>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-evaluator",
-  "version": "0.201.0",
+  "version": "0.203.0",
   "description": "EvalForge Evaluator",
   "bin": "./build/index.js",
   "files": [
@@ -22,9 +22,9 @@
     "@ai-sdk/openai": "^3.0.39",
     "@anthropic-ai/claude-agent-sdk": "^0.2.63",
     "@anthropic-ai/claude-code": "^2.1.63",
-    "@wix/eval-assertions": "0.75.0",
-    "@wix/evalforge-github-client": "0.74.0",
-    "@wix/evalforge-types": "0.99.0",
+    "@wix/eval-assertions": "0.76.0",
+    "@wix/evalforge-github-client": "0.75.0",
+    "@wix/evalforge-types": "0.100.0",
     "ai": "^6.0.107",
     "diff": "^7.0.0",
     "tar": "^7.5.3",
@@ -72,5 +72,5 @@
       "artifactId": "evalforge-evaluator"
     }
   },
-  "falconPackageHash": "78268e3b62b8999381067758e12c2b03fb02314ca181a3c7a6e22320"
+  "falconPackageHash": "888ae6871e97a8bf573fb3830003f9df6d3ef0a79d8449f179ba4260"
 }