@wix/evalforge-evaluator 0.103.0 → 0.105.0

package/build/types/run-scenario/agents/index.d.ts

@@ -11,6 +11,7 @@
  * Agent adapters are automatically registered when their modules are imported.
  * Currently supported adapters:
  * - Claude Code (command: 'claude')
+ * - Simple Agent (adapter ID: 'simple-agent')
  *
  * @example
  * ```typescript
@@ -22,3 +23,4 @@
  */
 export { AgentAdapterRegistry, defaultRegistry, getAdapter, hasAdapter } from './registry.js';
 import './claude-code/index.js';
+import './simple-agent/index.js';

package/build/types/run-scenario/agents/{claude-code → shared}/resolve-mcp-placeholders.d.ts

@@ -2,15 +2,18 @@
  * MCP Config Placeholder Resolution
  *
  * Resolves `{{placeholder}}` patterns in MCP config values at eval startup.
- * Currently supports Wix auth placeholders sourced from ~/.wix/auth/api-key.json
- * (the same file that @wix/mcp reads via WIX_API_THROUGH_FS).
  *
  * Supported placeholders:
- *   - {{wix-auth-token}}   → token from api-key.json
+ *   - {{wix-auth-token}}   → token from ~/.wix/auth/api-key.json
  *   - {{wix-auth-user-id}} → userInfo.userId from api-key.json
+ *   - {{cwd}}              → environment folder (working directory)
  */
 import { findPlaceholders, resolveValue } from '../../../resolve-placeholders.js';
 export { findPlaceholders, resolveValue };
+export interface ResolveMcpPlaceholdersOptions {
+    cwd?: string;
+    authFilePath?: string;
+}
 /**
  * Read Wix auth placeholders from ~/.wix/auth/api-key.json.
  * Returns an empty map when the file is missing or malformed.
@@ -21,11 +24,11 @@ export declare function loadWixAuthPlaceholders(authFilePath?: string): Promise<
 /**
  * Resolve all `{{...}}` placeholders in an MCP server config map.
  *
- * Loads available placeholder values (currently from Wix auth file),
+ * Loads available placeholder values (Wix auth file + cwd),
  * scans the config for placeholders, and fails if any are unresolvable.
  *
  * @param mcpServers - The mcpServers map (server-name → config)
- * @param authFilePath - Override for testing
+ * @param options - Optional cwd and authFilePath overrides
  * @throws When a config references placeholders that cannot be resolved
  */
-export declare function resolveMcpPlaceholders(mcpServers: Record<string, Record<string, unknown>>, authFilePath?: string): Promise<Record<string, Record<string, unknown>>>;
+export declare function resolveMcpPlaceholders(mcpServers: Record<string, Record<string, unknown>>, options?: ResolveMcpPlaceholdersOptions): Promise<Record<string, Record<string, unknown>>>;

package/build/types/run-scenario/agents/shared/trace-emit.d.ts

@@ -0,0 +1,6 @@
+import type { LiveTraceEvent } from '@wix/evalforge-types';
+/**
+ * Emit a live trace event to stdout for the backend to capture.
+ * Also pushes to HTTP endpoint if tracePushUrl is provided (for remote job execution).
+ */
+export declare function emitTraceEvent(event: LiveTraceEvent, tracePushUrl?: string, routeHeader?: string, authToken?: string): void;

package/build/types/run-scenario/agents/simple-agent/build-conversation.d.ts

@@ -0,0 +1,10 @@
+import type { ConversationMessage } from '@wix/evalforge-types';
+import type { StepResult, ToolSet } from 'ai';
+/**
+ * Build a serializable conversation from Vercel AI SDK step results.
+ *
+ * Maps the step-based execution model to the same ConversationMessage
+ * format used by the Claude Code adapter, so both agent types render
+ * identically in the Conversation tab.
+ */
+export declare function buildConversation(triggerPrompt: string, steps: StepResult<ToolSet>[], executionStartMs: number): ConversationMessage[];

package/build/types/run-scenario/agents/simple-agent/cost-calculation.d.ts

@@ -0,0 +1,10 @@
+import type { StepResult, ToolSet } from 'ai';
+/**
+ * Calculate the cost for a single LLM step.
+ * Prefers the gateway-reported cost (accurate with cache discounts),
+ * falling back to the built-in pricing table.
+ */
+export declare function calculateStepCost(step: StepResult<ToolSet>, modelId: string, provider: string, tokenUsage: {
+    prompt: number;
+    completion: number;
+}): number;

package/build/types/run-scenario/agents/simple-agent/execute.d.ts

@@ -0,0 +1,28 @@
+import { type LanguageModel, type StepResult, type ToolSet } from 'ai';
+import type { AgentExecutionContext, AgentExecutionResult, LLMTrace } from '@wix/evalforge-types';
+export declare function createModel(modelId: string, baseUrl: string, headers: Record<string, string>): LanguageModel;
+export declare function isClaudeModelId(modelId: string): boolean;
+/**
+ * Extract text content from a skill's latest version files.
+ * Concatenates all file contents, prioritizing SKILL.md if present.
+ */
+export declare function extractSkillContent(files: Array<{
+    path: string;
+    content: string;
+}> | undefined): string | undefined;
+export declare function executeWithAiSdk(context: AgentExecutionContext): Promise<AgentExecutionResult>;
+/**
+ * Build the system prompt for Simple Agent from skills, rules, and optional override.
+ *
+ * Unlike CLI agents (which have DEFAULT_EVALUATOR_SYSTEM_PROMPT), the Simple Agent
+ * has no built-in default instructions. The three-state `systemPrompt` contract
+ * (undefined = default, null = none, string = custom) effectively collapses to
+ * two states here: both undefined and null result in no base prompt being prepended,
+ * while a non-empty string is included as the first segment.
+ */
+export declare function composeSystemPrompt(context: AgentExecutionContext): string | undefined;
+export declare function buildLLMTrace(steps: StepResult<ToolSet>[], totalDurationMs: number, totalUsage: {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+}, modelId: string, provider: string, executionStartMs: number): LLMTrace;

package/build/types/run-scenario/agents/simple-agent/index.d.ts

@@ -0,0 +1 @@
+export {};

package/build/types/run-scenario/agents/simple-agent/mcp-tools.d.ts

@@ -0,0 +1,39 @@
+import { Experimental_StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio';
+import type { MCPEntity } from '@wix/evalforge-types';
+interface McpToolsResult {
+    tools: Record<string, unknown>;
+    clients: Array<{
+        close: () => Promise<void>;
+    }>;
+}
+/**
+ * Build AI SDK tools from MCP entity configs.
+ *
+ * Each `MCPEntity.config` is a keyed record matching the `.mcp.json` shape,
+ * e.g. `{ "server-name": { "type": "stdio", "command": "node", "args": ["server.js"] } }`.
+ * Each entry becomes a live MCP client whose tools are merged into the result.
+ *
+ * Tool names are namespaced as `${serverName}__${toolName}` to avoid collisions.
+ */
+export declare function buildMcpTools(mcps: MCPEntity[], cwd: string): Promise<McpToolsResult>;
+export declare function closeMcpClients(clients: Array<{
+    close: () => Promise<void>;
+}>): Promise<void>;
+/**
+ * Build an AI SDK MCP transport from a server config entry.
+ *
+ * Supports three transport types:
+ *  - stdio: local process (Experimental_StdioMCPTransport)
+ *  - http:  streamable HTTP (config object with type: 'http')
+ *  - sse:   Server-Sent Events (config object with type: 'sse')
+ */
+export declare function buildTransport(serverName: string, config: Record<string, unknown>, cwd: string): Experimental_StdioMCPTransport | {
+    headers?: Record<string, string> | undefined;
+    type: "http";
+    url: string;
+} | {
+    headers?: Record<string, string> | undefined;
+    type: "sse";
+    url: string;
+};
+export {};

package/build/types/run-scenario/agents/simple-agent/simple-agent-adapter.d.ts

@@ -0,0 +1,19 @@
+import type { AgentAdapter, AgentExecutionContext, AgentExecutionResult } from '@wix/evalforge-types';
+import type { AgentRunCommand } from '@wix/evalforge-types';
+/**
+ * Simple Agent adapter.
+ *
+ * Executes LLM calls in-process via the Vercel AI SDK, routing through
+ * our proxy to support both Claude and OpenAI models. Supports MCP tools
+ * for interacting with the working directory.
+ *
+ * Looked up by adapter ID (not CLI command) since SDK agents have
+ * empty supportedCommands.
+ */
+export declare class SimpleAgentAdapter implements AgentAdapter {
+    readonly id = "simple-agent";
+    readonly name = "Simple Agent";
+    readonly supportedCommands: readonly AgentRunCommand[];
+    execute(context: AgentExecutionContext): Promise<AgentExecutionResult>;
+}
+export declare const simpleAgentAdapter: SimpleAgentAdapter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-evaluator",
-  "version": "0.103.0",
+  "version": "0.105.0",
   "description": "EvalForge Evaluator",
   "bin": "./build/index.js",
   "files": [
@@ -16,13 +16,15 @@
     "test": "node --import tsx --test tests/**/*.test.ts"
   },
   "dependencies": {
-    "@ai-sdk/anthropic": "^3.0.2",
-    "@anthropic-ai/claude-agent-sdk": "^0.2.44",
-    "@anthropic-ai/claude-code": "^2.1.44",
-    "@wix/eval-assertions": "0.22.0",
-    "@wix/evalforge-github-client": "0.21.0",
-    "@wix/evalforge-types": "0.46.0",
-    "ai": "^6.0.6",
+    "@ai-sdk/anthropic": "^3.0.45",
+    "@ai-sdk/mcp": "^1.0.21",
+    "@ai-sdk/openai": "^3.0.30",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.49",
+    "@anthropic-ai/claude-code": "^2.1.49",
+    "@wix/eval-assertions": "0.23.0",
+    "@wix/evalforge-github-client": "0.23.0",
+    "@wix/evalforge-types": "0.48.0",
+    "ai": "^6.0.93",
     "diff": "^7.0.0",
     "tar": "^7.5.3",
     "zod": "^4.3.5"
@@ -60,5 +62,5 @@
       "artifactId": "evalforge-evaluator"
     }
   },
-  "falconPackageHash": "770ab064ab729180951ced129aaf6e04a6d3bdf65e4cc014a353566e"
+  "falconPackageHash": "7f3f2d1b27312daf480e195d122e0885ac476d0f881100f407a34755"
 }