@wix/evalforge-evaluator 0.6.0

package/build/types/api-client.d.ts

@@ -0,0 +1,22 @@
+/**
+ * API Client for fetching data from the eval server.
+ */
+import type { EvalRun, EvalRunResult, SkillsGroup, TestScenario, Template, Skill, Agent } from '@wix/evalforge-types';
+export interface ApiClient {
+    getEvalRun(projectId: string, id: string): Promise<EvalRun>;
+    getSkillsGroup(projectId: string, id: string): Promise<SkillsGroup>;
+    getScenario(projectId: string, id: string): Promise<TestScenario>;
+    getSkill(projectId: string, id: string): Promise<Skill>;
+    getAgent(projectId: string, id: string): Promise<Agent>;
+    getTemplate(projectId: string, id: string): Promise<Template>;
+    addResult(projectId: string, evalRunId: string, result: EvalRunResult): Promise<void>;
+    clearResults(projectId: string, evalRunId: string): Promise<void>;
+    updateEvalRun(projectId: string, evalRunId: string, update: Partial<EvalRun>): Promise<void>;
+}
+/**
+ * Create an API client for the eval server.
+ *
+ * @param serverUrl - Base URL of the server (e.g., "https://dev.wix.com/_api/evalforge-backend")
+ * @param apiPrefix - API path prefix (default: empty string, set to "/api/v1" if needed)
+ */
+export declare function createApiClient(serverUrl: string, apiPrefix?: string): ApiClient;

package/build/types/config.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Evaluator Configuration
+ *
+ * Reads configuration from environment variables set by the server
+ * when spawning the evaluator process.
+ */
+export interface EvaluatorConfig {
+    /** Backend server URL for fetching data and reporting results */
+    serverUrl: string;
+    /** API path prefix (default: empty, set to "/api/v1" if needed for production) */
+    apiPrefix: string;
+    /** AI Gateway base URL for LLM calls */
+    aiGatewayUrl: string;
+    /** Custom headers for AI Gateway requests (parsed from "key:value" lines) */
+    aiGatewayHeaders: Record<string, string>;
+    /** Directory for storing evaluation working directories */
+    evaluationsDir?: string;
+}
+/**
+ * Load evaluator configuration from environment variables.
+ *
+ * @throws Error if required environment variables are missing
+ */
+export declare function loadConfig(): EvaluatorConfig;

package/build/types/fetch-evaluation-data.d.ts

@@ -0,0 +1,29 @@
+import type { EvalRun, TestScenario, Skill, Agent, Template } from '@wix/evalforge-types';
+import type { ApiClient } from './api-client.js';
+/**
+ * A scenario paired with its optional template.
+ */
+export interface ScenarioItem {
+    scenario: TestScenario;
+    template?: Template;
+}
+/**
+ * Data shape returned by fetchEvaluationData for the run flow.
+ * Run-level code agent + skills; per-scenario template.
+ */
+export interface EvaluationData {
+    evalRun: EvalRun;
+    codeAgent: Agent | null;
+    skills: Skill[];
+    scenarioItems: ScenarioItem[];
+}
+/**
+ * Fetch all data needed to run an evaluation.
+ *
+ * Uses evalRun.agentId and evalRun.skillsGroupId as run-level inputs.
+ * Loads skills from the skills group; loads code agent; loads scenarios and templates by scenario.templateId.
+ *
+ * @throws Error if eval run has no code agent when expected
+ * @throws Error if eval run has no skills group or skills group has no skills when expected
+ */
+export declare function fetchEvaluationData(api: ApiClient, projectId: string, evalRunId: string): Promise<EvaluationData>;

package/build/types/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/build/types/local.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/build/types/run-scenario/callAgent.d.ts

@@ -0,0 +1,13 @@
+import type { Agent, TestScenario } from '@wix/evalforge-types';
+import type { EvaluatorConfig } from '../config.js';
+import type { PartialEvalRunResult } from './types.js';
+/**
+ * Execute a scenario against a standalone coding agent.
+ *
+ * @param config - Evaluator configuration
+ * @param scenario - The test scenario to run
+ * @param agent - The coding agent to test
+ * @param workDir - Optional working directory for the scenario
+ * @returns Partial result without assertion fields
+ */
+export declare function callAgent(config: EvaluatorConfig, scenario: TestScenario, agent: Agent, workDir?: string): Promise<PartialEvalRunResult>;

package/build/types/run-scenario/callSkill.d.ts

@@ -0,0 +1,15 @@
+import type { Skill, Agent, TestScenario } from '@wix/evalforge-types';
+import type { EvaluatorConfig } from '../config.js';
+import type { PartialEvalRunResult } from './types.js';
+/**
+ * Execute a scenario against a skill (optionally with a linked coding agent).
+ *
+ * @param config - Evaluator configuration
+ * @param evalRunId - The evaluation run ID (for live trace context)
+ * @param scenario - The test scenario to run
+ * @param skill - The skill to test
+ * @param agent - Optional linked coding agent to run the skill within
+ * @param workDir - Optional working directory for the scenario
+ * @returns Partial result without assertion fields
+ */
+export declare function callSkill(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, skill: Skill, agent?: Agent, workDir?: string): Promise<PartialEvalRunResult>;

package/build/types/run-scenario/claude-code/execute.d.ts

@@ -0,0 +1,14 @@
+import type { Skill, TestScenario, LLMTrace } from '@wix/evalforge-types';
+import type { ClaudeCodeExecutionOptions, ClaudeCodeExecutionResult } from './types.js';
+/**
+ * Execute a skill using the Claude Agent SDK.
+ *
+ * @param skill - The skill to execute (SKILL.md content used as system prompt)
+ * @param scenario - The test scenario containing the trigger prompt
+ * @param options - Execution options (cwd, mcpServers, etc.)
+ * @returns Execution result with output, usage, and LLM trace
+ */
+export declare function executeWithClaudeCode(skill: Skill, scenario: TestScenario, options: ClaudeCodeExecutionOptions): Promise<{
+    result: ClaudeCodeExecutionResult;
+    llmTrace: LLMTrace;
+}>;

package/build/types/run-scenario/claude-code/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Claude Agent SDK integration module.
+ *
+ * This module provides functionality to execute skills using the
+ * @anthropic-ai/claude-agent-sdk with full LLM trace support.
+ */
+export { executeWithClaudeCode } from './execute.js';
+export type { ClaudeCodeExecutionOptions, ClaudeCodeExecutionResult } from './types.js';

package/build/types/run-scenario/claude-code/types.d.ts

@@ -0,0 +1,68 @@
+/**
+ * MCP server configuration compatible with Claude Agent SDK.
+ */
+export interface McpServerConfig {
+    type?: 'stdio';
+    command: string;
+    args?: string[];
+    env?: Record<string, string>;
+}
+/**
+ * Trace context for live streaming of agent execution.
+ */
+export interface TraceContext {
+    /** The evaluation run ID */
+    evalRunId: string;
+    /** The scenario ID being executed */
+    scenarioId: string;
+    /** The scenario name for display */
+    scenarioName: string;
+    /** The target ID (skill, agent, etc.) */
+    targetId: string;
+    /** The target name for display */
+    targetName: string;
+}
+/**
+ * Options for Claude Agent SDK execution.
+ */
+export interface ClaudeCodeExecutionOptions {
+    /** Working directory for the execution */
+    cwd: string;
+    /** System prompt to append (typically the SKILL.md content) */
+    systemPrompt?: string;
+    /** MCP server configurations */
+    mcpServers?: Record<string, McpServerConfig>;
+    /** Model to use for generation */
+    model?: string;
+    /** Temperature for generation (0–1, from agent modelConfig) */
+    temperature?: number;
+    /** Max output tokens (from agent modelConfig) */
+    maxTokens?: number;
+    /** Maximum number of agent turns */
+    maxTurns?: number;
+    /** Maximum thinking tokens */
+    maxThinkingTokens?: number;
+    /** AI Gateway base URL for Anthropic API */
+    aiGatewayUrl?: string;
+    /** AI Gateway custom headers as key-value pairs */
+    aiGatewayHeaders?: Record<string, string>;
+    /** Trace context for live streaming (optional) */
+    traceContext?: TraceContext;
+}
+/**
+ * Result from Claude Code execution.
+ */
+export interface ClaudeCodeExecutionResult {
+    /** Final output text */
+    outputText: string;
+    /** Total duration in milliseconds */
+    durationMs: number;
+    /** Token usage statistics */
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    };
+    /** Cost in USD (if available) */
+    costUsd?: number;
+}

package/build/types/run-scenario/environment.d.ts

@@ -0,0 +1,13 @@
+import type { Template } from '@wix/evalforge-types';
+import type { EvaluatorConfig } from '../config.js';
+/**
+ * Prepare the working directory for a scenario.
+ * If a template is provided, download and extract it.
+ *
+ * @param config - Evaluator configuration
+ * @param evalRunId - The evaluation run ID
+ * @param targetId - The target ID (prompt agent, skill, or agent)
+ * @param template - Optional pre-fetched template entity
+ * @returns The working directory path, or undefined if no template
+ */
+export declare function prepareWorkingDirectory(config: EvaluatorConfig, evalRunId: string, targetId: string, template?: Template): Promise<string | undefined>;

package/build/types/run-scenario/index.d.ts

@@ -0,0 +1,16 @@
+import { type EvalRunResult, type TestScenario, type Template } from '@wix/evalforge-types';
+import type { EvaluatorConfig } from '../config.js';
+import type { TargetInput } from './types.js';
+export type { TargetInput, PartialEvalRunResult } from './types.js';
+export type { ScenarioItem, EvaluationData } from '../fetch-evaluation-data.js';
+/**
+ * Run a single scenario against a target.
+ *
+ * @param config - Evaluator configuration
+ * @param evalRunId - The evaluation run ID
+ * @param scenario - The test scenario to run
+ * @param target - The target to test (skill or agent)
+ * @param template - Optional pre-fetched template entity
+ * @returns Complete evaluation result
+ */
+export declare function runScenario(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, target: TargetInput, template?: Template): Promise<EvalRunResult>;

package/build/types/run-scenario/llm-trace.d.ts

@@ -0,0 +1,6 @@
+import { type ModelIds, type LLMTrace } from '@wix/evalforge-types';
+import type { CapturedStep } from './types.js';
+/**
+ * Build LLMTrace from captured steps.
+ */
+export declare function buildLLMTrace(capturedSteps: CapturedStep[], modelId: ModelIds): LLMTrace;

package/build/types/run-scenario/types.d.ts

@@ -0,0 +1,37 @@
+import type { EvalRunResult, Skill, Agent, LLMStepType } from '@wix/evalforge-types';
+/**
+ * Partial result returned by call functions (without assertion fields).
+ */
+export type PartialEvalRunResult = Omit<EvalRunResult, 'assertionResults' | 'passed' | 'failed' | 'passRate'>;
+/**
+ * Target types for dispatching scenario execution.
+ * PR4: only skill (with optional code agent) and standalone agent are used.
+ */
+export type TargetInput = {
+    type: 'skill';
+    skill: Skill;
+    agent?: Agent;
+} | {
+    type: 'agent';
+    agent: Agent;
+};
+/**
+ * Step data captured during streaming with timing info.
+ */
+export interface CapturedStep {
+    text: string;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    };
+    finishReason: string;
+    toolCalls?: Array<{
+        toolName: string;
+        args: unknown;
+    }>;
+    toolResults?: unknown[];
+    startedAt: Date;
+    durationMs: number;
+}
+export { LLMStepType };

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@wix/evalforge-evaluator",
+  "version": "0.6.0",
+  "description": "EvalForge Evaluator",
+  "bin": "./build/index.js",
+  "files": [
+    "build"
+  ],
+  "scripts": {
+    "clean": "rm -rf build",
+    "build:cjs": "esbuild src/index.ts --bundle --platform=node --outfile=build/index.js --format=cjs --sourcemap --external:@wix/evalforge-types --external:@anthropic-ai/claude-agent-sdk",
+    "build:esm": "esbuild src/index.ts --bundle --platform=node --outfile=build/index.mjs --format=esm --sourcemap --external:@wix/evalforge-types --external:@anthropic-ai/claude-agent-sdk",
+    "build:types": "tsc --emitDeclarationOnly --outDir ./build/types",
+    "build": "yarn run clean && yarn run build:cjs && yarn run build:esm && yarn run build:types",
+    "lint": "eslint .",
+    "test": "echo 'No tests specified' && exit 0"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "^3.0.2",
+    "@anthropic-ai/claude-code": "^2.0.76",
+    "@wix/evalforge-types": "0.3.0",
+    "ai": "^6.0.6",
+    "tar": "^7.5.3"
+  },
+  "peerDependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.12"
+  },
+  "peerDependenciesMeta": {
+    "@anthropic-ai/claude-agent-sdk": {
+      "optional": false
+    }
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.39.2",
+    "@types/node": "^22.19.3",
+    "@types/tar": "^6.1.13",
+    "dotenv": "^17.2.3",
+    "esbuild": "^0.27.2",
+    "eslint": "^9.39.2",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.4",
+    "prettier": "^3.7.4",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.51.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./build/types/index.d.ts",
+      "import": "./build/index.mjs",
+      "require": "./build/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "wix": {
+    "artifact": {
+      "groupId": "com.wixpress",
+      "artifactId": "evalforge-evaluator"
+    }
+  },
+  "falconPackageHash": "948c3ac942079bd1a8592d8449148cee8a5198ae1654ff42e6d30952"
+}