@wix/evalforge-evaluator 0.54.0 → 0.56.0

package/build/types/fetch-evaluation-data.d.ts

@@ -1,4 +1,4 @@
-import type { EvalRun, TestScenario, Skill, Agent, Template, Assertion } from '@wix/evalforge-types';
+import type { EvalRun, TestScenario, Skill, SkillsGroup, Agent, Template, Assertion } from '@wix/evalforge-types';
 import type { ApiClient } from './api-client.js';
 /**
  * A scenario paired with its optional template and resolved assertions.
@@ -11,12 +11,15 @@ export interface ScenarioItem {
 }
 /**
  * Data shape returned by fetchEvaluationData for the run flow.
- * Run-level code agent + skills; per-scenario template.
+ * Run-level code agent + skills + skills group; per-scenario template.
  */
 export interface EvaluationData {
     evalRun: EvalRun;
     codeAgent: Agent | null;
     skills: Skill[];
+    skillsGroup: SkillsGroup | null;
+    /** Display name for the skills group (from skillsGroup.name when present) */
+    skillsGroupName: string;
     scenarioItems: ScenarioItem[];
 }
 /**

package/build/types/run-scenario/agents/claude-code/execute.d.ts

@@ -1,14 +1,18 @@
 import type { Skill, TestScenario, LLMTrace } from '@wix/evalforge-types';
 import type { ClaudeCodeExecutionOptions, ClaudeCodeExecutionResult } from './types.js';
 /**
- * Execute a skill using the Claude Agent SDK.
+ * Execute skills using the Claude Agent SDK.
  *
- * @param skill - The skill to execute (SKILL.md content used as system prompt)
+ * All skills are written to .claude/skills/<name>/SKILL.md. The SDK discovers
+ * them from the filesystem; no system prompt is passed. The agent has access
+ * to all skills as a group.
+ *
+ * @param skills - All skills to execute (each written to its own directory)
  * @param scenario - The test scenario containing the trigger prompt
  * @param options - Execution options (cwd, mcpServers, etc.)
  * @returns Execution result with output, usage, and LLM trace
  */
-export declare function executeWithClaudeCode(skill: Skill, scenario: TestScenario, options: ClaudeCodeExecutionOptions): Promise<{
+export declare function executeWithClaudeCode(skills: Skill[], scenario: TestScenario, options: ClaudeCodeExecutionOptions): Promise<{
     result: ClaudeCodeExecutionResult;
     llmTrace: LLMTrace;
 }>;

package/build/types/run-scenario/agents/claude-code/types.d.ts

@@ -23,8 +23,6 @@ export interface McpServerConfig {
 export interface ClaudeCodeExecutionOptions {
     /** Working directory for the execution */
     cwd: string;
-    /** System prompt to append (typically the SKILL.md content) */
-    systemPrompt?: string;
     /** MCP server configurations */
     mcpServers?: Record<string, McpServerConfig>;
     /** Model to use for generation */

package/build/types/run-scenario/index.d.ts

@@ -1,18 +1,18 @@
 import type { EvalRunResult, TestScenario, Template } from '@wix/evalforge-types';
 import type { EvaluatorConfig } from '../config.js';
-import type { TargetInput } from './types.js';
 import type { Assertion } from '@wix/evalforge-types';
-export type { TargetInput, PartialEvalRunResult } from './types.js';
+import type { EvaluationData } from '../fetch-evaluation-data.js';
+export type { PartialEvalRunResult } from './types.js';
 export type { ScenarioItem, EvaluationData } from '../fetch-evaluation-data.js';
 /**
- * Run a single scenario against a target.
+ * Run a single scenario using evaluation data.
  *
  * @param config - Evaluator configuration
  * @param evalRunId - The evaluation run ID
  * @param scenario - The test scenario to run
- * @param target - The target to test (skill or agent)
+ * @param evalData - Fetched evaluation data (skills, skillsGroup, codeAgent)
  * @param template - Optional pre-fetched template entity
  * @param resolvedAssertions - Optional assertions resolved from assertionIds
  * @returns Complete evaluation result
  */
-export declare function runScenario(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, target: TargetInput, template?: Template, resolvedAssertions?: Assertion[]): Promise<EvalRunResult>;
+export declare function runScenario(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, evalData: EvaluationData, template?: Template, resolvedAssertions?: Assertion[]): Promise<EvalRunResult>;

package/build/types/run-scenario/run-agent-with-context.d.ts

@@ -0,0 +1,21 @@
+import type { Skill, Agent, TestScenario } from '@wix/evalforge-types';
+import type { EvaluatorConfig } from '../config.js';
+import type { PartialEvalRunResult } from './types.js';
+/**
+ * Run the agent with the full execution context (skills, MCPs, sub-agents, etc.).
+ *
+ * Uses the agent adapter registry to select the appropriate adapter based on
+ * the agent's runCommand. If no agent is specified, defaults to 'claude'.
+ * The context includes skills group, optional MCPs, optional sub-agents.
+ *
+ * @param config - Evaluator configuration
+ * @param evalRunId - The evaluation run ID (for live trace context)
+ * @param scenario - The test scenario to run
+ * @param skills - All skills in the group to test together
+ * @param skillsGroupId - ID of the skills group (for workDir and result targetId)
+ * @param skillsGroupName - Display name for the skills group
+ * @param agent - Optional linked coding agent to run within
+ * @param workDir - Optional working directory for the scenario
+ * @returns Partial result without assertion fields
+ */
+export declare function runAgentWithContext(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, skills: Skill[], skillsGroupId: string, skillsGroupName: string, agent?: Agent, workDir?: string): Promise<PartialEvalRunResult>;

package/build/types/run-scenario/types.d.ts

@@ -1,20 +1,8 @@
-import type { EvalRunResult, Skill, Agent, LLMStepType } from '@wix/evalforge-types';
+import type { EvalRunResult, LLMStepType } from '@wix/evalforge-types';
 /**
  * Partial result returned by call functions (without assertion fields).
  */
 export type PartialEvalRunResult = Omit<EvalRunResult, 'assertionResults' | 'passed' | 'failed' | 'passRate'>;
-/**
- * Target types for dispatching scenario execution.
- * PR4: only skill (with optional code agent) and standalone agent are used.
- */
-export type TargetInput = {
-    type: 'skill';
-    skill: Skill;
-    agent?: Agent;
-} | {
-    type: 'agent';
-    agent: Agent;
-};
 /**
  * Step data captured during streaming with timing info.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-evaluator",
-  "version": "0.54.0",
+  "version": "0.56.0",
   "description": "EvalForge Evaluator",
   "bin": "./build/index.js",
   "files": [
@@ -20,7 +20,7 @@
     "@anthropic-ai/claude-agent-sdk": "^0.2.12",
     "@anthropic-ai/claude-code": "^2.0.76",
     "@wix/eval-assertions": "0.7.0",
-    "@wix/evalforge-types": "0.20.0",
+    "@wix/evalforge-types": "0.22.0",
     "ai": "^6.0.6",
     "diff": "^7.0.0",
     "tar": "^7.5.3",
@@ -59,5 +59,5 @@
       "artifactId": "evalforge-evaluator"
     }
   },
-  "falconPackageHash": "c70c247a18a6cbb88083e45b4d517001dfeead7d50f93b4d43bd4f0e"
+  "falconPackageHash": "26eed2a12041f36af6cb43667cdd603be49ece4e6b44410dfcf29efb"
 }

package/build/types/run-scenario/callAgent.d.ts

@@ -1,13 +0,0 @@
-import type { Agent, TestScenario } from '@wix/evalforge-types';
-import type { EvaluatorConfig } from '../config.js';
-import type { PartialEvalRunResult } from './types.js';
-/**
- * Execute a scenario against a standalone coding agent.
- *
- * @param config - Evaluator configuration
- * @param scenario - The test scenario to run
- * @param agent - The coding agent to test
- * @param workDir - Optional working directory for the scenario
- * @returns Partial result without assertion fields
- */
-export declare function callAgent(config: EvaluatorConfig, scenario: TestScenario, agent: Agent, workDir?: string): Promise<PartialEvalRunResult>;

package/build/types/run-scenario/callSkill.d.ts

@@ -1,18 +0,0 @@
-import type { Skill, Agent, TestScenario } from '@wix/evalforge-types';
-import type { EvaluatorConfig } from '../config.js';
-import type { PartialEvalRunResult } from './types.js';
-/**
- * Execute a scenario against a skill (optionally with a linked coding agent).
- *
- * Uses the agent adapter registry to select the appropriate adapter based on
- * the agent's runCommand. If no agent is specified, defaults to 'claude'.
- *
- * @param config - Evaluator configuration
- * @param evalRunId - The evaluation run ID (for live trace context)
- * @param scenario - The test scenario to run
- * @param skill - The skill to test
- * @param agent - Optional linked coding agent to run the skill within
- * @param workDir - Optional working directory for the scenario
- * @returns Partial result without assertion fields
- */
-export declare function callSkill(config: EvaluatorConfig, evalRunId: string, scenario: TestScenario, skill: Skill, agent?: Agent, workDir?: string): Promise<PartialEvalRunResult>;