@wix/evalforge-evaluator 0.35.0 → 0.36.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-evaluator",
-  "version": "0.35.0",
+  "version": "0.36.0",
   "description": "EvalForge Evaluator",
   "bin": "./build/index.js",
   "files": [
@@ -13,13 +13,14 @@
     "build:types": "tsc --emitDeclarationOnly --outDir ./build/types",
     "build": "yarn run clean && yarn run build:cjs && yarn run build:esm && yarn run build:types",
     "lint": "eslint .",
-    "test": "node --import tsx --test tests/**/*.test.ts"
+    "test": "echo 'Tests moved to @wix/eval-assertions package' && exit 0"
   },
   "dependencies": {
     "@ai-sdk/anthropic": "^3.0.2",
     "@anthropic-ai/claude-agent-sdk": "^0.2.12",
     "@anthropic-ai/claude-code": "^2.0.76",
-    "@wix/evalforge-types": "0.9.0",
+    "@wix/eval-assertions": "0.1.0",
+    "@wix/evalforge-types": "0.10.0",
     "ai": "^6.0.6",
     "tar": "^7.5.3",
     "zod": "^4.3.5"
@@ -56,5 +57,5 @@
       "artifactId": "evalforge-evaluator"
     }
   },
-  "falconPackageHash": "0fc6b5a035fe98c64d80f6c5d204f42b06e9f3928a666f95795c94ab"
+  "falconPackageHash": "15d351fd4bdbc1ded924fcc82efa1cf47c410837b12f78bd9fdee310"
 }

package/build/types/run-scenario/assertions/assertionEvaluator.d.ts

@@ -1,37 +0,0 @@
-import type { Assertion, AssertionResult } from '@wix/evalforge-types';
-import type { EvaluatorConfig } from '../../config.js';
-import type { PartialEvalRunResult } from '../types.js';
-/**
- * Options passed to the LLM for llm_judge. Used by the optional stub for testing.
- */
-export interface LlmJudgeGenerateTextOptions {
-    prompt: string;
-    system: string;
-    maxOutputTokens: number;
-    temperature: number;
-}
-/**
- * Optional context passed when evaluating assertions (e.g. workDir for build_passed, config for llm_judge).
- */
-export interface AssertionContext {
-    /** Working directory for the scenario (used by build_passed) */
-    workDir?: string;
-    /** Evaluator config (used by llm_judge for AI gateway) */
-    config?: EvaluatorConfig;
-    /**
-     * Optional stub for llm_judge: when set, the evaluator uses this instead of the real AI call.
-     * Used only in tests to avoid hitting the API.
-     */
-    generateTextForLlmJudge?: (options: LlmJudgeGenerateTextOptions) => Promise<{
-        text: string;
-    }>;
-}
-/**
- * Abstract base for assertion evaluators.
- * Each assertion type has a concrete class that implements evaluate().
- * evaluate() may return a Promise for async assertions (e.g. llm_judge).
- */
-export declare abstract class AssertionEvaluator<T extends Assertion = Assertion> {
-    abstract readonly type: T['type'];
-    abstract evaluate(assertion: T, partialResult: PartialEvalRunResult, context?: AssertionContext): AssertionResult | Promise<AssertionResult>;
-}

package/build/types/run-scenario/assertions/buildPassedEvaluator.d.ts

@@ -1,12 +0,0 @@
-import type { BuildPassedAssertion, AssertionResult } from '@wix/evalforge-types';
-import type { PartialEvalRunResult } from '../types.js';
-import type { AssertionContext } from './assertionEvaluator.js';
-import { AssertionEvaluator } from './assertionEvaluator.js';
-/**
- * Evaluator for "build_passed" assertion: runs a build command in the scenario
- * working directory and passes if the command exits with the expected code (default 0).
- */
-export declare class BuildPassedEvaluator extends AssertionEvaluator<BuildPassedAssertion> {
-    readonly type: "build_passed";
-    evaluate(assertion: BuildPassedAssertion, partialResult: PartialEvalRunResult, context?: AssertionContext): AssertionResult;
-}

package/build/types/run-scenario/assertions/index.d.ts

@@ -1,13 +0,0 @@
-import type { Assertion, AssertionResult } from '@wix/evalforge-types';
-import type { PartialEvalRunResult } from '../types.js';
-import type { AssertionContext } from './assertionEvaluator.js';
-/**
- * Evaluate all assertions against the partial result.
- *
- * @param partialResult - Result from callSkill/callAgent (includes llmTrace)
- * @param assertions - Optional list of assertions to evaluate
- * @param context - Optional context (e.g. workDir for build_passed, config for llm_judge)
- * @returns Array of assertion results; empty if no assertions
- */
-export declare function evaluateAssertions(partialResult: PartialEvalRunResult, assertions: Assertion[], context?: AssertionContext): Promise<AssertionResult[]>;
-export { AssertionEvaluator } from './assertionEvaluator.js';

package/build/types/run-scenario/assertions/llmJudgeEvaluator.d.ts

@@ -1,25 +0,0 @@
-import type { LlmJudgeAssertion, AssertionResult, LLMTrace } from '@wix/evalforge-types';
-import type { PartialEvalRunResult } from '../types.js';
-import type { AssertionContext } from './assertionEvaluator.js';
-import { AssertionEvaluator } from './assertionEvaluator.js';
-export interface JudgeResult {
-    text: string;
-    score: number;
-    scoreReasoning: string;
-}
-/**
- * Format LLM trace as readable text for the judge (step number, type, tool name/args, output preview).
- */
-export declare function formatTraceForJudge(llmTrace: LLMTrace | undefined): string;
-export declare function replacePlaceholders(str: string, output: string, cwd: string, changedFiles: string, trace: string): string;
-export declare function validateJudgeResult(parsed: unknown): JudgeResult;
-/**
- * Evaluator for "llm_judge" assertion: an LLM judges the scenario output
- * (prompt with {{output}}, {{cwd}}, {{changedFiles}}, {{trace}}) and returns a score 0–100.
- * Passes if score >= minScore.
- */
-export declare class LlmJudgeEvaluator extends AssertionEvaluator<LlmJudgeAssertion> {
-    readonly type: "llm_judge";
-    evaluate(assertion: LlmJudgeAssertion, partialResult: PartialEvalRunResult, context?: AssertionContext): Promise<AssertionResult>;
-    private callGenerateText;
-}

package/build/types/run-scenario/assertions/skillWasCalledEvaluator.d.ts

@@ -1,12 +0,0 @@
-import type { SkillWasCalledAssertion, AssertionResult } from '@wix/evalforge-types';
-import type { PartialEvalRunResult } from '../types.js';
-import type { AssertionContext } from './assertionEvaluator.js';
-import { AssertionEvaluator } from './assertionEvaluator.js';
-/**
- * Evaluator for "skill_was_called" assertion: the LLM trace must contain a step
- * where the "Skill" tool was used with the expected skill (by name).
- */
-export declare class SkillWasCalledEvaluator extends AssertionEvaluator<SkillWasCalledAssertion> {
-    readonly type: "skill_was_called";
-    evaluate(assertion: SkillWasCalledAssertion, partialResult: PartialEvalRunResult, _context?: AssertionContext): AssertionResult;
-}