@wix/evalforge-types 0.36.0 → 0.37.0

package/build/types/assertion/assertion.d.ts

@@ -4,11 +4,13 @@ import { z } from 'zod';
  * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
  * - build_passed: Runs a command and checks exit code (deterministic, system-level)
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
+ * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
  * - llm_judge: LLM evaluates output with a prompt (LLM-based, user-created)
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
     build_passed: "build_passed";
+    cost: "cost";
     llm_judge: "llm_judge";
     time_limit: "time_limit";
 }>;
@@ -57,6 +59,15 @@ export declare const SkillWasCalledConfigSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>;
 export type SkillWasCalledConfig = z.infer<typeof SkillWasCalledConfigSchema>;
+/**
+ * Configuration for cost assertion type.
+ * Uses strictObject to reject objects with unknown keys (prevents matching other configs).
+ */
+export declare const CostConfigSchema: z.ZodObject<{
+    /** Maximum allowed cost in USD */
+    maxCostUsd: z.ZodNumber;
+}, z.core.$strict>;
+export type CostConfig = z.infer<typeof CostConfigSchema>;
 /**
  * Configuration for build_passed assertion type.
  * Uses strictObject to reject objects with unknown keys (prevents matching LlmJudge configs).
@@ -132,6 +143,9 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;
+}, z.core.$strict>, z.ZodObject<{
+    /** Maximum allowed cost in USD */
+    maxCostUsd: z.ZodNumber;
 }, z.core.$strict>, z.ZodObject<{
     /** Command to run (default: "yarn build") */
     command: z.ZodOptional<z.ZodString>;
@@ -154,6 +168,7 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>;
@@ -181,6 +196,9 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
@@ -196,6 +214,7 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>;
@@ -226,6 +245,9 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
@@ -241,6 +263,7 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>>;
@@ -271,6 +294,9 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;

package/build/types/assertion/system-assertions.d.ts

@@ -22,6 +22,7 @@ export declare const SYSTEM_ASSERTION_IDS: {
     readonly SKILL_WAS_CALLED: "system:skill_was_called";
     readonly BUILD_PASSED: "system:build_passed";
     readonly TIME_LIMIT: "system:time_limit";
+    readonly COST: "system:cost";
     readonly LLM_JUDGE: "system:llm_judge";
 };
 export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];

package/build/types/scenario/assertions.d.ts

@@ -21,6 +21,15 @@ export declare const BuildPassedAssertionSchema: z.ZodObject<{
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export type BuildPassedAssertion = z.infer<typeof BuildPassedAssertionSchema>;
+/**
+ * Assertion: the scenario LLM execution cost must stay within a USD threshold.
+ * Checked by reading llmTrace.summary.totalCostUsd.
+ */
+export declare const CostAssertionSchema: z.ZodObject<{
+    type: z.ZodLiteral<"cost">;
+    maxCostUsd: z.ZodNumber;
+}, z.core.$strip>;
+export type CostAssertion = z.infer<typeof CostAssertionSchema>;
 /**
  * Assertion: an LLM judges the scenario output (score 0-100).
  * Prompt can use {{output}}, {{cwd}}, {{changedFiles}}, {{trace}}.
@@ -60,6 +69,9 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"time_limit">;
     maxDurationMs: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"cost">;
+    maxCostUsd: z.ZodNumber;
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
     prompt: z.ZodString;

package/build/types/scenario/test-scenario.d.ts

@@ -35,6 +35,9 @@ export declare const TestScenarioSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -70,6 +73,9 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -105,6 +111,9 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.36.0",
+  "version": "0.37.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "1beee538e2fe877b490209a7f00a37f2524d6cece55e7c57bdd0f20a"
+  "falconPackageHash": "57c752f025e2a8ccb67557327757571462f50d8acadc4a28877770af"
 }