@wix/evalforge-types 0.35.0 → 0.36.0

package/build/types/assertion/assertion.d.ts

@@ -3,12 +3,14 @@ import { z } from 'zod';
  * Assertion types:
  * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
  * - build_passed: Runs a command and checks exit code (deterministic, system-level)
+ * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
  * - llm_judge: LLM evaluates output with a prompt (LLM-based, user-created)
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
     build_passed: "build_passed";
     llm_judge: "llm_judge";
+    time_limit: "time_limit";
 }>;
 /**
  * Parameter types supported in assertion parameters.
@@ -66,6 +68,15 @@ export declare const BuildPassedConfigSchema: z.ZodObject<{
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>;
 export type BuildPassedConfig = z.infer<typeof BuildPassedConfigSchema>;
+/**
+ * Configuration for time assertion type.
+ * Uses strictObject to reject objects with unknown keys.
+ */
+export declare const TimeConfigSchema: z.ZodObject<{
+    /** Maximum allowed duration in milliseconds */
+    maxDurationMs: z.ZodNumber;
+}, z.core.$strict>;
+export type TimeConfig = z.infer<typeof TimeConfigSchema>;
 /**
  * Configuration for llm_judge assertion type.
  * User-created assertions with customizable parameters.
@@ -119,6 +130,9 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>, z.ZodObject<{
+    /** Maximum allowed duration in milliseconds */
+    maxDurationMs: z.ZodNumber;
+}, z.core.$strict>, z.ZodObject<{
     /** Command to run (default: "yarn build") */
     command: z.ZodOptional<z.ZodString>;
     /** Expected exit code (default: 0) */
@@ -141,6 +155,7 @@ export declare const CustomAssertionSchema: z.ZodObject<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
+        time_limit: "time_limit";
     }>;
     config: z.ZodUnion<readonly [z.ZodObject<{
         prompt: z.ZodString;
@@ -164,6 +179,9 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Maximum allowed duration in milliseconds */
+        maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
         /** Expected exit code (default: 0) */
@@ -179,6 +197,7 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
+        time_limit: "time_limit";
     }>;
     name: z.ZodString;
     description: z.ZodString;
@@ -205,6 +224,9 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Maximum allowed duration in milliseconds */
+        maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
         /** Expected exit code (default: 0) */
@@ -220,6 +242,7 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
+        time_limit: "time_limit";
     }>>;
     name: z.ZodOptional<z.ZodString>;
     description: z.ZodOptional<z.ZodString>;
@@ -246,6 +269,9 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
     }, z.core.$strip>, z.ZodObject<{
+        /** Maximum allowed duration in milliseconds */
+        maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
         /** Expected exit code (default: 0) */

package/build/types/assertion/system-assertions.d.ts

@@ -21,6 +21,7 @@ export interface SystemAssertion {
 export declare const SYSTEM_ASSERTION_IDS: {
     readonly SKILL_WAS_CALLED: "system:skill_was_called";
     readonly BUILD_PASSED: "system:build_passed";
+    readonly TIME_LIMIT: "system:time_limit";
     readonly LLM_JUDGE: "system:llm_judge";
 };
 export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];

package/build/types/scenario/assertions.d.ts

@@ -36,6 +36,15 @@ export declare const LlmJudgeAssertionSchema: z.ZodObject<{
     temperature: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export type LlmJudgeAssertion = z.infer<typeof LlmJudgeAssertionSchema>;
+/**
+ * Assertion: scenario must complete within a maximum duration.
+ * Deterministic check against the scenario execution time.
+ */
+export declare const TimeAssertionSchema: z.ZodObject<{
+    type: z.ZodLiteral<"time_limit">;
+    maxDurationMs: z.ZodNumber;
+}, z.core.$strip>;
+export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
 /**
  * Union of all assertion types (per scenario).
  * Each assertion has a type and type-specific data.
@@ -48,6 +57,9 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"build_passed">;
     command: z.ZodOptional<z.ZodString>;
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"time_limit">;
+    maxDurationMs: z.ZodNumber;
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
     prompt: z.ZodString;

package/build/types/scenario/test-scenario.d.ts

@@ -32,6 +32,9 @@ export declare const TestScenarioSchema: z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"time_limit">;
+        maxDurationMs: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -64,6 +67,9 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"time_limit">;
+        maxDurationMs: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -96,6 +102,9 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
         type: z.ZodLiteral<"build_passed">;
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"time_limit">;
+        maxDurationMs: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.35.0",
+  "version": "0.36.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "be942689dd76b6a6a967bb66a1398535e5d2c86f7b405c99b92e7d12"
+  "falconPackageHash": "1beee538e2fe877b490209a7f00a37f2524d6cece55e7c57bdd0f20a"
 }