@wix/evalforge-types 0.60.0 → 0.62.0

package/build/types/assertion/assertion.d.ts

@@ -7,6 +7,9 @@ import { z } from 'zod';
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
  * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
  * - llm_judge: LLM evaluates output with a prompt (LLM-based, system-level)
+ * - api_call: Makes an HTTP request and validates response with JSON subset matching (deterministic, system-level)
+ *
+ * Any assertion can be negated by setting `negate: true` to invert the pass/fail logic.
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
@@ -15,6 +18,7 @@ export declare const AssertionTypeSchema: z.ZodEnum<{
     time_limit: "time_limit";
     cost: "cost";
     llm_judge: "llm_judge";
+    api_call: "api_call";
 }>;
 /**
  * Parameter types supported in assertion parameters.
@@ -72,12 +76,13 @@ export declare const CostConfigSchema: z.ZodObject<{
 export type CostConfig = z.infer<typeof CostConfigSchema>;
 /** Configuration for tool_called_with_param assertion type.
  * Uses strictObject to reject objects with unknown keys.
+ * When expectedParams is omitted, the assertion checks only that the tool was called (or not, if negated).
  */
 export declare const ToolCalledWithParamConfigSchema: z.ZodObject<{
     /** Name of the tool that must have been called */
     toolName: z.ZodString;
-    /** JSON string of key-value pairs for expected parameters (substring match) */
-    expectedParams: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
+    expectedParams: z.ZodOptional<z.ZodString>;
     /** If true, the matching tool call must also have succeeded (step.success === true) */
     requireSuccess: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strict>;
@@ -124,26 +129,54 @@ export declare const LlmJudgeConfigSchema: z.ZodObject<{
     }, z.core.$strip>>>;
 }, z.core.$strip>;
 export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
+/**
+ * Configuration for api_call assertion type.
+ * Makes an HTTP request after scenario execution and validates the response
+ * contains the expected data using JSON subset matching.
+ * Uses strictObject to reject objects with unknown keys.
+ */
+export declare const ApiCallConfigSchema: z.ZodObject<{
+    /** URL to call */
+    url: z.ZodString;
+    /** HTTP method (default GET) */
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    /** Request body (JSON string, for POST requests) */
+    requestBody: z.ZodOptional<z.ZodString>;
+    /** Expected JSON response to validate against (subset match — extra fields in actual are OK) */
+    expectedResponse: z.ZodString;
+    /** Request headers as JSON string of key-value pairs */
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    /** Request timeout in milliseconds (default 30000) */
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type ApiCallConfig = z.infer<typeof ApiCallConfigSchema>;
 export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"skill_was_called">;
 }, z.core.$strip>;
 export type SkillWasCalledAssertion = z.infer<typeof SkillWasCalledAssertionSchema>;
 export declare const ToolCalledWithParamAssertionSchema: z.ZodObject<{
     toolName: z.ZodString;
-    expectedParams: z.ZodString;
+    expectedParams: z.ZodOptional<z.ZodString>;
     requireSuccess: z.ZodOptional<z.ZodBoolean>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"tool_called_with_param">;
 }, z.core.$strict>;
 export type ToolCalledWithParamAssertion = z.infer<typeof ToolCalledWithParamAssertionSchema>;
 export declare const BuildPassedAssertionSchema: z.ZodObject<{
     command: z.ZodOptional<z.ZodString>;
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"build_passed">;
 }, z.core.$strict>;
 export type BuildPassedAssertion = z.infer<typeof BuildPassedAssertionSchema>;
 export declare const CostAssertionSchema: z.ZodObject<{
     maxCostUsd: z.ZodNumber;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"cost">;
 }, z.core.$strict>;
 export type CostAssertion = z.infer<typeof CostAssertionSchema>;
@@ -165,11 +198,27 @@ export declare const LlmJudgeAssertionSchema: z.ZodObject<{
         defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
         advanced: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"llm_judge">;
 }, z.core.$strip>;
 export type LlmJudgeAssertion = z.infer<typeof LlmJudgeAssertionSchema>;
+export declare const ApiCallAssertionSchema: z.ZodObject<{
+    url: z.ZodString;
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    requestBody: z.ZodOptional<z.ZodString>;
+    expectedResponse: z.ZodString;
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"api_call">;
+}, z.core.$strict>;
+export type ApiCallAssertion = z.infer<typeof ApiCallAssertionSchema>;
 export declare const TimeAssertionSchema: z.ZodObject<{
     maxDurationMs: z.ZodNumber;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"time_limit">;
 }, z.core.$strict>;
 export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
@@ -179,21 +228,26 @@ export type TimeAssertion = z.infer<typeof TimeAssertionSchema>;
  */
 export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"skill_was_called">;
 }, z.core.$strip>, z.ZodObject<{
     toolName: z.ZodString;
-    expectedParams: z.ZodString;
+    expectedParams: z.ZodOptional<z.ZodString>;
     requireSuccess: z.ZodOptional<z.ZodBoolean>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"tool_called_with_param">;
 }, z.core.$strict>, z.ZodObject<{
     command: z.ZodOptional<z.ZodString>;
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"build_passed">;
 }, z.core.$strict>, z.ZodObject<{
     maxDurationMs: z.ZodNumber;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"time_limit">;
 }, z.core.$strict>, z.ZodObject<{
     maxCostUsd: z.ZodNumber;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"cost">;
 }, z.core.$strict>, z.ZodObject<{
     prompt: z.ZodString;
@@ -213,8 +267,21 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
         defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
         advanced: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>>;
+    negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"llm_judge">;
-}, z.core.$strip>]>;
+}, z.core.$strip>, z.ZodObject<{
+    url: z.ZodString;
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    requestBody: z.ZodOptional<z.ZodString>;
+    expectedResponse: z.ZodString;
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"api_call">;
+}, z.core.$strict>]>;
 export type Assertion = z.infer<typeof AssertionSchema>;
 /**
  * Union of all assertion config types.
@@ -245,10 +312,26 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     /** Name of the tool that must have been called */
     toolName: z.ZodString;
-    /** JSON string of key-value pairs for expected parameters (substring match) */
-    expectedParams: z.ZodString;
+    /** JSON string of key-value pairs for expected parameters (substring match). Optional — when omitted, only checks tool presence. */
+    expectedParams: z.ZodOptional<z.ZodString>;
     /** If true, the matching tool call must also have succeeded (step.success === true) */
     requireSuccess: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>, z.ZodObject<{
+    /** URL to call */
+    url: z.ZodString;
+    /** HTTP method (default GET) */
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    /** Request body (JSON string, for POST requests) */
+    requestBody: z.ZodOptional<z.ZodString>;
+    /** Expected JSON response to validate against (subset match — extra fields in actual are OK) */
+    expectedResponse: z.ZodString;
+    /** Request headers as JSON string of key-value pairs */
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    /** Request timeout in milliseconds (default 30000) */
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;

package/build/types/assertion/system-assertions.d.ts

@@ -25,6 +25,7 @@ export declare const SYSTEM_ASSERTION_IDS: {
     readonly TIME_LIMIT: "system:time_limit";
     readonly COST: "system:cost";
     readonly LLM_JUDGE: "system:llm_judge";
+    readonly API_CALL: "system:api_call";
 };
 export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];
 /**

package/build/types/scenario/test-scenario.d.ts

@@ -27,21 +27,26 @@ export declare const TestScenarioSchema: z.ZodObject<{
     templateId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"skill_was_called">;
     }, z.core.$strip>, z.ZodObject<{
         toolName: z.ZodString;
-        expectedParams: z.ZodString;
+        expectedParams: z.ZodOptional<z.ZodString>;
         requireSuccess: z.ZodOptional<z.ZodBoolean>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"tool_called_with_param">;
     }, z.core.$strict>, z.ZodObject<{
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"build_passed">;
     }, z.core.$strict>, z.ZodObject<{
         maxDurationMs: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"time_limit">;
     }, z.core.$strict>, z.ZodObject<{
         maxCostUsd: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"cost">;
     }, z.core.$strict>, z.ZodObject<{
         prompt: z.ZodString;
@@ -61,8 +66,21 @@ export declare const TestScenarioSchema: z.ZodObject<{
             defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
             advanced: z.ZodOptional<z.ZodBoolean>;
         }, z.core.$strip>>>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;
@@ -82,21 +100,26 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     triggerPrompt: z.ZodString;
     assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"skill_was_called">;
     }, z.core.$strip>, z.ZodObject<{
         toolName: z.ZodString;
-        expectedParams: z.ZodString;
+        expectedParams: z.ZodOptional<z.ZodString>;
         requireSuccess: z.ZodOptional<z.ZodBoolean>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"tool_called_with_param">;
     }, z.core.$strict>, z.ZodObject<{
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"build_passed">;
     }, z.core.$strict>, z.ZodObject<{
         maxDurationMs: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"time_limit">;
     }, z.core.$strict>, z.ZodObject<{
         maxCostUsd: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"cost">;
     }, z.core.$strict>, z.ZodObject<{
         prompt: z.ZodString;
@@ -116,8 +139,21 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
             defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
             advanced: z.ZodOptional<z.ZodBoolean>;
         }, z.core.$strip>>>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;
@@ -137,21 +173,26 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     triggerPrompt: z.ZodOptional<z.ZodString>;
     assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         skillNames: z.ZodArray<z.ZodString>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"skill_was_called">;
     }, z.core.$strip>, z.ZodObject<{
         toolName: z.ZodString;
-        expectedParams: z.ZodString;
+        expectedParams: z.ZodOptional<z.ZodString>;
         requireSuccess: z.ZodOptional<z.ZodBoolean>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"tool_called_with_param">;
     }, z.core.$strict>, z.ZodObject<{
         command: z.ZodOptional<z.ZodString>;
         expectedExitCode: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"build_passed">;
     }, z.core.$strict>, z.ZodObject<{
         maxDurationMs: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"time_limit">;
     }, z.core.$strict>, z.ZodObject<{
         maxCostUsd: z.ZodNumber;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"cost">;
     }, z.core.$strict>, z.ZodObject<{
         prompt: z.ZodString;
@@ -171,8 +212,21 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
             defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
             advanced: z.ZodOptional<z.ZodBoolean>;
         }, z.core.$strip>>>;
+        negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>>;
     assertionIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
     assertionLinks: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.60.0",
+  "version": "0.62.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -46,5 +46,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "f5906c2fb77b34a5faf6bead27516ae314576b92df11ade8fa2580bd"
+  "falconPackageHash": "940ab3ce9b65fa090b0b5759049e93a2c0d6517bbe81a37d3f388678"
 }