@wix/evalforge-types 0.61.0 → 0.62.0

package/build/types/assertion/assertion.d.ts

@@ -7,6 +7,7 @@ import { z } from 'zod';
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
  * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
  * - llm_judge: LLM evaluates output with a prompt (LLM-based, system-level)
+ * - api_call: Makes an HTTP request and validates response with JSON subset matching (deterministic, system-level)
  *
  * Any assertion can be negated by setting `negate: true` to invert the pass/fail logic.
  */
@@ -17,6 +18,7 @@ export declare const AssertionTypeSchema: z.ZodEnum<{
     time_limit: "time_limit";
     cost: "cost";
     llm_judge: "llm_judge";
+    api_call: "api_call";
 }>;
 /**
  * Parameter types supported in assertion parameters.
@@ -127,6 +129,30 @@ export declare const LlmJudgeConfigSchema: z.ZodObject<{
     }, z.core.$strip>>>;
 }, z.core.$strip>;
 export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
+/**
+ * Configuration for api_call assertion type.
+ * Makes an HTTP request after scenario execution and validates the response
+ * contains the expected data using JSON subset matching.
+ * Uses strictObject to reject objects with unknown keys.
+ */
+export declare const ApiCallConfigSchema: z.ZodObject<{
+    /** URL to call */
+    url: z.ZodString;
+    /** HTTP method (default GET) */
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    /** Request body (JSON string, for POST requests) */
+    requestBody: z.ZodOptional<z.ZodString>;
+    /** Expected JSON response to validate against (subset match — extra fields in actual are OK) */
+    expectedResponse: z.ZodString;
+    /** Request headers as JSON string of key-value pairs */
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    /** Request timeout in milliseconds (default 30000) */
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type ApiCallConfig = z.infer<typeof ApiCallConfigSchema>;
 export declare const SkillWasCalledAssertionSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
     negate: z.ZodOptional<z.ZodBoolean>;
@@ -176,6 +202,20 @@ export declare const LlmJudgeAssertionSchema: z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
 }, z.core.$strip>;
 export type LlmJudgeAssertion = z.infer<typeof LlmJudgeAssertionSchema>;
+export declare const ApiCallAssertionSchema: z.ZodObject<{
+    url: z.ZodString;
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    requestBody: z.ZodOptional<z.ZodString>;
+    expectedResponse: z.ZodString;
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"api_call">;
+}, z.core.$strict>;
+export type ApiCallAssertion = z.infer<typeof ApiCallAssertionSchema>;
 export declare const TimeAssertionSchema: z.ZodObject<{
     maxDurationMs: z.ZodNumber;
     negate: z.ZodOptional<z.ZodBoolean>;
@@ -229,7 +269,19 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
     }, z.core.$strip>>>;
     negate: z.ZodOptional<z.ZodBoolean>;
     type: z.ZodLiteral<"llm_judge">;
-}, z.core.$strip>]>;
+}, z.core.$strip>, z.ZodObject<{
+    url: z.ZodString;
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    requestBody: z.ZodOptional<z.ZodString>;
+    expectedResponse: z.ZodString;
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    negate: z.ZodOptional<z.ZodBoolean>;
+    type: z.ZodLiteral<"api_call">;
+}, z.core.$strict>]>;
 export type Assertion = z.infer<typeof AssertionSchema>;
 /**
  * Union of all assertion config types.
@@ -264,6 +316,22 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
     expectedParams: z.ZodOptional<z.ZodString>;
     /** If true, the matching tool call must also have succeeded (step.success === true) */
     requireSuccess: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>, z.ZodObject<{
+    /** URL to call */
+    url: z.ZodString;
+    /** HTTP method (default GET) */
+    method: z.ZodOptional<z.ZodEnum<{
+        GET: "GET";
+        POST: "POST";
+    }>>;
+    /** Request body (JSON string, for POST requests) */
+    requestBody: z.ZodOptional<z.ZodString>;
+    /** Expected JSON response to validate against (subset match — extra fields in actual are OK) */
+    expectedResponse: z.ZodString;
+    /** Request headers as JSON string of key-value pairs */
+    requestHeaders: z.ZodOptional<z.ZodString>;
+    /** Request timeout in milliseconds (default 30000) */
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;

package/build/types/assertion/system-assertions.d.ts

@@ -25,6 +25,7 @@ export declare const SYSTEM_ASSERTION_IDS: {
     readonly TIME_LIMIT: "system:time_limit";
     readonly COST: "system:cost";
     readonly LLM_JUDGE: "system:llm_judge";
+    readonly API_CALL: "system:api_call";
 };
 export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];
 /**

package/build/types/scenario/test-scenario.d.ts

@@ -68,7 +68,19 @@ export declare const TestScenarioSchema: z.ZodObject<{
         }, z.core.$strip>>>;
         negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;
@@ -129,7 +141,19 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
         }, z.core.$strip>>>;
         negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;
@@ -190,7 +214,19 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
         }, z.core.$strip>>>;
         negate: z.ZodOptional<z.ZodBoolean>;
         type: z.ZodLiteral<"llm_judge">;
-    }, z.core.$strip>]>>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        url: z.ZodString;
+        method: z.ZodOptional<z.ZodEnum<{
+            GET: "GET";
+            POST: "POST";
+        }>>;
+        requestBody: z.ZodOptional<z.ZodString>;
+        expectedResponse: z.ZodString;
+        requestHeaders: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+        negate: z.ZodOptional<z.ZodBoolean>;
+        type: z.ZodLiteral<"api_call">;
+    }, z.core.$strict>]>>>>;
     assertionIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
     assertionLinks: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
         assertionId: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.61.0",
+  "version": "0.62.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -46,5 +46,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "1829a0f485cce6f1adaf0ec93b47528012e55ab746b19e3533f5b605"
+  "falconPackageHash": "940ab3ce9b65fa090b0b5759049e93a2c0d6517bbe81a37d3f388678"
 }