@wix/evalforge-types 0.16.0 → 0.18.0

package/build/types/assertion/assertion.d.ts

@@ -1,17 +1,49 @@
 import { z } from 'zod';
 /**
  * Assertion types:
- * - skill_was_called: Checks if a specific skill was invoked (deterministic)
- * - build_passed: Runs a command and checks exit code (deterministic)
- * - llm_judge: LLM evaluates output with a prompt (LLM-based)
- * - custom: User-defined LLM-based assertion (uses LlmJudgeEvaluator)
+ * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
+ * - build_passed: Runs a command and checks exit code (deterministic, system-level)
+ * - llm_judge: LLM evaluates output with a prompt (LLM-based, user-created)
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
-    custom: "custom";
     skill_was_called: "skill_was_called";
     build_passed: "build_passed";
     llm_judge: "llm_judge";
 }>;
+/**
+ * Parameter types supported in assertion parameters.
+ */
+export declare const AssertionParameterTypeSchema: z.ZodEnum<{
+    string: "string";
+    number: "number";
+    boolean: "boolean";
+}>;
+export type AssertionParameterType = z.infer<typeof AssertionParameterTypeSchema>;
+/**
+ * Schema for defining assertion parameters.
+ * Parameters can be required or optional, with optional default values.
+ */
+export declare const AssertionParameterSchema: z.ZodObject<{
+    name: z.ZodString;
+    label: z.ZodString;
+    type: z.ZodEnum<{
+        string: "string";
+        number: "number";
+        boolean: "boolean";
+    }>;
+    required: z.ZodBoolean;
+    defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+}, z.core.$strip>;
+export type AssertionParameter = z.infer<typeof AssertionParameterSchema>;
+/**
+ * Schema for scenario-assertion link with parameter values.
+ * Used when linking assertions to scenarios with specific parameter values.
+ */
+export declare const ScenarioAssertionLinkSchema: z.ZodObject<{
+    assertionId: z.ZodString;
+    params: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>>>;
+}, z.core.$strip>;
+export type ScenarioAssertionLink = z.infer<typeof ScenarioAssertionLinkSchema>;
 export type AssertionType = z.infer<typeof AssertionTypeSchema>;
 /**
  * Configuration for skill_was_called assertion type.
@@ -22,15 +54,18 @@ export declare const SkillWasCalledConfigSchema: z.ZodObject<{
 export type SkillWasCalledConfig = z.infer<typeof SkillWasCalledConfigSchema>;
 /**
  * Configuration for build_passed assertion type.
+ * Uses strictObject to reject objects with unknown keys (prevents matching LlmJudge configs).
  */
 export declare const BuildPassedConfigSchema: z.ZodObject<{
+    /** Command to run (default: "yarn build") */
     command: z.ZodOptional<z.ZodString>;
+    /** Expected exit code (default: 0) */
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
-}, z.core.$strip>;
+}, z.core.$strict>;
 export type BuildPassedConfig = z.infer<typeof BuildPassedConfigSchema>;
 /**
- * Configuration for llm_judge and custom assertion types.
- * Custom assertions use the same LLM-based evaluation as llm_judge.
+ * Configuration for llm_judge assertion type.
+ * User-created assertions with customizable parameters.
  */
 export declare const LlmJudgeConfigSchema: z.ZodObject<{
     prompt: z.ZodString;
@@ -39,25 +74,51 @@ export declare const LlmJudgeConfigSchema: z.ZodObject<{
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     temperature: z.ZodOptional<z.ZodNumber>;
+    parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        label: z.ZodString;
+        type: z.ZodEnum<{
+            string: "string";
+            number: "number";
+            boolean: "boolean";
+        }>;
+        required: z.ZodBoolean;
+        defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+    }, z.core.$strip>>>;
 }, z.core.$strip>;
 export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
 /**
  * Union of all assertion config types.
- * The actual config shape depends on the assertion type.
+ * Order matters: schemas with required fields first, then optional-only schemas.
+ * This ensures LlmJudge (requires prompt) and SkillWasCalled (requires skillName)
+ * are matched before BuildPassed (all optional) or empty object.
  */
 export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
-    skillName: z.ZodString;
-}, z.core.$strip>, z.ZodObject<{
-    command: z.ZodOptional<z.ZodString>;
-    expectedExitCode: z.ZodOptional<z.ZodNumber>;
-}, z.core.$strip>, z.ZodObject<{
     prompt: z.ZodString;
     systemPrompt: z.ZodOptional<z.ZodString>;
     minScore: z.ZodOptional<z.ZodNumber>;
     model: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     temperature: z.ZodOptional<z.ZodNumber>;
-}, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+    parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        label: z.ZodString;
+        type: z.ZodEnum<{
+            string: "string";
+            number: "number";
+            boolean: "boolean";
+        }>;
+        required: z.ZodBoolean;
+        defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+    }, z.core.$strip>>>;
+}, z.core.$strip>, z.ZodObject<{
+    skillName: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    /** Command to run (default: "yarn build") */
+    command: z.ZodOptional<z.ZodString>;
+    /** Expected exit code (default: 0) */
+    expectedExitCode: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
 export type AssertionConfig = z.infer<typeof AssertionConfigSchema>;
 /**
  * Custom Assertion entity - stored in the database.
@@ -72,24 +133,36 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     deleted: z.ZodOptional<z.ZodBoolean>;
     projectId: z.ZodString;
     type: z.ZodEnum<{
-        custom: "custom";
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
     }>;
     config: z.ZodUnion<readonly [z.ZodObject<{
-        skillName: z.ZodString;
-    }, z.core.$strip>, z.ZodObject<{
-        command: z.ZodOptional<z.ZodString>;
-        expectedExitCode: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{
         prompt: z.ZodString;
         systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
         temperature: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+        parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodString;
+            type: z.ZodEnum<{
+                string: "string";
+                number: "number";
+                boolean: "boolean";
+            }>;
+            required: z.ZodBoolean;
+            defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        /** Command to run (default: "yarn build") */
+        command: z.ZodOptional<z.ZodString>;
+        /** Expected exit code (default: 0) */
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
 }, z.core.$strip>;
 export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
 /**
@@ -97,7 +170,6 @@ export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
  */
 export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodEnum<{
-        custom: "custom";
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
@@ -106,18 +178,31 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     description: z.ZodString;
     projectId: z.ZodString;
     config: z.ZodUnion<readonly [z.ZodObject<{
-        skillName: z.ZodString;
-    }, z.core.$strip>, z.ZodObject<{
-        command: z.ZodOptional<z.ZodString>;
-        expectedExitCode: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{
         prompt: z.ZodString;
         systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
         temperature: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+        parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodString;
+            type: z.ZodEnum<{
+                string: "string";
+                number: "number";
+                boolean: "boolean";
+            }>;
+            required: z.ZodBoolean;
+            defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        /** Command to run (default: "yarn build") */
+        command: z.ZodOptional<z.ZodString>;
+        /** Expected exit code (default: 0) */
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>;
 }, z.core.$strip>;
 export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInputSchema>;
 /**
@@ -125,7 +210,6 @@ export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInp
  */
 export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodEnum<{
-        custom: "custom";
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
         llm_judge: "llm_judge";
@@ -134,18 +218,31 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     description: z.ZodOptional<z.ZodString>;
     projectId: z.ZodOptional<z.ZodString>;
     config: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
-        skillName: z.ZodString;
-    }, z.core.$strip>, z.ZodObject<{
-        command: z.ZodOptional<z.ZodString>;
-        expectedExitCode: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{
         prompt: z.ZodString;
         systemPrompt: z.ZodOptional<z.ZodString>;
         minScore: z.ZodOptional<z.ZodNumber>;
         model: z.ZodOptional<z.ZodString>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
         temperature: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>>;
+        parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodString;
+            type: z.ZodEnum<{
+                string: "string";
+                number: "number";
+                boolean: "boolean";
+            }>;
+            required: z.ZodBoolean;
+            defaultValue: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>, z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        /** Command to run (default: "yarn build") */
+        command: z.ZodOptional<z.ZodString>;
+        /** Expected exit code (default: 0) */
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{}, z.core.$strip>]>>;
 }, z.core.$strip>;
 export type UpdateCustomAssertionInput = z.infer<typeof UpdateCustomAssertionInputSchema>;
 /**
@@ -162,6 +259,6 @@ export declare function getSkillWasCalledConfig(assertion: CustomAssertion): Ski
  */
 export declare function getBuildPassedConfig(assertion: CustomAssertion): BuildPassedConfig | null;
 /**
- * Get typed config for llm_judge or custom assertion.
+ * Get typed config for llm_judge assertion.
  */
 export declare function getLlmJudgeConfig(assertion: CustomAssertion): LlmJudgeConfig | null;

package/build/types/assertion/index.d.ts

@@ -1,4 +1,5 @@
 /**
- * Assertion types - custom assertions stored in database.
+ * Assertion types - custom assertions stored in database and system assertions.
  */
 export * from './assertion.js';
+export * from './system-assertions.js';

package/build/types/assertion/system-assertions.d.ts

@@ -0,0 +1,42 @@
+import type { AssertionParameter, AssertionType } from './assertion.js';
+/**
+ * System assertion definition - built-in assertions available to all projects.
+ * These are not stored in the database but defined in code.
+ */
+export interface SystemAssertion {
+    /** Unique ID prefixed with 'system:' */
+    id: string;
+    /** Display name */
+    name: string;
+    /** Description of what the assertion checks */
+    description: string;
+    /** Assertion type */
+    type: AssertionType;
+    /** Parameters that can be configured per-scenario */
+    parameters: AssertionParameter[];
+}
+/**
+ * System assertion IDs - prefixed with 'system:' to distinguish from custom assertions.
+ */
+export declare const SYSTEM_ASSERTION_IDS: {
+    readonly SKILL_WAS_CALLED: "system:skill_was_called";
+    readonly BUILD_PASSED: "system:build_passed";
+};
+export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];
+/**
+ * Check if an assertion ID is a system assertion.
+ */
+export declare function isSystemAssertionId(id: string): id is SystemAssertionId;
+/**
+ * Built-in system assertions.
+ * These are available to all projects without needing to create them.
+ */
+export declare const SYSTEM_ASSERTIONS: Record<SystemAssertionId, SystemAssertion>;
+/**
+ * Get all system assertions as an array.
+ */
+export declare function getSystemAssertions(): SystemAssertion[];
+/**
+ * Get a system assertion by ID.
+ */
+export declare function getSystemAssertion(id: SystemAssertionId): SystemAssertion | undefined;

package/build/types/scenario/test-scenario.d.ts

@@ -42,6 +42,10 @@ export declare const TestScenarioSchema: z.ZodObject<{
         temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        assertionId: z.ZodString;
+        params: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>>>;
+    }, z.core.$strip>>>;
 }, z.core.$strip>;
 export type TestScenario = z.infer<typeof TestScenarioSchema>;
 /**
@@ -70,6 +74,10 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
         temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>;
     assertionIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    assertionLinks: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        assertionId: z.ZodString;
+        params: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>>>;
+    }, z.core.$strip>>>;
 }, z.core.$strip>;
 export type CreateTestScenarioInput = z.infer<typeof CreateTestScenarioInputSchema>;
 /**
@@ -98,5 +106,9 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
         temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>>;
     assertionIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    assertionLinks: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
+        assertionId: z.ZodString;
+        params: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean, z.ZodNull]>>>;
+    }, z.core.$strip>>>>;
 }, z.core.$strip>;
 export type UpdateTestScenarioInput = z.infer<typeof UpdateTestScenarioInputSchema>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -46,5 +46,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "998d2b657ab6cce997be349567ed0aeec0dce0593ef672dcfcd6805d"
+  "falconPackageHash": "99055e21a0b36ab3ea066f8ffe6d6f2e0c8b2458f81f89cec56968a6"
 }