@wix/evalforge-types 0.14.0 → 0.15.0

package/build/types/assertion/assertion.d.ts

@@ -0,0 +1,167 @@
+import { z } from 'zod';
+/**
+ * Assertion types:
+ * - skill_was_called: Checks if a specific skill was invoked (deterministic)
+ * - build_passed: Runs a command and checks exit code (deterministic)
+ * - llm_judge: LLM evaluates output with a prompt (LLM-based)
+ * - custom: User-defined LLM-based assertion (uses LlmJudgeEvaluator)
+ */
+export declare const AssertionTypeSchema: z.ZodEnum<{
+    custom: "custom";
+    skill_was_called: "skill_was_called";
+    build_passed: "build_passed";
+    llm_judge: "llm_judge";
+}>;
+export type AssertionType = z.infer<typeof AssertionTypeSchema>;
+/**
+ * Configuration for skill_was_called assertion type.
+ */
+export declare const SkillWasCalledConfigSchema: z.ZodObject<{
+    skillName: z.ZodString;
+}, z.core.$strip>;
+export type SkillWasCalledConfig = z.infer<typeof SkillWasCalledConfigSchema>;
+/**
+ * Configuration for build_passed assertion type.
+ */
+export declare const BuildPassedConfigSchema: z.ZodObject<{
+    command: z.ZodOptional<z.ZodString>;
+    expectedExitCode: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export type BuildPassedConfig = z.infer<typeof BuildPassedConfigSchema>;
+/**
+ * Configuration for llm_judge and custom assertion types.
+ * Custom assertions use the same LLM-based evaluation as llm_judge.
+ */
+export declare const LlmJudgeConfigSchema: z.ZodObject<{
+    prompt: z.ZodString;
+    systemPrompt: z.ZodOptional<z.ZodString>;
+    minScore: z.ZodOptional<z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    maxTokens: z.ZodOptional<z.ZodNumber>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export type LlmJudgeConfig = z.infer<typeof LlmJudgeConfigSchema>;
+/**
+ * Union of all assertion config types.
+ * The actual config shape depends on the assertion type.
+ */
+export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
+    skillName: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    command: z.ZodOptional<z.ZodString>;
+    expectedExitCode: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodObject<{
+    prompt: z.ZodString;
+    systemPrompt: z.ZodOptional<z.ZodString>;
+    minScore: z.ZodOptional<z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    maxTokens: z.ZodOptional<z.ZodNumber>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+export type AssertionConfig = z.infer<typeof AssertionConfigSchema>;
+/**
+ * Custom Assertion entity - stored in the database.
+ * Replaces inline assertions in test scenarios.
+ */
+export declare const CustomAssertionSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    createdAt: z.ZodString;
+    updatedAt: z.ZodString;
+    deleted: z.ZodOptional<z.ZodBoolean>;
+    projectId: z.ZodString;
+    type: z.ZodEnum<{
+        custom: "custom";
+        skill_was_called: "skill_was_called";
+        build_passed: "build_passed";
+        llm_judge: "llm_judge";
+    }>;
+    config: z.ZodUnion<readonly [z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        command: z.ZodOptional<z.ZodString>;
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        prompt: z.ZodString;
+        systemPrompt: z.ZodOptional<z.ZodString>;
+        minScore: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        maxTokens: z.ZodOptional<z.ZodNumber>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+}, z.core.$strip>;
+export type CustomAssertion = z.infer<typeof CustomAssertionSchema>;
+/**
+ * Input schema for creating a new CustomAssertion.
+ */
+export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
+    type: z.ZodEnum<{
+        custom: "custom";
+        skill_was_called: "skill_was_called";
+        build_passed: "build_passed";
+        llm_judge: "llm_judge";
+    }>;
+    name: z.ZodString;
+    description: z.ZodString;
+    projectId: z.ZodString;
+    config: z.ZodUnion<readonly [z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        command: z.ZodOptional<z.ZodString>;
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        prompt: z.ZodString;
+        systemPrompt: z.ZodOptional<z.ZodString>;
+        minScore: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        maxTokens: z.ZodOptional<z.ZodNumber>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>;
+}, z.core.$strip>;
+export type CreateCustomAssertionInput = z.infer<typeof CreateCustomAssertionInputSchema>;
+/**
+ * Input schema for updating a CustomAssertion.
+ */
+export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
+    type: z.ZodOptional<z.ZodEnum<{
+        custom: "custom";
+        skill_was_called: "skill_was_called";
+        build_passed: "build_passed";
+        llm_judge: "llm_judge";
+    }>>;
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    projectId: z.ZodOptional<z.ZodString>;
+    config: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
+        skillName: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        command: z.ZodOptional<z.ZodString>;
+        expectedExitCode: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        prompt: z.ZodString;
+        systemPrompt: z.ZodOptional<z.ZodString>;
+        minScore: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        maxTokens: z.ZodOptional<z.ZodNumber>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{}, z.core.$strip>]>>;
+}, z.core.$strip>;
+export type UpdateCustomAssertionInput = z.infer<typeof UpdateCustomAssertionInputSchema>;
+/**
+ * Helper function to validate config based on assertion type.
+ * Returns true if config is valid for the given type.
+ */
+export declare function validateAssertionConfig(type: AssertionType, config: unknown): boolean;
+/**
+ * Get typed config for skill_was_called assertion.
+ */
+export declare function getSkillWasCalledConfig(assertion: CustomAssertion): SkillWasCalledConfig | null;
+/**
+ * Get typed config for build_passed assertion.
+ */
+export declare function getBuildPassedConfig(assertion: CustomAssertion): BuildPassedConfig | null;
+/**
+ * Get typed config for llm_judge or custom assertion.
+ */
+export declare function getLlmJudgeConfig(assertion: CustomAssertion): LlmJudgeConfig | null;

package/build/types/assertion/index.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Assertion types - custom assertions stored in database.
+ */
+export * from './assertion.js';

package/build/types/index.d.ts

@@ -23,3 +23,4 @@ export * from './evaluation/index.js';
 export * from './project/index.js';
 export * from './template/index.js';
 export * from './agent/index.js';
+export * from './assertion/index.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -46,5 +46,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "4edafe20c61e27855e5740b7e20d9e37d66f4517e726e49cf047798c"
+  "falconPackageHash": "3e43e12908b60b5173ac6b8adb89ff350604e1d30d351b28111b2df7"
 }