@wix/evalforge-types 0.36.0 → 0.38.0

package/build/types/agent/adapter.d.ts

@@ -5,6 +5,7 @@ import type { ModelConfig } from '../common/models.js';
 import type { LLMTrace } from '../evaluation/metrics.js';
 import type { MCPEntity } from '../common/mcp.js';
 import type { SubAgent } from '../target/sub-agent.js';
+import type { Rule } from '../common/rule.js';
 /**
  * Trace context for live streaming of agent execution.
  * This is agent-agnostic and can be used by any adapter implementation.
@@ -57,6 +58,8 @@ export interface AgentExecutionContext {
     mcps?: MCPEntity[];
     /** Sub-agents to load (when present, written to .claude/agents/*.md) */
     subAgents?: SubAgent[];
+    /** Rules to write (CLAUDE.md, AGENTS.md, .cursor/rules/*.md based on ruleType) */
+    rules?: Rule[];
 }
 /**
  * Token usage statistics from agent execution.

package/build/types/assertion/assertion.d.ts

@@ -4,11 +4,13 @@ import { z } from 'zod';
  * - skill_was_called: Checks if a specific skill was invoked (deterministic, system-level)
  * - build_passed: Runs a command and checks exit code (deterministic, system-level)
  * - time_limit: Checks that scenario completed within a duration threshold (deterministic, system-level)
+ * - cost: Checks that scenario LLM cost stays within a USD threshold (deterministic, system-level)
  * - llm_judge: LLM evaluates output with a prompt (LLM-based, user-created)
  */
 export declare const AssertionTypeSchema: z.ZodEnum<{
     skill_was_called: "skill_was_called";
     build_passed: "build_passed";
+    cost: "cost";
     llm_judge: "llm_judge";
     time_limit: "time_limit";
 }>;
@@ -57,6 +59,15 @@ export declare const SkillWasCalledConfigSchema: z.ZodObject<{
     skillNames: z.ZodArray<z.ZodString>;
 }, z.core.$strip>;
 export type SkillWasCalledConfig = z.infer<typeof SkillWasCalledConfigSchema>;
+/**
+ * Configuration for cost assertion type.
+ * Uses strictObject to reject objects with unknown keys (prevents matching other configs).
+ */
+export declare const CostConfigSchema: z.ZodObject<{
+    /** Maximum allowed cost in USD */
+    maxCostUsd: z.ZodNumber;
+}, z.core.$strict>;
+export type CostConfig = z.infer<typeof CostConfigSchema>;
 /**
  * Configuration for build_passed assertion type.
  * Uses strictObject to reject objects with unknown keys (prevents matching LlmJudge configs).
@@ -132,6 +143,9 @@ export declare const AssertionConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     /** Maximum allowed duration in milliseconds */
     maxDurationMs: z.ZodNumber;
+}, z.core.$strict>, z.ZodObject<{
+    /** Maximum allowed cost in USD */
+    maxCostUsd: z.ZodNumber;
 }, z.core.$strict>, z.ZodObject<{
     /** Command to run (default: "yarn build") */
     command: z.ZodOptional<z.ZodString>;
@@ -154,6 +168,7 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>;
@@ -181,6 +196,9 @@ export declare const CustomAssertionSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
@@ -196,6 +214,7 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>;
@@ -226,6 +245,9 @@ export declare const CreateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;
@@ -241,6 +263,7 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     type: z.ZodOptional<z.ZodEnum<{
         skill_was_called: "skill_was_called";
         build_passed: "build_passed";
+        cost: "cost";
         llm_judge: "llm_judge";
         time_limit: "time_limit";
     }>>;
@@ -271,6 +294,9 @@ export declare const UpdateCustomAssertionInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         /** Maximum allowed duration in milliseconds */
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strict>, z.ZodObject<{
+        /** Maximum allowed cost in USD */
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         /** Command to run (default: "yarn build") */
         command: z.ZodOptional<z.ZodString>;

package/build/types/assertion/system-assertions.d.ts

@@ -22,6 +22,7 @@ export declare const SYSTEM_ASSERTION_IDS: {
     readonly SKILL_WAS_CALLED: "system:skill_was_called";
     readonly BUILD_PASSED: "system:build_passed";
     readonly TIME_LIMIT: "system:time_limit";
+    readonly COST: "system:cost";
     readonly LLM_JUDGE: "system:llm_judge";
 };
 export type SystemAssertionId = (typeof SYSTEM_ASSERTION_IDS)[keyof typeof SYSTEM_ASSERTION_IDS];

package/build/types/common/index.d.ts

@@ -2,3 +2,4 @@ export * from './base-entity.js';
 export * from './github-source.js';
 export * from './mcp.js';
 export * from './models.js';
+export * from './rule.js';

package/build/types/common/rule.d.ts

@@ -0,0 +1,47 @@
+import { z } from 'zod';
+export declare const RuleTypeSchema: z.ZodEnum<{
+    "claude-md": "claude-md";
+    "agents-md": "agents-md";
+    "cursor-rule": "cursor-rule";
+}>;
+export type RuleType = z.infer<typeof RuleTypeSchema>;
+export declare const RuleSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    createdAt: z.ZodString;
+    updatedAt: z.ZodString;
+    deleted: z.ZodOptional<z.ZodBoolean>;
+    projectId: z.ZodString;
+    ruleType: z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>;
+    content: z.ZodString;
+}, z.core.$strip>;
+export type Rule = z.infer<typeof RuleSchema>;
+export declare const CreateRuleInputSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    projectId: z.ZodString;
+    ruleType: z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>;
+    content: z.ZodString;
+}, z.core.$strip>;
+export type CreateRuleInput = z.infer<typeof CreateRuleInputSchema>;
+export declare const UpdateRuleInputSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    projectId: z.ZodOptional<z.ZodString>;
+    ruleType: z.ZodOptional<z.ZodEnum<{
+        "claude-md": "claude-md";
+        "agents-md": "agents-md";
+        "cursor-rule": "cursor-rule";
+    }>>;
+    content: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type UpdateRuleInput = z.infer<typeof UpdateRuleInputSchema>;

package/build/types/evaluation/eval-run.d.ts

@@ -495,6 +495,7 @@ export declare const EvalRunSchema: z.ZodObject<{
     jobStatusCheckedAt: z.ZodOptional<z.ZodString>;
     mcpIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     subAgentIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export type EvalRun = z.infer<typeof EvalRunSchema>;
 /**
@@ -612,6 +613,7 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
     jobStatusCheckedAt: z.ZodOptional<z.ZodString>;
     mcpIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
     subAgentIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export type CreateEvalRunInput = z.infer<typeof CreateEvalRunInputSchema>;
 /**

package/build/types/scenario/assertions.d.ts

@@ -21,6 +21,15 @@ export declare const BuildPassedAssertionSchema: z.ZodObject<{
     expectedExitCode: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export type BuildPassedAssertion = z.infer<typeof BuildPassedAssertionSchema>;
+/**
+ * Assertion: the scenario LLM execution cost must stay within a USD threshold.
+ * Checked by reading llmTrace.summary.totalCostUsd.
+ */
+export declare const CostAssertionSchema: z.ZodObject<{
+    type: z.ZodLiteral<"cost">;
+    maxCostUsd: z.ZodNumber;
+}, z.core.$strip>;
+export type CostAssertion = z.infer<typeof CostAssertionSchema>;
 /**
  * Assertion: an LLM judges the scenario output (score 0-100).
  * Prompt can use {{output}}, {{cwd}}, {{changedFiles}}, {{trace}}.
@@ -60,6 +69,9 @@ export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"time_limit">;
     maxDurationMs: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"cost">;
+    maxCostUsd: z.ZodNumber;
 }, z.core.$strip>, z.ZodObject<{
     type: z.ZodLiteral<"llm_judge">;
     prompt: z.ZodString;

package/build/types/scenario/test-scenario.d.ts

@@ -35,6 +35,9 @@ export declare const TestScenarioSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -70,6 +73,9 @@ export declare const CreateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;
@@ -105,6 +111,9 @@ export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"time_limit">;
         maxDurationMs: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"cost">;
+        maxCostUsd: z.ZodNumber;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"llm_judge">;
         prompt: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.36.0",
+  "version": "0.38.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "1beee538e2fe877b490209a7f00a37f2524d6cece55e7c57bdd0f20a"
+  "falconPackageHash": "dc3f87434a7b2a1b350369f78ca98d0fc64183d39b35b8a111054878"
 }