@wix/evalforge-types 0.41.0 → 0.43.0

package/build/types/agent/adapter.d.ts

@@ -1,11 +1,11 @@
 import type { SkillWithLatestVersion } from '../target/skill.js';
-import type { AgentRunCommand } from '../target/agent.js';
 import type { TestScenario } from '../scenario/test-scenario.js';
 import type { ModelConfig } from '../common/models.js';
 import type { LLMTrace } from '../evaluation/metrics.js';
 import type { MCPEntity } from '../common/mcp.js';
 import type { SubAgent } from '../target/sub-agent.js';
 import type { Rule } from '../common/rule.js';
+import type { AgentRunCommand } from '../target/agent.js';
 /**
  * Trace context for live streaming of agent execution.
  * This is agent-agnostic and can be used by any adapter implementation.
@@ -60,6 +60,13 @@ export interface AgentExecutionContext {
     subAgents?: SubAgent[];
     /** Rules to write (CLAUDE.md, AGENTS.md, .cursor/rules/*.md based on ruleType) */
     rules?: Rule[];
+    /**
+     * System prompt override for evaluation runs.
+     * - undefined: use default evaluator behavioral instructions
+     * - null: no system prompt (raw agent behavior)
+     * - string: custom system prompt text
+     */
+    systemPrompt?: string | null;
 }
 /**
  * Token usage statistics from agent execution.
@@ -107,23 +114,29 @@ export interface AgentExecutionResult {
  *
  * @example
  * ```typescript
+ * // CLI-based adapter (looked up by command)
  * class CursorAdapter implements AgentAdapter {
  *   readonly id = 'cursor';
  *   readonly name = 'Cursor CLI';
- *   readonly supportedCommands = [AgentRunCommand.CURSOR];
+ *   readonly supportedCommands = [AgentRunCommand.CLAUDE];
+ *   async execute(context: AgentExecutionContext): Promise<AgentExecutionResult> { ... }
+ * }
  *
- *   async execute(context: AgentExecutionContext): Promise<AgentExecutionResult> {
- *     // Implementation
- *   }
+ * // SDK-based adapter (looked up by adapter ID)
+ * class SimpleAgentAdapter implements AgentAdapter {
+ *   readonly id = 'simple-agent';
+ *   readonly name = 'Simple Agent';
+ *   readonly supportedCommands: AgentRunCommand[] = [];
+ *   async execute(context: AgentExecutionContext): Promise<AgentExecutionResult> { ... }
  * }
  * ```
  */
 export interface AgentAdapter {
-    /** Unique identifier for this adapter */
+    /** Unique identifier for this adapter (used for SDK agent lookup) */
     readonly id: string;
     /** Human-readable name for display */
     readonly name: string;
-    /** CLI commands this adapter handles (e.g., [AgentRunCommand.CLAUDE]) */
+    /** CLI commands this adapter handles (e.g., ['claude']). Can be empty for SDK-only adapters. */
     readonly supportedCommands: readonly AgentRunCommand[];
     /**
      * Execute a skill against a test scenario using this agent.

package/build/types/common/models.d.ts

@@ -10,6 +10,10 @@ export declare const AVAILABLE_OPENAI_MODEL_IDS: OpenAIModel[];
 export declare const OpenAIModelSchema: z.ZodEnum<{
     [x: string]: string;
 }>;
+export declare const ALL_AVAILABLE_MODEL_IDS: string[];
+export declare const AnyModelSchema: z.ZodEnum<{
+    [x: string]: string;
+}>;
 export declare const ModelConfigSchema: z.ZodObject<{
     model: z.ZodEnum<{
         [x: string]: string;

package/build/types/index.d.ts

@@ -7,7 +7,7 @@
  * - BaseEntity: id, name, description, dates
  * - TenantEntity: extends BaseEntity with projectId
  * - Target: extends TenantEntity (base for testable entities)
- *   - Agent: CLI-based agent (runCommand, modelConfig)
+ *   - Agent: CLI-based agent (runCommand, modelConfig, systemPrompt)
  *   - Skill: SKILL.md-based capability
  *
  * Test Types (9 total):

package/build/types/target/agent.d.ts

@@ -1,9 +1,31 @@
 import { z } from 'zod';
+/**
+ * Default behavioral instructions appended to the Claude Code system prompt
+ * during evaluation runs. These ensure the agent executes autonomously without
+ * waiting for human confirmation.
+ */
+export declare const DEFAULT_EVALUATOR_SYSTEM_PROMPT = "IMPORTANT: This is an automated evaluation run. Follow these guidelines:\n1. Execute the requested changes immediately without asking for confirmation.\n2. Do NOT ask \"would you like me to proceed?\" or similar questions.\n3. Do NOT use the Task tool to delegate simple operations - do them directly yourself.\n4. Keep your approach simple and direct - avoid excessive planning.\n5. Make targeted edits using Read and Edit tools rather than exploring the entire codebase.\n6. If you encounter an error, fix it directly rather than starting over.\n7. Your project root is the current working directory. Always create and modify source code files relative to the project root, NOT inside .claude/skills/ directories.\n8. Before finishing, run the project's package manager install command (e.g. `npm install`, `yarn install`, or `pnpm install` depending on the lockfile present) to ensure all dependencies are installed and the project is ready to build.";
+/**
+ * Agent type discriminator.
+ * - CLI: External CLI tool (e.g. Claude Code, Codex)
+ * - SDK: In-process SDK agent (e.g. Simple Agent via AI SDK)
+ */
+export declare const AgentType: {
+    readonly CLI: "cli";
+    readonly SDK: "sdk";
+};
+export type AgentTypeValue = (typeof AgentType)[keyof typeof AgentType];
+export declare const AgentTypeSchema: z.ZodEnum<{
+    cli: "cli";
+    sdk: "sdk";
+}>;
+/** Labels for agent types in UI dropdowns. */
+export declare const AGENT_TYPE_LABELS: Record<AgentTypeValue, string>;
 /**
  * Supported agent CLI commands.
  *
  * Each value corresponds to a registered AgentAdapter in the evaluator.
- * When adding a new agent adapter, add its command here first.
+ * When adding a new CLI agent adapter, add its command here first.
  */
 export declare enum AgentRunCommand {
     CLAUDE = "claude"
@@ -14,10 +36,10 @@ export declare const AVAILABLE_RUN_COMMANDS: AgentRunCommand[];
 export declare const RUN_COMMAND_LABELS: Record<AgentRunCommand, string>;
 export declare const AgentRunCommandSchema: z.ZodEnum<typeof AgentRunCommand>;
 /**
- * Agent schema - a CLI-based coding agent.
+ * Agent schema.
  *
- * Agents are external CLI tools that can execute coding tasks.
- * Examples: Claude Code CLI, Codex CLI, Cursor CLI.
+ * Agents can be CLI-based (external process) or SDK-based (in-process).
+ * CLI agents use runCommand to select the adapter; SDK agents use their adapter ID.
  */
 export declare const AgentSchema: z.ZodObject<{
     id: z.ZodString;
@@ -27,7 +49,11 @@ export declare const AgentSchema: z.ZodObject<{
     updatedAt: z.ZodString;
     deleted: z.ZodOptional<z.ZodBoolean>;
     projectId: z.ZodString;
-    runCommand: z.ZodEnum<typeof AgentRunCommand>;
+    agentType: z.ZodDefault<z.ZodEnum<{
+        cli: "cli";
+        sdk: "sdk";
+    }>>;
+    runCommand: z.ZodOptional<z.ZodEnum<typeof AgentRunCommand>>;
     modelConfig: z.ZodOptional<z.ZodObject<{
         model: z.ZodEnum<{
             [x: string]: string;
@@ -35,6 +61,7 @@ export declare const AgentSchema: z.ZodObject<{
         temperature: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
     }, z.core.$strip>>;
+    systemPrompt: z.ZodOptional<z.ZodNullable<z.ZodString>>;
 }, z.core.$strip>;
 export type Agent = z.infer<typeof AgentSchema>;
 /**
@@ -44,7 +71,11 @@ export declare const CreateAgentInputSchema: z.ZodObject<{
     name: z.ZodString;
     description: z.ZodString;
     projectId: z.ZodString;
-    runCommand: z.ZodEnum<typeof AgentRunCommand>;
+    agentType: z.ZodDefault<z.ZodEnum<{
+        cli: "cli";
+        sdk: "sdk";
+    }>>;
+    runCommand: z.ZodOptional<z.ZodEnum<typeof AgentRunCommand>>;
     modelConfig: z.ZodOptional<z.ZodObject<{
         model: z.ZodEnum<{
             [x: string]: string;
@@ -52,17 +83,22 @@ export declare const CreateAgentInputSchema: z.ZodObject<{
         temperature: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
     }, z.core.$strip>>;
+    systemPrompt: z.ZodOptional<z.ZodNullable<z.ZodString>>;
 }, z.core.$strip>;
 export type CreateAgentInput = z.infer<typeof CreateAgentInputSchema>;
 /**
  * Input schema for updating an Agent.
- * modelConfig can be null to explicitly clear it (vs undefined = keep existing).
+ * modelConfig and systemPrompt can be null to explicitly clear (vs undefined = keep existing).
  */
 export declare const UpdateAgentInputSchema: z.ZodObject<{
     name: z.ZodOptional<z.ZodString>;
     description: z.ZodOptional<z.ZodString>;
     projectId: z.ZodOptional<z.ZodString>;
-    runCommand: z.ZodOptional<z.ZodEnum<typeof AgentRunCommand>>;
+    agentType: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+        cli: "cli";
+        sdk: "sdk";
+    }>>>;
+    runCommand: z.ZodOptional<z.ZodOptional<z.ZodEnum<typeof AgentRunCommand>>>;
     modelConfig: z.ZodNullable<z.ZodOptional<z.ZodObject<{
         model: z.ZodEnum<{
             [x: string]: string;
@@ -70,5 +106,6 @@ export declare const UpdateAgentInputSchema: z.ZodObject<{
         temperature: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
     }, z.core.$strip>>>;
+    systemPrompt: z.ZodNullable<z.ZodOptional<z.ZodString>>;
 }, z.core.$strip>;
 export type UpdateAgentInput = z.infer<typeof UpdateAgentInputSchema>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/evalforge-types",
-  "version": "0.41.0",
+  "version": "0.43.0",
   "description": "Unified types for EvalForge agent evaluation system",
   "files": [
     "build"
@@ -47,5 +47,5 @@
       "artifactId": "evalforge-types"
     }
   },
-  "falconPackageHash": "05f127add95c6bdf6bee7bc611c14e751a58f44328d5e4f9621c92b2"
+  "falconPackageHash": "22e0c8e929bd0d6cd5bc802978eb373db3d7d290794c908608da12c0"
 }