@agentv/sdk 4.41.5-next.1 → 4.41.6

package/dist/index.d.cts

@@ -6483,6 +6483,201 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
 }>;
 type PromptTemplateInput = CodeGraderInput;
 
+declare const EVAL_SUITE_SYMBOL: unique symbol;
+declare const TO_EVAL_YAML_OBJECT_SYMBOL: unique symbol;
+declare const KNOWN_SNAKE_CASE_KEYS: {
+    readonly afterAll: "after_all";
+    readonly afterEach: "after_each";
+    readonly argsMatch: "args_match";
+    readonly baseCommit: "base_commit";
+    readonly beforeAll: "before_all";
+    readonly beforeEach: "before_each";
+    readonly budgetUsd: "budget_usd";
+    readonly conversationId: "conversation_id";
+    readonly costLimitUsd: "cost_limit_usd";
+    readonly dependsOn: "depends_on";
+    readonly expectedOutput: "expected_output";
+    readonly explorationTolerance: "exploration_tolerance";
+    readonly failOnError: "fail_on_error";
+    readonly inputFiles: "input_files";
+    readonly keepWorkspaces: "keep_workspaces";
+    readonly maxCostUsd: "max_cost_usd";
+    readonly maxDurationMs: "max_duration_ms";
+    readonly maxInput: "max_input";
+    readonly maxLlmCalls: "max_llm_calls";
+    readonly maxOutput: "max_output";
+    readonly maxTokens: "max_tokens";
+    readonly maxToolCalls: "max_tool_calls";
+    readonly minScore: "min_score";
+    readonly onDependencyFailure: "on_dependency_failure";
+    readonly onTurnFailure: "on_turn_failure";
+    readonly outputPath: "output_path";
+    readonly scoreRanges: "score_ranges";
+    readonly skipDefaults: "skip_defaults";
+    readonly targetExplorationRatio: "target_exploration_ratio";
+    readonly timeoutMs: "timeout_ms";
+    readonly useTarget: "use_target";
+    readonly windowSize: "window_size";
+};
+type KnownSnakeCaseKeyMap = typeof KNOWN_SNAKE_CASE_KEYS;
+type LowerEvalKey<Key extends string> = Key extends keyof KnownSnakeCaseKeyMap ? KnownSnakeCaseKeyMap[Key] : Key;
+type LowerEvalYamlValue<Value> = Value extends readonly (infer Item)[] ? LowerEvalYamlValue<Item>[] : Value extends object ? {
+    [Key in keyof Value as Key extends string ? LowerEvalKey<Key> : never]: LowerEvalYamlValue<Value[Key]>;
+} : Value;
+type EvalMessageContent = string | Readonly<Record<string, unknown>> | readonly (string | Readonly<Record<string, unknown>>)[];
+interface EvalMessage {
+    readonly role: 'system' | 'user' | 'assistant' | 'tool';
+    readonly content: EvalMessageContent;
+    readonly [key: string]: unknown;
+}
+interface EvalAssertionConfig {
+    readonly type: string;
+    readonly [key: string]: unknown;
+}
+interface EvalPreprocessor {
+    readonly type: string;
+    readonly command: string | readonly string[];
+    readonly [key: string]: unknown;
+}
+interface EvalWorkspaceHook {
+    readonly command?: string | readonly string[];
+    readonly script?: string | readonly string[];
+    readonly timeoutMs?: number;
+    readonly cwd?: string;
+    readonly reset?: 'none' | 'fast' | 'strict';
+    readonly [key: string]: unknown;
+}
+interface EvalWorkspaceHooks {
+    readonly enabled?: boolean;
+    readonly beforeAll?: EvalWorkspaceHook;
+    readonly beforeEach?: EvalWorkspaceHook;
+    readonly afterEach?: EvalWorkspaceHook;
+    readonly afterAll?: EvalWorkspaceHook;
+}
+interface EvalWorkspaceRepo {
+    readonly path?: string;
+    readonly repo?: string;
+    readonly commit?: string;
+    readonly baseCommit?: string;
+    readonly ancestor?: number;
+    readonly sparse?: readonly string[];
+}
+interface EvalDockerWorkspace {
+    readonly image: string;
+    readonly timeout?: number;
+    readonly memory?: string;
+    readonly cpus?: number;
+}
+interface EvalWorkspace {
+    readonly template?: string;
+    readonly isolation?: 'shared' | 'per_test';
+    readonly repos?: readonly EvalWorkspaceRepo[];
+    readonly hooks?: EvalWorkspaceHooks;
+    readonly mode?: 'pooled' | 'temp' | 'static';
+    readonly path?: string;
+    readonly docker?: EvalDockerWorkspace;
+}
+interface EvalTargetRef {
+    readonly name: string;
+    readonly useTarget?: string;
+    readonly hooks?: EvalWorkspaceHooks;
+}
+interface EvalTrials {
+    readonly count: number;
+    readonly strategy?: 'pass_at_k' | 'mean' | 'confidence_interval';
+    readonly costLimitUsd?: number;
+}
+interface EvalExecution {
+    readonly target?: string;
+    readonly targets?: readonly (string | EvalTargetRef)[];
+    readonly workers?: number;
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly skipDefaults?: boolean;
+    readonly cache?: boolean;
+    readonly trials?: EvalTrials;
+    readonly budgetUsd?: number;
+    readonly failOnError?: boolean;
+    readonly threshold?: number;
+    readonly [key: string]: unknown;
+}
+interface EvalTurn {
+    readonly input: EvalMessageContent;
+    readonly expectedOutput?: EvalMessageContent;
+    readonly assertions?: readonly (string | EvalAssertionConfig)[];
+}
+interface EvalTest {
+    readonly id: string;
+    readonly vars?: Readonly<Record<string, unknown>>;
+    readonly criteria?: string;
+    readonly input?: string | readonly EvalMessage[];
+    readonly inputFiles?: readonly string[];
+    readonly expectedOutput?: string | Readonly<Record<string, unknown>> | readonly EvalMessage[];
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly execution?: EvalExecution;
+    readonly workspace?: EvalWorkspace;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+    readonly conversationId?: string;
+    readonly suite?: string;
+    readonly dependsOn?: readonly string[];
+    readonly onDependencyFailure?: 'skip' | 'fail' | 'run';
+    readonly mode?: 'conversation';
+    readonly turns?: readonly EvalTurn[];
+    readonly aggregation?: 'mean' | 'min' | 'max';
+    readonly onTurnFailure?: 'continue' | 'stop';
+    readonly windowSize?: number;
+}
+interface EvalRequires {
+    readonly agentv?: string;
+    readonly [key: string]: unknown;
+}
+interface EvalDefinition {
+    readonly $schema?: string;
+    readonly name?: string;
+    readonly description?: string;
+    readonly category?: string;
+    readonly version?: string;
+    readonly author?: string;
+    readonly tags?: readonly string[];
+    readonly license?: string;
+    readonly requires?: EvalRequires;
+    readonly input?: string | readonly EvalMessage[];
+    readonly inputFiles?: readonly string[];
+    readonly tests: readonly EvalTest[] | string;
+    readonly target?: string;
+    readonly execution?: EvalExecution;
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly preprocessors?: readonly EvalPreprocessor[];
+    readonly workspace?: EvalWorkspace | string;
+}
+interface DefinedEvalSuite {
+    readonly [EVAL_SUITE_SYMBOL]: true;
+    readonly [TO_EVAL_YAML_OBJECT_SYMBOL]: () => Record<string, unknown>;
+}
+/**
+ * Define a YAML-aligned eval suite in TypeScript.
+ *
+ * The returned object preserves the TypeScript authoring shape and carries a
+ * non-enumerable lowering hook so AgentV can materialize the canonical
+ * snake_case eval contract when the suite is loaded from a `.eval.ts` file.
+ */
+declare function defineEval<T extends EvalDefinition>(definition: T): T & DefinedEvalSuite;
+/**
+ * Alias for `defineEval()` when a suite reads more clearly as a plain object.
+ */
+declare function evalSuite<T extends EvalDefinition>(definition: T): T & DefinedEvalSuite;
+/**
+ * Lower a TypeScript-authored eval suite into the canonical snake_case object
+ * contract used by YAML files and the runtime loader.
+ *
+ * Only known AgentV wire keys are converted. Unknown keys are preserved as-is
+ * so opaque assertion, provider, and metadata payloads are not corrupted.
+ */
+declare function toEvalYamlObject<T extends EvalDefinition | DefinedEvalSuite>(definition: T): LowerEvalYamlValue<T>;
+/**
+ * Serialize an eval suite to canonical YAML.
+ */
+declare function serializeEvalYaml<T extends EvalDefinition | DefinedEvalSuite>(definition: T): string;
+
 /**
  * Client for invoking configured targets from code-grader scripts.
  *
@@ -6827,4 +7022,4 @@ declare function definePromptTemplate(handler: PromptTemplateHandler): void;
  */
 declare function defineAssertion(handler: AssertionHandler): void;
 
-export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type Content, type ContentFile, ContentFileSchema, type ContentImage, ContentImageSchema, ContentSchema, type ContentText, ContentTextSchema, type Message, MessageSchema, type PromptTemplateHandler, type PromptTemplateInput, PromptTemplateInputSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type Trace, type TraceArtifact, TraceArtifactSchema, type TraceBranch, TraceBranchSchema, type TraceError, TraceErrorSchema, type TraceEvent, TraceEventSchema, type TraceMessage, TraceMessageSchema, type TraceModel, TraceModelSchema, type TraceRawEvidence, TraceRawEvidenceSchema, type TraceRedactionState, TraceRedactionStateSchema, TraceSchema, type TraceSession, TraceSessionSchema, type TraceSource, type TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, type TraceSummary, TraceSummarySchema, type TraceTool, TraceToolSchema, createTargetClient, defineAssertion, defineCodeGrader, definePromptTemplate };
+export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type Content, type ContentFile, ContentFileSchema, type ContentImage, ContentImageSchema, ContentSchema, type ContentText, ContentTextSchema, type DefinedEvalSuite, type EvalAssertionConfig, type EvalDefinition, type EvalDockerWorkspace, type EvalExecution, type EvalMessage, type EvalMessageContent, type EvalPreprocessor, type EvalRequires, type EvalTargetRef, type EvalTest, type EvalTrials, type EvalTurn, type EvalWorkspace, type EvalWorkspaceHook, type EvalWorkspaceHooks, type EvalWorkspaceRepo, type LowerEvalYamlValue, type Message, MessageSchema, type PromptTemplateHandler, type PromptTemplateInput, PromptTemplateInputSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type Trace, type TraceArtifact, TraceArtifactSchema, type TraceBranch, TraceBranchSchema, type TraceError, TraceErrorSchema, type TraceEvent, TraceEventSchema, type TraceMessage, TraceMessageSchema, type TraceModel, TraceModelSchema, type TraceRawEvidence, TraceRawEvidenceSchema, type TraceRedactionState, TraceRedactionStateSchema, TraceSchema, type TraceSession, TraceSessionSchema, type TraceSource, type TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, type TraceSummary, TraceSummarySchema, type TraceTool, TraceToolSchema, createTargetClient, defineAssertion, defineCodeGrader, defineEval, definePromptTemplate, evalSuite, serializeEvalYaml, toEvalYamlObject };

package/dist/index.d.ts

@@ -6483,6 +6483,201 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
 }>;
 type PromptTemplateInput = CodeGraderInput;
 
+declare const EVAL_SUITE_SYMBOL: unique symbol;
+declare const TO_EVAL_YAML_OBJECT_SYMBOL: unique symbol;
+declare const KNOWN_SNAKE_CASE_KEYS: {
+    readonly afterAll: "after_all";
+    readonly afterEach: "after_each";
+    readonly argsMatch: "args_match";
+    readonly baseCommit: "base_commit";
+    readonly beforeAll: "before_all";
+    readonly beforeEach: "before_each";
+    readonly budgetUsd: "budget_usd";
+    readonly conversationId: "conversation_id";
+    readonly costLimitUsd: "cost_limit_usd";
+    readonly dependsOn: "depends_on";
+    readonly expectedOutput: "expected_output";
+    readonly explorationTolerance: "exploration_tolerance";
+    readonly failOnError: "fail_on_error";
+    readonly inputFiles: "input_files";
+    readonly keepWorkspaces: "keep_workspaces";
+    readonly maxCostUsd: "max_cost_usd";
+    readonly maxDurationMs: "max_duration_ms";
+    readonly maxInput: "max_input";
+    readonly maxLlmCalls: "max_llm_calls";
+    readonly maxOutput: "max_output";
+    readonly maxTokens: "max_tokens";
+    readonly maxToolCalls: "max_tool_calls";
+    readonly minScore: "min_score";
+    readonly onDependencyFailure: "on_dependency_failure";
+    readonly onTurnFailure: "on_turn_failure";
+    readonly outputPath: "output_path";
+    readonly scoreRanges: "score_ranges";
+    readonly skipDefaults: "skip_defaults";
+    readonly targetExplorationRatio: "target_exploration_ratio";
+    readonly timeoutMs: "timeout_ms";
+    readonly useTarget: "use_target";
+    readonly windowSize: "window_size";
+};
+type KnownSnakeCaseKeyMap = typeof KNOWN_SNAKE_CASE_KEYS;
+type LowerEvalKey<Key extends string> = Key extends keyof KnownSnakeCaseKeyMap ? KnownSnakeCaseKeyMap[Key] : Key;
+type LowerEvalYamlValue<Value> = Value extends readonly (infer Item)[] ? LowerEvalYamlValue<Item>[] : Value extends object ? {
+    [Key in keyof Value as Key extends string ? LowerEvalKey<Key> : never]: LowerEvalYamlValue<Value[Key]>;
+} : Value;
+type EvalMessageContent = string | Readonly<Record<string, unknown>> | readonly (string | Readonly<Record<string, unknown>>)[];
+interface EvalMessage {
+    readonly role: 'system' | 'user' | 'assistant' | 'tool';
+    readonly content: EvalMessageContent;
+    readonly [key: string]: unknown;
+}
+interface EvalAssertionConfig {
+    readonly type: string;
+    readonly [key: string]: unknown;
+}
+interface EvalPreprocessor {
+    readonly type: string;
+    readonly command: string | readonly string[];
+    readonly [key: string]: unknown;
+}
+interface EvalWorkspaceHook {
+    readonly command?: string | readonly string[];
+    readonly script?: string | readonly string[];
+    readonly timeoutMs?: number;
+    readonly cwd?: string;
+    readonly reset?: 'none' | 'fast' | 'strict';
+    readonly [key: string]: unknown;
+}
+interface EvalWorkspaceHooks {
+    readonly enabled?: boolean;
+    readonly beforeAll?: EvalWorkspaceHook;
+    readonly beforeEach?: EvalWorkspaceHook;
+    readonly afterEach?: EvalWorkspaceHook;
+    readonly afterAll?: EvalWorkspaceHook;
+}
+interface EvalWorkspaceRepo {
+    readonly path?: string;
+    readonly repo?: string;
+    readonly commit?: string;
+    readonly baseCommit?: string;
+    readonly ancestor?: number;
+    readonly sparse?: readonly string[];
+}
+interface EvalDockerWorkspace {
+    readonly image: string;
+    readonly timeout?: number;
+    readonly memory?: string;
+    readonly cpus?: number;
+}
+interface EvalWorkspace {
+    readonly template?: string;
+    readonly isolation?: 'shared' | 'per_test';
+    readonly repos?: readonly EvalWorkspaceRepo[];
+    readonly hooks?: EvalWorkspaceHooks;
+    readonly mode?: 'pooled' | 'temp' | 'static';
+    readonly path?: string;
+    readonly docker?: EvalDockerWorkspace;
+}
+interface EvalTargetRef {
+    readonly name: string;
+    readonly useTarget?: string;
+    readonly hooks?: EvalWorkspaceHooks;
+}
+interface EvalTrials {
+    readonly count: number;
+    readonly strategy?: 'pass_at_k' | 'mean' | 'confidence_interval';
+    readonly costLimitUsd?: number;
+}
+interface EvalExecution {
+    readonly target?: string;
+    readonly targets?: readonly (string | EvalTargetRef)[];
+    readonly workers?: number;
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly skipDefaults?: boolean;
+    readonly cache?: boolean;
+    readonly trials?: EvalTrials;
+    readonly budgetUsd?: number;
+    readonly failOnError?: boolean;
+    readonly threshold?: number;
+    readonly [key: string]: unknown;
+}
+interface EvalTurn {
+    readonly input: EvalMessageContent;
+    readonly expectedOutput?: EvalMessageContent;
+    readonly assertions?: readonly (string | EvalAssertionConfig)[];
+}
+interface EvalTest {
+    readonly id: string;
+    readonly vars?: Readonly<Record<string, unknown>>;
+    readonly criteria?: string;
+    readonly input?: string | readonly EvalMessage[];
+    readonly inputFiles?: readonly string[];
+    readonly expectedOutput?: string | Readonly<Record<string, unknown>> | readonly EvalMessage[];
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly execution?: EvalExecution;
+    readonly workspace?: EvalWorkspace;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+    readonly conversationId?: string;
+    readonly suite?: string;
+    readonly dependsOn?: readonly string[];
+    readonly onDependencyFailure?: 'skip' | 'fail' | 'run';
+    readonly mode?: 'conversation';
+    readonly turns?: readonly EvalTurn[];
+    readonly aggregation?: 'mean' | 'min' | 'max';
+    readonly onTurnFailure?: 'continue' | 'stop';
+    readonly windowSize?: number;
+}
+interface EvalRequires {
+    readonly agentv?: string;
+    readonly [key: string]: unknown;
+}
+interface EvalDefinition {
+    readonly $schema?: string;
+    readonly name?: string;
+    readonly description?: string;
+    readonly category?: string;
+    readonly version?: string;
+    readonly author?: string;
+    readonly tags?: readonly string[];
+    readonly license?: string;
+    readonly requires?: EvalRequires;
+    readonly input?: string | readonly EvalMessage[];
+    readonly inputFiles?: readonly string[];
+    readonly tests: readonly EvalTest[] | string;
+    readonly target?: string;
+    readonly execution?: EvalExecution;
+    readonly assertions?: readonly EvalAssertionConfig[];
+    readonly preprocessors?: readonly EvalPreprocessor[];
+    readonly workspace?: EvalWorkspace | string;
+}
+interface DefinedEvalSuite {
+    readonly [EVAL_SUITE_SYMBOL]: true;
+    readonly [TO_EVAL_YAML_OBJECT_SYMBOL]: () => Record<string, unknown>;
+}
+/**
+ * Define a YAML-aligned eval suite in TypeScript.
+ *
+ * The returned object preserves the TypeScript authoring shape and carries a
+ * non-enumerable lowering hook so AgentV can materialize the canonical
+ * snake_case eval contract when the suite is loaded from a `.eval.ts` file.
+ */
+declare function defineEval<T extends EvalDefinition>(definition: T): T & DefinedEvalSuite;
+/**
+ * Alias for `defineEval()` when a suite reads more clearly as a plain object.
+ */
+declare function evalSuite<T extends EvalDefinition>(definition: T): T & DefinedEvalSuite;
+/**
+ * Lower a TypeScript-authored eval suite into the canonical snake_case object
+ * contract used by YAML files and the runtime loader.
+ *
+ * Only known AgentV wire keys are converted. Unknown keys are preserved as-is
+ * so opaque assertion, provider, and metadata payloads are not corrupted.
+ */
+declare function toEvalYamlObject<T extends EvalDefinition | DefinedEvalSuite>(definition: T): LowerEvalYamlValue<T>;
+/**
+ * Serialize an eval suite to canonical YAML.
+ */
+declare function serializeEvalYaml<T extends EvalDefinition | DefinedEvalSuite>(definition: T): string;
+
 /**
  * Client for invoking configured targets from code-grader scripts.
  *
@@ -6827,4 +7022,4 @@ declare function definePromptTemplate(handler: PromptTemplateHandler): void;
  */
 declare function defineAssertion(handler: AssertionHandler): void;
 
-export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type Content, type ContentFile, ContentFileSchema, type ContentImage, ContentImageSchema, ContentSchema, type ContentText, ContentTextSchema, type Message, MessageSchema, type PromptTemplateHandler, type PromptTemplateInput, PromptTemplateInputSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type Trace, type TraceArtifact, TraceArtifactSchema, type TraceBranch, TraceBranchSchema, type TraceError, TraceErrorSchema, type TraceEvent, TraceEventSchema, type TraceMessage, TraceMessageSchema, type TraceModel, TraceModelSchema, type TraceRawEvidence, TraceRawEvidenceSchema, type TraceRedactionState, TraceRedactionStateSchema, TraceSchema, type TraceSession, TraceSessionSchema, type TraceSource, type TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, type TraceSummary, TraceSummarySchema, type TraceTool, TraceToolSchema, createTargetClient, defineAssertion, defineCodeGrader, definePromptTemplate };
+export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type Content, type ContentFile, ContentFileSchema, type ContentImage, ContentImageSchema, ContentSchema, type ContentText, ContentTextSchema, type DefinedEvalSuite, type EvalAssertionConfig, type EvalDefinition, type EvalDockerWorkspace, type EvalExecution, type EvalMessage, type EvalMessageContent, type EvalPreprocessor, type EvalRequires, type EvalTargetRef, type EvalTest, type EvalTrials, type EvalTurn, type EvalWorkspace, type EvalWorkspaceHook, type EvalWorkspaceHooks, type EvalWorkspaceRepo, type LowerEvalYamlValue, type Message, MessageSchema, type PromptTemplateHandler, type PromptTemplateInput, PromptTemplateInputSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type Trace, type TraceArtifact, TraceArtifactSchema, type TraceBranch, TraceBranchSchema, type TraceError, TraceErrorSchema, type TraceEvent, TraceEventSchema, type TraceMessage, TraceMessageSchema, type TraceModel, TraceModelSchema, type TraceRawEvidence, TraceRawEvidenceSchema, type TraceRedactionState, TraceRedactionStateSchema, TraceSchema, type TraceSession, TraceSessionSchema, type TraceSource, type TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, type TraceSummary, TraceSummarySchema, type TraceTool, TraceToolSchema, createTargetClient, defineAssertion, defineCodeGrader, defineEval, definePromptTemplate, evalSuite, serializeEvalYaml, toEvalYamlObject };

package/dist/index.js

@@ -226,6 +226,92 @@ var CodeGraderResultSchema = z.object({
 });
 var PromptTemplateInputSchema = CodeGraderInputSchema;
 
+// src/eval.ts
+import { stringify as stringifyYaml } from "yaml";
+var EVAL_SUITE_SYMBOL = Symbol.for("@agentv/sdk/eval-suite");
+var TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for("@agentv/sdk/to-eval-yaml-object");
+var KNOWN_SNAKE_CASE_KEYS = {
+  afterAll: "after_all",
+  afterEach: "after_each",
+  argsMatch: "args_match",
+  baseCommit: "base_commit",
+  beforeAll: "before_all",
+  beforeEach: "before_each",
+  budgetUsd: "budget_usd",
+  conversationId: "conversation_id",
+  costLimitUsd: "cost_limit_usd",
+  dependsOn: "depends_on",
+  expectedOutput: "expected_output",
+  explorationTolerance: "exploration_tolerance",
+  failOnError: "fail_on_error",
+  inputFiles: "input_files",
+  keepWorkspaces: "keep_workspaces",
+  maxCostUsd: "max_cost_usd",
+  maxDurationMs: "max_duration_ms",
+  maxInput: "max_input",
+  maxLlmCalls: "max_llm_calls",
+  maxOutput: "max_output",
+  maxTokens: "max_tokens",
+  maxToolCalls: "max_tool_calls",
+  minScore: "min_score",
+  onDependencyFailure: "on_dependency_failure",
+  onTurnFailure: "on_turn_failure",
+  outputPath: "output_path",
+  scoreRanges: "score_ranges",
+  skipDefaults: "skip_defaults",
+  targetExplorationRatio: "target_exploration_ratio",
+  timeoutMs: "timeout_ms",
+  useTarget: "use_target",
+  windowSize: "window_size"
+};
+function lowerEvalYamlValue(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => lowerEvalYamlValue(item));
+  }
+  if (value && typeof value === "object") {
+    const result = {};
+    for (const [key, nestedValue] of Object.entries(value)) {
+      const loweredKey = KNOWN_SNAKE_CASE_KEYS[key] ?? key;
+      result[loweredKey] = lowerEvalYamlValue(nestedValue);
+    }
+    return result;
+  }
+  return value;
+}
+function attachEvalSuiteBrand(definition) {
+  const branded = definition;
+  if (branded[EVAL_SUITE_SYMBOL] === true) {
+    return branded;
+  }
+  Object.defineProperties(branded, {
+    [EVAL_SUITE_SYMBOL]: {
+      value: true,
+      enumerable: false,
+      configurable: false,
+      writable: false
+    },
+    [TO_EVAL_YAML_OBJECT_SYMBOL]: {
+      value: () => toEvalYamlObject(definition),
+      enumerable: false,
+      configurable: false,
+      writable: false
+    }
+  });
+  return branded;
+}
+function defineEval(definition) {
+  return attachEvalSuiteBrand(definition);
+}
+function evalSuite(definition) {
+  return defineEval(definition);
+}
+function toEvalYamlObject(definition) {
+  return lowerEvalYamlValue(definition);
+}
+function serializeEvalYaml(definition) {
+  return stringifyYaml(toEvalYamlObject(definition), { lineWidth: 0 }).trimEnd();
+}
+
 // src/target-client.ts
 var TargetNotAvailableError = class extends Error {
   constructor(message) {
@@ -556,7 +642,11 @@ export {
   createTargetClient,
   defineAssertion,
   defineCodeGrader,
+  defineEval,
   definePromptTemplate,
+  evalSuite,
+  serializeEvalYaml,
+  toEvalYamlObject,
   z2 as z
 };
 //# sourceMappingURL=index.js.map