@agentv/core 0.11.0 → 0.14.2

package/dist/index.d.cts

@@ -1,5 +1,3 @@
-import { AxChatRequest, AxAI } from '@ax-llm/ax';
-
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -117,6 +115,7 @@ interface EvalCase {
  * Evaluator scorecard for a single eval case run.
  */
 interface EvaluationResult {
+    readonly timestamp: string;
     readonly eval_id: string;
     readonly dataset?: string;
     readonly conversation_id?: string;
@@ -124,14 +123,12 @@ interface EvaluationResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly candidate_answer: string;
-    readonly expected_aspect_count: number;
     readonly target: string;
-    readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly agent_provider_request?: JsonObject;
     readonly lm_provider_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
@@ -143,17 +140,24 @@ interface EvaluatorResult {
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-type ChatPrompt = AxChatRequest["chatPrompt"];
+type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+interface ChatMessage {
+    readonly role: ChatMessageRole;
+    readonly content: string;
+    readonly name?: string;
+}
+type ChatPrompt = readonly ChatMessage[];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
+    readonly systemPrompt?: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -185,11 +189,6 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
-    /**
-     * Optional access to the underlying AxAI instance.
-     * This enables using advanced Ax features like structured output signatures.
-     */
-    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -264,38 +263,41 @@ interface TargetDefinition {
 }
 
 /**
- * Read metadata from a test suite file (like target name).
- * This is a convenience function for CLI tools that need metadata without loading all eval cases.
- */
-declare function readTestSuiteMetadata(testFilePath: string): Promise<{
-    target?: string;
-}>;
-/**
- * Determine whether a path references guideline content (instructions or prompts).
+ * Build prompt inputs by consolidating user request context and guideline content.
  */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
 declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+
 type LoadOptions = {
     readonly verbose?: boolean;
     readonly evalId?: string;
 };
 /**
- * Load eval cases from a AgentV YAML specification file.
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
  */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
- * Build prompt inputs by consolidating user request context and guideline content.
+ * Load eval cases from a AgentV YAML specification file.
  */
-interface PromptInputs {
-    readonly question: string;
-    readonly guidelines: string;
-    readonly chatPrompt?: ChatPrompt;
-    readonly systemMessage?: string;
-}
-declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
 /**
@@ -338,6 +340,9 @@ interface RetryConfig {
     readonly backoffFactor?: number;
     readonly retryableStatusCodes?: readonly number[];
 }
+/**
+ * Azure OpenAI settings used by the Vercel AI SDK.
+ */
 interface AzureResolvedConfig {
     readonly resourceName: string;
     readonly deploymentName: string;
@@ -347,6 +352,9 @@ interface AzureResolvedConfig {
     readonly maxOutputTokens?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Anthropic Claude settings used by the Vercel AI SDK.
+ */
 interface AnthropicResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -355,6 +363,9 @@ interface AnthropicResolvedConfig {
     readonly thinkingBudget?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Google Gemini settings used by the Vercel AI SDK.
+ */
 interface GeminiResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -399,6 +410,7 @@ interface CliResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
+    readonly verbose?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: "azure";
@@ -500,7 +512,7 @@ interface EvaluationContext {
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
-    readonly systemPrompt?: string;
+    readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
 }
 interface EvaluationScore {
@@ -521,14 +533,14 @@ interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
-    readonly customPrompt?: string;
+    readonly evaluatorTemplate?: string;
 }
 declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    private readonly customPrompt?;
+    private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
     private evaluateWithPrompt;

package/dist/index.d.ts

@@ -1,5 +1,3 @@
-import { AxChatRequest, AxAI } from '@ax-llm/ax';
-
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -117,6 +115,7 @@ interface EvalCase {
  * Evaluator scorecard for a single eval case run.
  */
 interface EvaluationResult {
+    readonly timestamp: string;
     readonly eval_id: string;
     readonly dataset?: string;
     readonly conversation_id?: string;
@@ -124,14 +123,12 @@ interface EvaluationResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly candidate_answer: string;
-    readonly expected_aspect_count: number;
     readonly target: string;
-    readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly agent_provider_request?: JsonObject;
     readonly lm_provider_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
@@ -143,17 +140,24 @@ interface EvaluatorResult {
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-type ChatPrompt = AxChatRequest["chatPrompt"];
+type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
+interface ChatMessage {
+    readonly role: ChatMessageRole;
+    readonly content: string;
+    readonly name?: string;
+}
+type ChatPrompt = readonly ChatMessage[];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
+    readonly systemPrompt?: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -185,11 +189,6 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
-    /**
-     * Optional access to the underlying AxAI instance.
-     * This enables using advanced Ax features like structured output signatures.
-     */
-    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -264,38 +263,41 @@ interface TargetDefinition {
 }
 
 /**
- * Read metadata from a test suite file (like target name).
- * This is a convenience function for CLI tools that need metadata without loading all eval cases.
- */
-declare function readTestSuiteMetadata(testFilePath: string): Promise<{
-    target?: string;
-}>;
-/**
- * Determine whether a path references guideline content (instructions or prompts).
+ * Build prompt inputs by consolidating user request context and guideline content.
  */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
 declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+
 type LoadOptions = {
     readonly verbose?: boolean;
     readonly evalId?: string;
 };
 /**
- * Load eval cases from a AgentV YAML specification file.
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
  */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
- * Build prompt inputs by consolidating user request context and guideline content.
+ * Load eval cases from a AgentV YAML specification file.
  */
-interface PromptInputs {
-    readonly question: string;
-    readonly guidelines: string;
-    readonly chatPrompt?: ChatPrompt;
-    readonly systemMessage?: string;
-}
-declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
 /**
@@ -338,6 +340,9 @@ interface RetryConfig {
     readonly backoffFactor?: number;
     readonly retryableStatusCodes?: readonly number[];
 }
+/**
+ * Azure OpenAI settings used by the Vercel AI SDK.
+ */
 interface AzureResolvedConfig {
     readonly resourceName: string;
     readonly deploymentName: string;
@@ -347,6 +352,9 @@ interface AzureResolvedConfig {
     readonly maxOutputTokens?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Anthropic Claude settings used by the Vercel AI SDK.
+ */
 interface AnthropicResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -355,6 +363,9 @@ interface AnthropicResolvedConfig {
     readonly thinkingBudget?: number;
     readonly retry?: RetryConfig;
 }
+/**
+ * Google Gemini settings used by the Vercel AI SDK.
+ */
 interface GeminiResolvedConfig {
     readonly apiKey: string;
     readonly model: string;
@@ -399,6 +410,7 @@ interface CliResolvedConfig {
     readonly cwd?: string;
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
+    readonly verbose?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: "azure";
@@ -500,7 +512,7 @@ interface EvaluationContext {
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
-    readonly systemPrompt?: string;
+    readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
 }
 interface EvaluationScore {
@@ -521,14 +533,14 @@ interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
-    readonly customPrompt?: string;
+    readonly evaluatorTemplate?: string;
 }
 declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    private readonly customPrompt?;
+    private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
     private evaluateWithPrompt;