@agentv/core 0.5.3 → 0.7.0

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { AxChatRequest } from '@ax-llm/ax';
+import { AxChatRequest, AxAI } from '@ax-llm/ax';
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -99,16 +99,18 @@ type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
  */
 interface EvalCase {
     readonly id: string;
+    readonly dataset?: string;
     readonly conversation_id?: string;
-    readonly task: string;
-    readonly user_segments: readonly JsonObject[];
+    readonly question: string;
+    readonly input_segments: readonly JsonObject[];
+    readonly output_segments: readonly JsonObject[];
     readonly system_message?: string;
-    readonly expected_assistant_raw: string;
+    readonly reference_answer: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
     readonly file_paths: readonly string[];
     readonly code_snippets: readonly string[];
-    readonly outcome: string;
+    readonly expected_outcome: string;
     readonly evaluator?: EvaluatorKind;
     readonly evaluators?: readonly EvaluatorConfig[];
 }
@@ -117,11 +119,12 @@ interface EvalCase {
  */
 interface EvaluationResult {
     readonly eval_id: string;
+    readonly dataset?: string;
     readonly conversation_id?: string;
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
-    readonly model_answer: string;
+    readonly candidate_answer: string;
     readonly expected_aspect_count: number;
     readonly target: string;
     readonly timestamp: string;
@@ -165,12 +168,17 @@ declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, opt
  * Build prompt inputs by consolidating user request context and guideline content.
  */
 declare function buildPromptInputs(testCase: EvalCase): Promise<{
-    request: string;
+    question: string;
     guidelines: string;
     systemMessage?: string;
 }>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
+/**
+ * Read a text file and normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function readTextFile(filePath: string): Promise<string>;
 /**
  * Find git repository root by walking up the directory tree.
  */
@@ -197,7 +205,7 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
 type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
-    readonly prompt: string;
+    readonly question: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -229,6 +237,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional access to the underlying AxAI instance.
+     * This enables using advanced Ax features like structured output signatures.
+     */
+    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -372,6 +385,16 @@ interface EnsureSubagentsResult {
  */
 declare function ensureVSCodeSubagents(options: EnsureSubagentsOptions): Promise<EnsureSubagentsResult>;
 
+type CodexLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type CodexLogListener = (entry: CodexLogEntry) => void;
+declare function consumeCodexLogEntries(): CodexLogEntry[];
+declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => void;
+
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
@@ -382,7 +405,7 @@ interface EvaluationContext {
     readonly provider: Provider;
     readonly attempt: number;
     readonly promptInputs: {
-        readonly request: string;
+        readonly question: string;
         readonly guidelines: string;
         readonly systemMessage?: string;
     };
@@ -420,6 +443,7 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly customPrompt?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private evaluateWithPrompt;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -492,4 +516,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { AxChatRequest } from '@ax-llm/ax';
+import { AxChatRequest, AxAI } from '@ax-llm/ax';
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -99,16 +99,18 @@ type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
  */
 interface EvalCase {
     readonly id: string;
+    readonly dataset?: string;
     readonly conversation_id?: string;
-    readonly task: string;
-    readonly user_segments: readonly JsonObject[];
+    readonly question: string;
+    readonly input_segments: readonly JsonObject[];
+    readonly output_segments: readonly JsonObject[];
     readonly system_message?: string;
-    readonly expected_assistant_raw: string;
+    readonly reference_answer: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
     readonly file_paths: readonly string[];
     readonly code_snippets: readonly string[];
-    readonly outcome: string;
+    readonly expected_outcome: string;
     readonly evaluator?: EvaluatorKind;
     readonly evaluators?: readonly EvaluatorConfig[];
 }
@@ -117,11 +119,12 @@ interface EvalCase {
  */
 interface EvaluationResult {
     readonly eval_id: string;
+    readonly dataset?: string;
     readonly conversation_id?: string;
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
-    readonly model_answer: string;
+    readonly candidate_answer: string;
     readonly expected_aspect_count: number;
     readonly target: string;
     readonly timestamp: string;
@@ -165,12 +168,17 @@ declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, opt
  * Build prompt inputs by consolidating user request context and guideline content.
  */
 declare function buildPromptInputs(testCase: EvalCase): Promise<{
-    request: string;
+    question: string;
     guidelines: string;
     systemMessage?: string;
 }>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
+/**
+ * Read a text file and normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function readTextFile(filePath: string): Promise<string>;
 /**
  * Find git repository root by walking up the directory tree.
  */
@@ -197,7 +205,7 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
 type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
-    readonly prompt: string;
+    readonly question: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -229,6 +237,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional access to the underlying AxAI instance.
+     * This enables using advanced Ax features like structured output signatures.
+     */
+    getAxAI?(): AxAI;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -372,6 +385,16 @@ interface EnsureSubagentsResult {
  */
 declare function ensureVSCodeSubagents(options: EnsureSubagentsOptions): Promise<EnsureSubagentsResult>;
 
+type CodexLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type CodexLogListener = (entry: CodexLogEntry) => void;
+declare function consumeCodexLogEntries(): CodexLogEntry[];
+declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => void;
+
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
@@ -382,7 +405,7 @@ interface EvaluationContext {
     readonly provider: Provider;
     readonly attempt: number;
     readonly promptInputs: {
-        readonly request: string;
+        readonly question: string;
         readonly guidelines: string;
         readonly systemMessage?: string;
     };
@@ -420,6 +443,7 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly customPrompt?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private evaluateWithPrompt;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -492,4 +516,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };