@agentv/core 0.11.0 → 0.13.0

package/dist/index.d.cts

@@ -117,6 +117,7 @@ interface EvalCase {
  * Evaluator scorecard for a single eval case run.
  */
 interface EvaluationResult {
+    readonly timestamp: string;
     readonly eval_id: string;
     readonly dataset?: string;
     readonly conversation_id?: string;
@@ -124,14 +125,12 @@ interface EvaluationResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly candidate_answer: string;
-    readonly expected_aspect_count: number;
     readonly target: string;
-    readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly agent_provider_request?: JsonObject;
     readonly lm_provider_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
@@ -143,7 +142,7 @@ interface EvaluatorResult {
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -154,6 +153,7 @@ type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
+    readonly systemPrompt?: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -264,38 +264,41 @@ interface TargetDefinition {
 }
 
 /**
- * Read metadata from a test suite file (like target name).
- * This is a convenience function for CLI tools that need metadata without loading all eval cases.
- */
-declare function readTestSuiteMetadata(testFilePath: string): Promise<{
-    target?: string;
-}>;
-/**
- * Determine whether a path references guideline content (instructions or prompts).
+ * Build prompt inputs by consolidating user request context and guideline content.
  */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
 declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+
 type LoadOptions = {
     readonly verbose?: boolean;
     readonly evalId?: string;
 };
 /**
- * Load eval cases from a AgentV YAML specification file.
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
  */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
- * Build prompt inputs by consolidating user request context and guideline content.
+ * Load eval cases from a AgentV YAML specification file.
  */
-interface PromptInputs {
-    readonly question: string;
-    readonly guidelines: string;
-    readonly chatPrompt?: ChatPrompt;
-    readonly systemMessage?: string;
-}
-declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
 /**
@@ -500,7 +503,7 @@ interface EvaluationContext {
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
-    readonly systemPrompt?: string;
+    readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
 }
 interface EvaluationScore {
@@ -521,14 +524,14 @@ interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
-    readonly customPrompt?: string;
+    readonly evaluatorTemplate?: string;
 }
 declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    private readonly customPrompt?;
+    private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
     private evaluateWithPrompt;

package/dist/index.d.ts

@@ -117,6 +117,7 @@ interface EvalCase {
  * Evaluator scorecard for a single eval case run.
  */
 interface EvaluationResult {
+    readonly timestamp: string;
     readonly eval_id: string;
     readonly dataset?: string;
     readonly conversation_id?: string;
@@ -124,14 +125,12 @@ interface EvaluationResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly candidate_answer: string;
-    readonly expected_aspect_count: number;
     readonly target: string;
-    readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly agent_provider_request?: JsonObject;
     readonly lm_provider_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
@@ -143,7 +142,7 @@ interface EvaluatorResult {
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
-    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_provider_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -154,6 +153,7 @@ type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly question: string;
+    readonly systemPrompt?: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
@@ -264,38 +264,41 @@ interface TargetDefinition {
 }
 
 /**
- * Read metadata from a test suite file (like target name).
- * This is a convenience function for CLI tools that need metadata without loading all eval cases.
- */
-declare function readTestSuiteMetadata(testFilePath: string): Promise<{
-    target?: string;
-}>;
-/**
- * Determine whether a path references guideline content (instructions or prompts).
+ * Build prompt inputs by consolidating user request context and guideline content.
  */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
 /**
  * Extract fenced code blocks from AgentV user segments.
  */
 declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+
 type LoadOptions = {
     readonly verbose?: boolean;
     readonly evalId?: string;
 };
 /**
- * Load eval cases from a AgentV YAML specification file.
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
  */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
- * Build prompt inputs by consolidating user request context and guideline content.
+ * Load eval cases from a AgentV YAML specification file.
  */
-interface PromptInputs {
-    readonly question: string;
-    readonly guidelines: string;
-    readonly chatPrompt?: ChatPrompt;
-    readonly systemMessage?: string;
-}
-declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
 
 declare function fileExists(filePath: string): Promise<boolean>;
 /**
@@ -500,7 +503,7 @@ interface EvaluationContext {
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
-    readonly systemPrompt?: string;
+    readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
 }
 interface EvaluationScore {
@@ -521,14 +524,14 @@ interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
-    readonly customPrompt?: string;
+    readonly evaluatorTemplate?: string;
 }
 declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    private readonly customPrompt?;
+    private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
     private evaluateWithPrompt;