@agentv/core 0.2.3

package/dist/index.d.ts

@@ -0,0 +1,406 @@
+import { AxChatRequest } from '@ax-llm/ax';
+
+/**
+ * JSON primitive values appearing in BbEval payloads.
+ */
+type JsonPrimitive = string | number | boolean | null;
+/**
+ * Immutable JSON object representation for test fixtures.
+ */
+interface JsonObject {
+    readonly [key: string]: JsonValue;
+}
+/**
+ * Recursive JSON value supporting nested structures.
+ */
+type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[];
+declare const TEST_MESSAGE_ROLE_VALUES: readonly ["system", "user", "assistant", "tool"];
+/**
+ * Immutable list of supported message roles.
+ */
+declare const TEST_MESSAGE_ROLES: readonly ["system", "user", "assistant", "tool"];
+/**
+ * Role literals used by test messages.
+ */
+type TestMessageRole = (typeof TEST_MESSAGE_ROLE_VALUES)[number];
+/**
+ * Text or structured payload attached to a message.
+ */
+type TestMessageContent = string | readonly JsonObject[];
+/**
+ * System-authored instruction message.
+ */
+type SystemTestMessage = {
+    readonly role: "system";
+    readonly content: TestMessageContent;
+};
+/**
+ * User-authored prompt message.
+ */
+type UserTestMessage = {
+    readonly role: "user";
+    readonly content: TestMessageContent;
+};
+/**
+ * Assistant response message.
+ */
+type AssistantTestMessage = {
+    readonly role: "assistant";
+    readonly content: TestMessageContent;
+};
+/**
+ * Tool invocation message.
+ */
+type ToolTestMessage = {
+    readonly role: "tool";
+    readonly content: TestMessageContent;
+};
+/**
+ * Conversation message union with role discrimination.
+ */
+type TestMessage = SystemTestMessage | UserTestMessage | AssistantTestMessage | ToolTestMessage;
+/**
+ * Guard validating supported message roles.
+ */
+declare function isTestMessageRole(value: unknown): value is TestMessageRole;
+/**
+ * Guard matching BbEval JSON objects.
+ */
+declare function isJsonObject(value: unknown): value is JsonObject;
+/**
+ * Guard matching BbEval JSON values.
+ */
+declare function isJsonValue(value: unknown): value is JsonValue;
+/**
+ * Guard validating raw test messages.
+ */
+declare function isTestMessage(value: unknown): value is TestMessage;
+declare const GRADER_KIND_VALUES: readonly ["heuristic", "llm_judge"];
+/**
+ * Supported grader implementations.
+ */
+declare const GRADER_KINDS: readonly ["heuristic", "llm_judge"];
+/**
+ * Grader identifiers available to the pipeline.
+ */
+type GraderKind = (typeof GRADER_KIND_VALUES)[number];
+/**
+ * Guard validating grader identifiers.
+ */
+declare function isGraderKind(value: unknown): value is GraderKind;
+/**
+ * Test case definition sourced from BbEval specs.
+ */
+interface TestCase {
+    readonly id: string;
+    readonly conversation_id?: string;
+    readonly task: string;
+    readonly user_segments: readonly JsonObject[];
+    readonly expected_assistant_raw: string;
+    readonly guideline_paths: readonly string[];
+    readonly code_snippets: readonly string[];
+    readonly outcome: string;
+    readonly grader: GraderKind;
+}
+/**
+ * Evaluator scorecard for a single test case run.
+ */
+interface EvaluationResult {
+    readonly test_id: string;
+    readonly conversation_id?: string;
+    readonly score: number;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly model_answer: string;
+    readonly expected_aspect_count: number;
+    readonly target: string;
+    readonly timestamp: string;
+    readonly reasoning?: string;
+    readonly raw_aspects?: readonly string[];
+    readonly raw_request?: JsonObject;
+    readonly grader_raw_request?: JsonObject;
+}
+/**
+ * Convenience accessor matching the Python hit_count property.
+ */
+declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
+
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string): boolean;
+/**
+ * Extract fenced code blocks from BbEval user segments.
+ */
+declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+type LoadOptions = {
+    readonly verbose?: boolean;
+};
+/**
+ * Load test cases from a BbEval YAML specification file.
+ */
+declare function loadTestCases(testFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly TestCase[]>;
+/**
+ * Build prompt inputs by consolidating user request context and guideline content.
+ */
+declare function buildPromptInputs(testCase: TestCase): Promise<{
+    request: string;
+    guidelines: string;
+}>;
+
+type ChatPrompt = AxChatRequest["chatPrompt"];
+type ProviderKind = "azure" | "anthropic" | "gemini" | "mock" | "vscode" | "vscode-insiders";
+interface ProviderRequest {
+    readonly prompt: string;
+    readonly guidelines?: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly attachments?: readonly string[];
+    readonly testCaseId?: string;
+    readonly attempt?: number;
+    readonly maxOutputTokens?: number;
+    readonly temperature?: number;
+    readonly metadata?: JsonObject;
+    readonly signal?: AbortSignal;
+}
+interface ProviderResponse {
+    readonly text: string;
+    readonly reasoning?: string;
+    readonly raw?: unknown;
+    readonly usage?: JsonObject;
+}
+interface Provider {
+    readonly id: string;
+    readonly kind: ProviderKind;
+    readonly targetName: string;
+    invoke(request: ProviderRequest): Promise<ProviderResponse>;
+}
+type EnvLookup = Readonly<Record<string, string | undefined>>;
+interface TargetDefinition {
+    readonly name: string;
+    readonly provider: ProviderKind | string;
+    readonly settings?: Record<string, unknown> | undefined;
+    readonly judge_target?: string | undefined;
+    readonly workers?: number | undefined;
+}
+
+interface AzureResolvedConfig {
+    readonly resourceName: string;
+    readonly deploymentName: string;
+    readonly apiKey: string;
+    readonly version?: string;
+    readonly temperature?: number;
+    readonly maxOutputTokens?: number;
+}
+interface AnthropicResolvedConfig {
+    readonly apiKey: string;
+    readonly model: string;
+    readonly temperature?: number;
+    readonly maxOutputTokens?: number;
+    readonly thinkingBudget?: number;
+}
+interface GeminiResolvedConfig {
+    readonly apiKey: string;
+    readonly model: string;
+    readonly temperature?: number;
+    readonly maxOutputTokens?: number;
+}
+interface MockResolvedConfig {
+    readonly response?: string;
+    readonly delayMs?: number;
+    readonly delayMinMs?: number;
+    readonly delayMaxMs?: number;
+}
+interface VSCodeResolvedConfig {
+    readonly command: string;
+    readonly waitForResponse: boolean;
+    readonly dryRun: boolean;
+    readonly subagentRoot?: string;
+    readonly workspaceTemplate?: string;
+}
+type ResolvedTarget = {
+    readonly kind: "azure";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly config: AzureResolvedConfig;
+} | {
+    readonly kind: "anthropic";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly config: AnthropicResolvedConfig;
+} | {
+    readonly kind: "gemini";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly config: GeminiResolvedConfig;
+} | {
+    readonly kind: "mock";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly config: MockResolvedConfig;
+} | {
+    readonly kind: "vscode" | "vscode-insiders";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly config: VSCodeResolvedConfig;
+};
+declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;
+
+declare function readTargetDefinitions(filePath: string): Promise<readonly TargetDefinition[]>;
+declare function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[];
+
+interface EnsureSubagentsOptions {
+    readonly kind: "vscode" | "vscode-insiders";
+    readonly count: number;
+    readonly verbose?: boolean;
+}
+interface EnsureSubagentsResult {
+    readonly provisioned: boolean;
+    readonly message?: string;
+}
+/**
+ * Ensures the required number of VSCode subagents are provisioned using the subagent package.
+ * This guarantees version compatibility by using the same subagent package version.
+ *
+ * @param options - Configuration for subagent provisioning
+ * @returns Information about the provisioning result
+ */
+declare function ensureVSCodeSubagents(options: EnsureSubagentsOptions): Promise<EnsureSubagentsResult>;
+
+declare function createProvider(target: ResolvedTarget): Provider;
+declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
+
+interface HeuristicScore {
+    readonly score: number;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly hitCount: number;
+    readonly totalAspects: number;
+    readonly rawAspects: readonly string[];
+}
+/**
+ * Extract individual evaluation aspects from the expected assistant response.
+ */
+declare function extractAspects(expectedResponse: string): readonly string[];
+/**
+ * Determine which aspects were covered by the candidate response.
+ */
+declare function calculateHits(candidateResponse: string, expectedAspects: readonly string[]): readonly string[];
+/**
+ * Determine which aspects were not satisfied by the candidate response.
+ */
+declare function calculateMisses(candidateResponse: string, expectedAspects: readonly string[], resolvedHits?: readonly string[]): readonly string[];
+/**
+ * Evaluate the candidate response against the expected aspects.
+ */
+declare function scoreCandidateResponse(candidateResponse: string, expectedAspects: readonly string[]): HeuristicScore;
+/**
+ * Detect common error-prefixed outputs from providers.
+ */
+declare function isErrorLike(text: string | undefined | null): boolean;
+
+interface GradeContext {
+    readonly testCase: TestCase;
+    readonly candidate: string;
+    readonly target: ResolvedTarget;
+    readonly provider: Provider;
+    readonly attempt: number;
+    readonly promptInputs: {
+        readonly request: string;
+        readonly guidelines: string;
+    };
+    readonly now: Date;
+    readonly judgeProvider?: Provider;
+}
+interface GradeResult {
+    readonly score: number;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly expectedAspectCount: number;
+    readonly reasoning?: string;
+    readonly rawAspects?: readonly string[];
+    readonly graderRawRequest?: JsonObject;
+}
+interface Grader {
+    readonly kind: string;
+    grade(context: GradeContext): Promise<GradeResult> | GradeResult;
+}
+declare class HeuristicGrader implements Grader {
+    readonly kind = "heuristic";
+    grade(context: GradeContext): GradeResult;
+}
+type JudgeProviderResolver = (context: GradeContext) => Promise<Provider | undefined>;
+interface QualityGraderOptions {
+    readonly resolveJudgeProvider: JudgeProviderResolver;
+    readonly maxOutputTokens?: number;
+    readonly temperature?: number;
+}
+declare class QualityGrader implements Grader {
+    readonly kind = "llm_judge";
+    private readonly resolveJudgeProvider;
+    private readonly maxOutputTokens?;
+    private readonly temperature?;
+    constructor(options: QualityGraderOptions);
+    grade(context: GradeContext): Promise<GradeResult>;
+}
+
+type MaybePromise<T> = T | Promise<T>;
+interface EvaluationCache {
+    get(key: string): MaybePromise<ProviderResponse | undefined>;
+    set(key: string, value: ProviderResponse): MaybePromise<void>;
+}
+interface RunTestCaseOptions {
+    readonly testCase: TestCase;
+    readonly provider: Provider;
+    readonly target: ResolvedTarget;
+    readonly graders: Partial<Record<string, Grader>>;
+    readonly now?: () => Date;
+    readonly maxRetries?: number;
+    readonly agentTimeoutMs?: number;
+    readonly promptDumpDir?: string;
+    readonly cache?: EvaluationCache;
+    readonly useCache?: boolean;
+    readonly signal?: AbortSignal;
+    readonly judgeProvider?: Provider;
+}
+interface ProgressEvent {
+    readonly workerId: number;
+    readonly testId: string;
+    readonly status: "pending" | "running" | "completed" | "failed";
+    readonly startedAt?: number;
+    readonly completedAt?: number;
+    readonly error?: string;
+}
+interface RunEvaluationOptions {
+    readonly testFilePath: string;
+    readonly repoRoot: URL | string;
+    readonly target: ResolvedTarget;
+    readonly targets?: readonly TargetDefinition[];
+    readonly env?: EnvLookup;
+    readonly providerFactory?: (target: ResolvedTarget) => Provider;
+    readonly graders?: Partial<Record<string, Grader>>;
+    readonly maxRetries?: number;
+    readonly agentTimeoutMs?: number;
+    readonly promptDumpDir?: string;
+    readonly cache?: EvaluationCache;
+    readonly useCache?: boolean;
+    readonly now?: () => Date;
+    readonly testId?: string;
+    readonly verbose?: boolean;
+    readonly maxConcurrency?: number;
+    readonly onResult?: (result: EvaluationResult) => MaybePromise<void>;
+    readonly onProgress?: (event: ProgressEvent) => MaybePromise<void>;
+}
+declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
+declare function runTestCase(options: RunTestCaseOptions): Promise<EvaluationResult>;
+
+type AgentKernel = {
+    status: string;
+};
+declare function createAgentKernel(): AgentKernel;
+
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvaluationCache, type EvaluationResult, GRADER_KINDS, type GeminiResolvedConfig, type GradeContext, type GradeResult, type Grader, type GraderKind, HeuristicGrader, type HeuristicScore, type JsonObject, type JsonPrimitive, type JsonValue, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, QualityGrader, type QualityGraderOptions, type ResolvedTarget, type RunEvaluationOptions, type RunTestCaseOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestCase, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildPromptInputs, calculateHits, calculateMisses, createAgentKernel, createProvider, ensureVSCodeSubagents, extractAspects, extractCodeBlocks, getHitCount, isErrorLike, isGraderKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadTestCases, readTargetDefinitions, resolveAndCreateProvider, resolveTargetDefinition, runEvaluation, runTestCase, scoreCandidateResponse };