@lesliechan721/agent-skill-eval 0.1.0-beta.2 → 0.1.0-beta.20

package/dist/providers.d.ts

@@ -0,0 +1,83 @@
+import { runCodexConversationProviderCase, type AppServerRunEvidence, type CodexEvalProcessManagementEvidence } from './codex-eval-app-server.js';
+import { type AwSkillEvalIsolation, type AwSkillEvalTriggerSuiteCaseContract } from './codex-eval-cases.js';
+import { type CodexEvalRuntimeConfig } from './codex-eval-runtime-config.js';
+import { type TriggerRunEvidence } from './codex-trigger-eval.js';
+import { type EvalProviderGrade, type EvalProviderRunInput, type EvalProviderRunResult } from './custom-process.js';
+export declare const BUILTIN_EVAL_PROVIDER_IDS: readonly ["codex-trigger", "codex-conversation"];
+export type BuiltinEvalProviderId = typeof BUILTIN_EVAL_PROVIDER_IDS[number];
+export interface BuiltinEvalProviderRunInput extends EvalProviderRunInput {
+    evalRoot: string;
+}
+export interface CodexTriggerProviderCase {
+    suite: AwSkillEvalTriggerSuiteCaseContract;
+    value: Record<string, unknown>;
+}
+export interface CodexConversationProviderCase {
+    value: Record<string, unknown>;
+    skillName: string;
+    skillNames: string[];
+    isolation: AwSkillEvalIsolation;
+    suitePathPrepend?: string[];
+    assertions: unknown;
+    allowedVariance?: unknown;
+}
+export interface CodexTriggerTransportInput extends Omit<BuiltinEvalProviderRunInput, 'case'> {
+    case: CodexTriggerProviderCase;
+}
+export interface CodexConversationTransportInput extends Omit<BuiltinEvalProviderRunInput, 'case'> {
+    case: CodexConversationProviderCase;
+}
+export interface CodexGraderTransportInput {
+    prompt: string;
+    outputDir: string;
+    timeoutMs: number;
+    signal?: AbortSignal;
+}
+export interface CodexTriggerTransportResult {
+    evidence: TriggerRunEvidence;
+    usage: unknown;
+}
+export interface CodexConversationTransportResult {
+    evidence: AppServerRunEvidence;
+    modelOutput: string;
+    usage: unknown;
+}
+export interface CodexGraderTransportResult {
+    output: string;
+    usage: unknown;
+    mechanicalPassed: boolean;
+    diagnostics?: string[];
+}
+export interface CodexEvalTransport {
+    runTrigger(input: CodexTriggerTransportInput): Promise<CodexTriggerTransportResult>;
+    runConversation(input: CodexConversationTransportInput): Promise<CodexConversationTransportResult>;
+    runGrader(input: CodexGraderTransportInput): Promise<CodexGraderTransportResult>;
+}
+export interface BuiltinEvalProviderDependencies {
+    transport?: CodexEvalTransport;
+    runConversationProviderCase?: typeof runCodexConversationProviderCase;
+    sourceEnv?: NodeJS.ProcessEnv;
+    codexBin?: string;
+    model?: string;
+    codexRuntime?: CodexEvalRuntimeConfig;
+    processManagement?: CodexEvalProcessManagementEvidence;
+}
+export interface BuiltinEvalProvider {
+    readonly id: BuiltinEvalProviderId;
+    run(input: BuiltinEvalProviderRunInput, dependencies?: BuiltinEvalProviderDependencies): Promise<EvalProviderRunResult>;
+}
+export declare class EvalProviderExecutionError extends Error {
+    readonly providerId: BuiltinEvalProviderId;
+    readonly stage: 'evaluation' | 'grader' | 'output';
+    constructor(providerId: BuiltinEvalProviderId, stage: 'evaluation' | 'grader' | 'output', message: string);
+}
+export declare function conversationModelOutputOf(evidence: AppServerRunEvidence): string;
+export declare function createConversationGraderPrompt(options: {
+    assertions: unknown;
+    allowedVariance?: unknown;
+    modelOutput: string;
+    mechanicalEvidence: Record<string, unknown>;
+}): string;
+export declare function parseConversationGraderOutput(output: string): EvalProviderGrade;
+export declare function getBuiltinEvalProvider(id: string): BuiltinEvalProvider;
+export declare function runBuiltinEvalProvider(id: string, input: BuiltinEvalProviderRunInput, dependencies?: BuiltinEvalProviderDependencies): Promise<EvalProviderRunResult>;