@granular-software/sdk 0.4.17 → 0.4.19

package/dist/agent-evals.d.mts

@@ -0,0 +1,320 @@
+import { P as Prompt, c as Environment, b as SessionHeapSnapshot, aZ as ManifestContent, ax as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, y as CreateEnvironmentData, i as GranularOptions } from './client-BQw_gUK3.mjs';
+import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
+import '@automerge/automerge';
+import '@automerge/automerge/slim';
+
+type EvalMatcher = string | RegExp;
+interface AgentEvalPromptInteraction {
+    promptId: string;
+    type: string;
+    title: string;
+    message: string;
+    answer: unknown;
+}
+interface AgentEvalGenerationInput {
+    systemPrompt: string;
+    history: Array<{
+        role: "user" | "assistant";
+        content: string;
+    }>;
+    request: string;
+    attempt: number;
+    repairIssues?: GeneratedJobCodeIssue[];
+}
+interface AgentEvalGenerationOutput {
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
+interface ScriptedPromptRule {
+    type?: Prompt["type"];
+    when?: EvalMatcher | EvalMatcher[];
+    answer: unknown | ((input: {
+        prompt: Prompt;
+        history: AgentEvalPromptInteraction[];
+    }) => unknown | Promise<unknown>);
+}
+type AgentEvalPromptResponder = (input: {
+    prompt: Prompt;
+    history: AgentEvalPromptInteraction[];
+}) => Promise<unknown>;
+interface AgentEvalExpectations {
+    replyIncludes?: EvalMatcher[];
+    replyExcludes?: EvalMatcher[];
+    actionIncludes?: EvalMatcher[];
+    actionExcludes?: EvalMatcher[];
+    codeIncludes?: EvalMatcher[];
+    codeExcludes?: EvalMatcher[];
+}
+interface AgentEvalCheckContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    result: unknown;
+    heap: SessionHeapSnapshot;
+    openPrompts: Prompt[];
+    liveDoc: Record<string, unknown>;
+    inspect: (code: string) => Promise<unknown>;
+    assertMatches: (label: string, text: string, includes?: EvalMatcher[], excludes?: EvalMatcher[]) => void;
+}
+type AgentEvalCheck = (context: AgentEvalCheckContext) => Promise<void> | void;
+interface AgentEvalInspection {
+    code: string;
+    includes?: EvalMatcher[];
+    excludes?: EvalMatcher[];
+    check?: AgentEvalCheck;
+}
+interface AgentEvalSetupContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalSetup {
+    manifest?: ManifestContent;
+    records?: RecordObjectOptions[];
+    effects?: ToolWithHandler[];
+    run?: (input: AgentEvalSetupContext) => Promise<void>;
+}
+interface AgentEvalPrepareContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalStep {
+    id?: string;
+    title?: string;
+    request: string;
+    setup?: AgentEvalSetup;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+    autoAnswerPrompts?: boolean;
+}
+interface AgentEvalScenario {
+    id: string;
+    request?: string;
+    steps?: AgentEvalStep[];
+    setup?: AgentEvalSetup;
+    prepareRecords?: RecordObjectOptions[];
+    prepareTools?: ToolWithHandler[];
+    prepare?: (input: AgentEvalPrepareContext) => Promise<void>;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    verify?: AgentEvalInspection;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+}
+interface AgentEvalStepResult {
+    id: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    inspectionResults: unknown[];
+    turnDir: string;
+}
+interface AgentEvalResult {
+    scenario: AgentEvalScenario;
+    status: "passed" | "failed";
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    steps?: AgentEvalStepResult[];
+    turnDir: string;
+    error?: string;
+}
+interface AgentEvalSuiteResult {
+    artifactDir: string;
+    results: AgentEvalResult[];
+}
+interface AgentEvalHarnessOptions {
+    granular: Granular;
+    environmentId?: string;
+    openEnvironment?: (input: {
+        label: string;
+        clientId: string;
+    }) => Promise<Environment>;
+    generator: AgentEvalTurnGenerator;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+interface AgentEvalConversation {
+    label: string;
+    environment: Environment;
+    history: AgentEvalHistoryEntry[];
+    promptEvents: PendingPromptEvent[];
+    artifactDir: string;
+    turnCount: number;
+}
+interface AgentEvalPendingTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    boundaryTimestamp: number;
+    finalCode: string;
+    finalReply: string;
+    stdout: string[];
+    stderr: string[];
+    prompts: Prompt[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    job: {
+        id: string;
+        result: Promise<unknown>;
+        on: (event: string, handler: (data: unknown) => void) => void;
+    };
+}
+interface AgentEvalCompletedTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    result: unknown;
+}
+interface AgentEvalHistoryEntry {
+    role: "user" | "assistant";
+    content: string;
+    code?: string;
+    jobStatus?: string;
+    jobResultPreview?: string;
+    error?: string;
+}
+interface PendingPromptEvent {
+    prompt: Prompt;
+    receivedAt: number;
+}
+declare function createTimestampedArtifactDirectory(options?: {
+    baseDir?: string;
+    suiteName?: string;
+}): string;
+declare function createScriptedPromptResponder(rules: ScriptedPromptRule[], fallback?: AgentEvalPromptResponder): AgentEvalPromptResponder;
+declare function createOpenAIChatTurnGenerator(options: {
+    apiKey: string;
+    model?: string;
+    baseUrl?: string;
+    temperature?: number;
+    headers?: Record<string, string>;
+}): AgentEvalTurnGenerator;
+declare function runAgentEvalSuite(options: {
+    harness: ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+type AgentTestTarget = {
+    environmentId: string;
+} | {
+    connect: ConnectOptions;
+} | {
+    createEnvironment: {
+        sandboxId: string;
+        data: CreateEnvironmentData;
+    };
+};
+interface AgentTesterOptions {
+    granular?: Granular;
+    client?: Granular;
+    apiKey?: string;
+    token?: string;
+    apiUrl?: string;
+    local?: boolean;
+    granularOptions?: Omit<GranularOptions, "apiKey" | "token" | "apiUrl" | "endpointMode">;
+    generator?: AgentEvalTurnGenerator;
+    openai?: {
+        apiKey: string;
+        model?: string;
+        baseUrl?: string;
+        temperature?: number;
+        headers?: Record<string, string>;
+    };
+    model?: string;
+    target: AgentTestTarget;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+declare function createAgentTester(options: AgentTesterOptions): {
+    granular: Granular;
+    generator: AgentEvalTurnGenerator;
+    environmentId: string | null;
+    openThread: (label: string) => Promise<AgentEvalConversation>;
+    closeThread: (conversation: AgentEvalConversation) => Promise<void>;
+    runStep: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+declare function runAgentTests(options: {
+    tester: ReturnType<typeof createAgentTester> | ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare const createHumanResponder: typeof createScriptedPromptResponder;
+declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
+declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
+
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };

package/dist/agent-evals.d.ts

@@ -0,0 +1,320 @@
+import { P as Prompt, c as Environment, b as SessionHeapSnapshot, aZ as ManifestContent, ax as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, y as CreateEnvironmentData, i as GranularOptions } from './client-BQw_gUK3.js';
+import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
+import '@automerge/automerge';
+import '@automerge/automerge/slim';
+
+type EvalMatcher = string | RegExp;
+interface AgentEvalPromptInteraction {
+    promptId: string;
+    type: string;
+    title: string;
+    message: string;
+    answer: unknown;
+}
+interface AgentEvalGenerationInput {
+    systemPrompt: string;
+    history: Array<{
+        role: "user" | "assistant";
+        content: string;
+    }>;
+    request: string;
+    attempt: number;
+    repairIssues?: GeneratedJobCodeIssue[];
+}
+interface AgentEvalGenerationOutput {
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
+interface ScriptedPromptRule {
+    type?: Prompt["type"];
+    when?: EvalMatcher | EvalMatcher[];
+    answer: unknown | ((input: {
+        prompt: Prompt;
+        history: AgentEvalPromptInteraction[];
+    }) => unknown | Promise<unknown>);
+}
+type AgentEvalPromptResponder = (input: {
+    prompt: Prompt;
+    history: AgentEvalPromptInteraction[];
+}) => Promise<unknown>;
+interface AgentEvalExpectations {
+    replyIncludes?: EvalMatcher[];
+    replyExcludes?: EvalMatcher[];
+    actionIncludes?: EvalMatcher[];
+    actionExcludes?: EvalMatcher[];
+    codeIncludes?: EvalMatcher[];
+    codeExcludes?: EvalMatcher[];
+}
+interface AgentEvalCheckContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    result: unknown;
+    heap: SessionHeapSnapshot;
+    openPrompts: Prompt[];
+    liveDoc: Record<string, unknown>;
+    inspect: (code: string) => Promise<unknown>;
+    assertMatches: (label: string, text: string, includes?: EvalMatcher[], excludes?: EvalMatcher[]) => void;
+}
+type AgentEvalCheck = (context: AgentEvalCheckContext) => Promise<void> | void;
+interface AgentEvalInspection {
+    code: string;
+    includes?: EvalMatcher[];
+    excludes?: EvalMatcher[];
+    check?: AgentEvalCheck;
+}
+interface AgentEvalSetupContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalSetup {
+    manifest?: ManifestContent;
+    records?: RecordObjectOptions[];
+    effects?: ToolWithHandler[];
+    run?: (input: AgentEvalSetupContext) => Promise<void>;
+}
+interface AgentEvalPrepareContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalStep {
+    id?: string;
+    title?: string;
+    request: string;
+    setup?: AgentEvalSetup;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+    autoAnswerPrompts?: boolean;
+}
+interface AgentEvalScenario {
+    id: string;
+    request?: string;
+    steps?: AgentEvalStep[];
+    setup?: AgentEvalSetup;
+    prepareRecords?: RecordObjectOptions[];
+    prepareTools?: ToolWithHandler[];
+    prepare?: (input: AgentEvalPrepareContext) => Promise<void>;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    verify?: AgentEvalInspection;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+}
+interface AgentEvalStepResult {
+    id: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    inspectionResults: unknown[];
+    turnDir: string;
+}
+interface AgentEvalResult {
+    scenario: AgentEvalScenario;
+    status: "passed" | "failed";
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    steps?: AgentEvalStepResult[];
+    turnDir: string;
+    error?: string;
+}
+interface AgentEvalSuiteResult {
+    artifactDir: string;
+    results: AgentEvalResult[];
+}
+interface AgentEvalHarnessOptions {
+    granular: Granular;
+    environmentId?: string;
+    openEnvironment?: (input: {
+        label: string;
+        clientId: string;
+    }) => Promise<Environment>;
+    generator: AgentEvalTurnGenerator;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+interface AgentEvalConversation {
+    label: string;
+    environment: Environment;
+    history: AgentEvalHistoryEntry[];
+    promptEvents: PendingPromptEvent[];
+    artifactDir: string;
+    turnCount: number;
+}
+interface AgentEvalPendingTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    boundaryTimestamp: number;
+    finalCode: string;
+    finalReply: string;
+    stdout: string[];
+    stderr: string[];
+    prompts: Prompt[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    job: {
+        id: string;
+        result: Promise<unknown>;
+        on: (event: string, handler: (data: unknown) => void) => void;
+    };
+}
+interface AgentEvalCompletedTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    result: unknown;
+}
+interface AgentEvalHistoryEntry {
+    role: "user" | "assistant";
+    content: string;
+    code?: string;
+    jobStatus?: string;
+    jobResultPreview?: string;
+    error?: string;
+}
+interface PendingPromptEvent {
+    prompt: Prompt;
+    receivedAt: number;
+}
+declare function createTimestampedArtifactDirectory(options?: {
+    baseDir?: string;
+    suiteName?: string;
+}): string;
+declare function createScriptedPromptResponder(rules: ScriptedPromptRule[], fallback?: AgentEvalPromptResponder): AgentEvalPromptResponder;
+declare function createOpenAIChatTurnGenerator(options: {
+    apiKey: string;
+    model?: string;
+    baseUrl?: string;
+    temperature?: number;
+    headers?: Record<string, string>;
+}): AgentEvalTurnGenerator;
+declare function runAgentEvalSuite(options: {
+    harness: ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+type AgentTestTarget = {
+    environmentId: string;
+} | {
+    connect: ConnectOptions;
+} | {
+    createEnvironment: {
+        sandboxId: string;
+        data: CreateEnvironmentData;
+    };
+};
+interface AgentTesterOptions {
+    granular?: Granular;
+    client?: Granular;
+    apiKey?: string;
+    token?: string;
+    apiUrl?: string;
+    local?: boolean;
+    granularOptions?: Omit<GranularOptions, "apiKey" | "token" | "apiUrl" | "endpointMode">;
+    generator?: AgentEvalTurnGenerator;
+    openai?: {
+        apiKey: string;
+        model?: string;
+        baseUrl?: string;
+        temperature?: number;
+        headers?: Record<string, string>;
+    };
+    model?: string;
+    target: AgentTestTarget;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+declare function createAgentTester(options: AgentTesterOptions): {
+    granular: Granular;
+    generator: AgentEvalTurnGenerator;
+    environmentId: string | null;
+    openThread: (label: string) => Promise<AgentEvalConversation>;
+    closeThread: (conversation: AgentEvalConversation) => Promise<void>;
+    runStep: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+declare function runAgentTests(options: {
+    tester: ReturnType<typeof createAgentTester> | ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare const createHumanResponder: typeof createScriptedPromptResponder;
+declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
+declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
+
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };