@granular-software/sdk 0.4.41 → 0.4.43

package/dist/agent-evals.d.mts

@@ -1,6 +1,6 @@
 import { a as EnvironmentSession, G as Granular } from './client-CBQFvuKf.mjs';
-import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bM as ManifestContent, ba as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.mjs';
-import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
+import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, ba as RecordObjectOptions, bM as ManifestContent, c as SessionHeapSnapshot, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.mjs';
+import { GranularAgentToolInfo, GeneratedJobCodeIssue, BuildGranularAgentSystemPromptInput, HarnessRenderedPrompt, HarnessRenderedContinuation, HarnessControllerBudgets } from './agent-harness.mjs';
 import '@automerge/automerge';
 import '@automerge/automerge/slim';
 import '@granular-software/policy-engine';
@@ -26,6 +26,8 @@ interface AgentEvalGenerationInput {
     onTextDelta?: (delta: string) => void | Promise<void>;
     usageContext?: GranularSpendContext;
 }
+type AgentEvalPromptRenderer = (input: BuildGranularAgentSystemPromptInput) => HarnessRenderedPrompt;
+type AgentEvalContinuationRenderer = (resultPreview?: string) => HarnessRenderedContinuation;
 interface AgentEvalGenerationOutput {
     reply?: string;
     code?: string;
@@ -82,6 +84,92 @@ interface AgentEvalExpectations {
     actionExcludes?: EvalMatcher[];
     codeIncludes?: EvalMatcher[];
     codeExcludes?: EvalMatcher[];
+    behaviorBuckets?: string[];
+    actions?: {
+        required?: EvalMatcher[];
+        forbidden?: EvalMatcher[];
+    };
+    prompts?: {
+        requiredChoiceWhenAmbiguous?: boolean;
+        requiredConfirmationBefore?: EvalMatcher[];
+        forbiddenPromptTypes?: string[];
+    };
+    presentation?: {
+        mustMention?: EvalMatcher[];
+        mustNotMention?: EvalMatcher[];
+        mustDisplayOrSave?: EvalMatcher[];
+    };
+}
+interface AgentEvalScenarioFilePromptRule {
+    type?: Prompt["type"];
+    when?: string | string[];
+    answer: unknown;
+}
+interface AgentEvalScenarioFileStep {
+    id?: string;
+    request: string;
+    behaviorBuckets?: string[];
+    human?: AgentEvalScenarioFilePromptRule[];
+    inspect?: AgentEvalScenarioFileInspection | AgentEvalScenarioFileInspection[];
+    expect?: {
+        replyIncludes?: string[];
+        replyExcludes?: string[];
+        actionIncludes?: string[];
+        actionExcludes?: string[];
+        codeIncludes?: string[];
+        codeExcludes?: string[];
+        behaviorBuckets?: string[];
+        actions?: {
+            required?: string[];
+            forbidden?: string[];
+        };
+        prompts?: AgentEvalExpectations["prompts"];
+        presentation?: {
+            mustMention?: string[];
+            mustNotMention?: string[];
+            mustDisplayOrSave?: string[];
+        };
+    };
+}
+interface AgentEvalScenarioFileInspection {
+    code: string;
+    includes?: string[];
+    excludes?: string[];
+}
+interface AgentEvalScenarioFileScenario {
+    id: string;
+    description?: string;
+    request?: string;
+    behaviorBuckets?: string[];
+    human?: AgentEvalScenarioFilePromptRule[];
+    steps?: AgentEvalScenarioFileStep[];
+    prepareRecords?: RecordObjectOptions[];
+    verify?: AgentEvalScenarioFileInspection;
+    inspect?: AgentEvalScenarioFileInspection | AgentEvalScenarioFileInspection[];
+    expect?: AgentEvalScenarioFileStep["expect"];
+}
+interface AgentEvalScenarioFileEnvironmentTarget {
+    mode: "environment";
+}
+interface AgentEvalScenarioFileSelfContainedTarget {
+    mode: "self-contained";
+    sandboxName?: string;
+    tag?: string;
+    userId?: string;
+    name?: string;
+    email?: string;
+    permissions?: string[];
+    effectPreset?: string;
+    extraRecords?: RecordObjectOptions[];
+}
+interface AgentEvalScenarioFile {
+    id: string;
+    title?: string;
+    description?: string;
+    domainDocumentation?: string;
+    manifest?: ManifestContent;
+    target?: AgentEvalScenarioFileSelfContainedTarget | AgentEvalScenarioFileEnvironmentTarget;
+    scenarios: AgentEvalScenarioFileScenario[];
 }
 interface AgentEvalCheckContext {
     conversation: AgentEvalConversation;
@@ -127,6 +215,7 @@ interface AgentEvalStep {
     id?: string;
     title?: string;
     request: string;
+    behaviorBuckets?: string[];
     setup?: AgentEvalSetup;
     human?: AgentEvalPromptResponder;
     expect?: AgentEvalExpectations;
@@ -139,6 +228,7 @@ interface AgentEvalScenario {
     id: string;
     description?: string;
     jobSource?: "llm" | "hardcoded" | "mixed";
+    behaviorBuckets?: string[];
     request?: string;
     steps?: AgentEvalStep[];
     setup?: AgentEvalSetup;
@@ -161,8 +251,15 @@ interface AgentEvalStepResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     inspectionResults: unknown[];
+    assertions?: AgentEvalAssertion[];
     turnDir: string;
 }
+interface AgentEvalAssertion {
+    id: string;
+    label: string;
+    status: "passed" | "failed";
+    message?: string;
+}
 interface AgentEvalResult {
     scenario: AgentEvalScenario;
     status: "passed" | "failed";
@@ -172,6 +269,7 @@ interface AgentEvalResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     verification: unknown | null;
+    assertions?: AgentEvalAssertion[];
     tokenUsage?: AgentEvalTokenUsage;
     steps?: AgentEvalStepResult[];
     turnDir: string;
@@ -195,6 +293,26 @@ interface AgentEvalHarnessOptions {
     chatTimeoutMs?: number;
     jobTimeoutMs?: number;
     pollIntervalMs?: number;
+    harnessTemplateId?: string;
+    promptRenderer?: AgentEvalPromptRenderer;
+    continuationRenderer?: AgentEvalContinuationRenderer;
+    onProgress?: (event: AgentEvalProgressEvent) => void | Promise<void>;
+}
+type AgentEvalProgressStatus = "info" | "running" | "passed" | "failed";
+interface AgentEvalProgressEvent {
+    id: string;
+    timestamp: string;
+    phase: "scenario" | "setup" | "step" | "prompt" | "generation" | "job" | "interaction" | "continuation" | "assertion" | "inspection" | "check";
+    status: AgentEvalProgressStatus;
+    scenarioId?: string;
+    stepId?: string;
+    iteration?: number;
+    jobId?: string;
+    templateId?: string;
+    templateVersion?: string;
+    title: string;
+    message?: string;
+    data?: unknown;
 }
 interface AgentEvalConversation {
     label: string;
@@ -209,6 +327,10 @@ interface AgentEvalSessionLogIteration {
     iteration: number;
     request: string;
     systemPrompt: string;
+    templateId?: string;
+    templateVersion?: string;
+    templateHash?: string;
+    promptInstanceHash?: string;
     generationReply?: string;
     generatedCode?: string;
     rawGeneration?: unknown;
@@ -278,6 +400,8 @@ interface PendingPromptEvent {
     prompt: Prompt;
     receivedAt: number;
 }
+declare function scenariosFromAgentEvalFile(file: AgentEvalScenarioFile): AgentEvalScenario[];
+declare function loadAgentEvalScenarioFile(filePath: string): Promise<AgentEvalScenario[]>;
 declare function createTimestampedArtifactDirectory(options?: {
     baseDir?: string;
     suiteName?: string;
@@ -300,6 +424,9 @@ declare function runAgentEvalSuite(options: {
 declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
     artifactDir: string;
     granular: Granular;
+    emitProgress: (event: Omit<AgentEvalProgressEvent, "id" | "timestamp"> & {
+        id?: string;
+    }) => Promise<void>;
     openConversation: (label: string) => Promise<AgentEvalConversation>;
     closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
     runTurn: (input: {
@@ -349,6 +476,10 @@ interface AgentTesterOptions {
     chatTimeoutMs?: number;
     jobTimeoutMs?: number;
     pollIntervalMs?: number;
+    harnessTemplateId?: string;
+    promptRenderer?: AgentEvalPromptRenderer;
+    continuationRenderer?: AgentEvalContinuationRenderer;
+    onProgress?: (event: AgentEvalProgressEvent) => void | Promise<void>;
 }
 declare function createAgentTester(options: AgentTesterOptions): {
     granular: Granular;
@@ -370,6 +501,9 @@ declare function createAgentTester(options: AgentTesterOptions): {
     continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
     runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
     artifactDir: string;
+    emitProgress: (event: Omit<AgentEvalProgressEvent, "id" | "timestamp"> & {
+        id?: string;
+    }) => Promise<void>;
     openConversation: (label: string) => Promise<AgentEvalConversation>;
     closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
     runTurn: (input: {
@@ -393,4 +527,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
 declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
 declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
 
-export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, runAgentEvalSuite, runAgentTests };
+export { type AgentEvalAssertion, type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalContinuationRenderer, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalProgressEvent, type AgentEvalProgressStatus, type AgentEvalPromptInteraction, type AgentEvalPromptRenderer, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalScenarioFile, type AgentEvalScenarioFileEnvironmentTarget, type AgentEvalScenarioFileInspection, type AgentEvalScenarioFilePromptRule, type AgentEvalScenarioFileScenario, type AgentEvalScenarioFileSelfContainedTarget, type AgentEvalScenarioFileStep, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, loadAgentEvalScenarioFile, runAgentEvalSuite, runAgentTests, scenariosFromAgentEvalFile };

package/dist/agent-evals.d.ts

@@ -1,6 +1,6 @@
 import { a as EnvironmentSession, G as Granular } from './client-BZ8NuQ_e.js';
-import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, c as SessionHeapSnapshot, bM as ManifestContent, ba as RecordObjectOptions, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.js';
-import { GranularAgentToolInfo, GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
+import { G as GranularSpendContext, h as OpenAITokenSpend, P as Prompt, ba as RecordObjectOptions, bM as ManifestContent, c as SessionHeapSnapshot, T as ToolWithHandler, C as ConnectOptions, a6 as CreateEnvironmentData, q as GranularOptions } from './spend-tAz2a16I.js';
+import { GranularAgentToolInfo, GeneratedJobCodeIssue, BuildGranularAgentSystemPromptInput, HarnessRenderedPrompt, HarnessRenderedContinuation, HarnessControllerBudgets } from './agent-harness.js';
 import '@automerge/automerge';
 import '@automerge/automerge/slim';
 import '@granular-software/policy-engine';
@@ -26,6 +26,8 @@ interface AgentEvalGenerationInput {
     onTextDelta?: (delta: string) => void | Promise<void>;
     usageContext?: GranularSpendContext;
 }
+type AgentEvalPromptRenderer = (input: BuildGranularAgentSystemPromptInput) => HarnessRenderedPrompt;
+type AgentEvalContinuationRenderer = (resultPreview?: string) => HarnessRenderedContinuation;
 interface AgentEvalGenerationOutput {
     reply?: string;
     code?: string;
@@ -82,6 +84,92 @@ interface AgentEvalExpectations {
     actionExcludes?: EvalMatcher[];
     codeIncludes?: EvalMatcher[];
     codeExcludes?: EvalMatcher[];
+    behaviorBuckets?: string[];
+    actions?: {
+        required?: EvalMatcher[];
+        forbidden?: EvalMatcher[];
+    };
+    prompts?: {
+        requiredChoiceWhenAmbiguous?: boolean;
+        requiredConfirmationBefore?: EvalMatcher[];
+        forbiddenPromptTypes?: string[];
+    };
+    presentation?: {
+        mustMention?: EvalMatcher[];
+        mustNotMention?: EvalMatcher[];
+        mustDisplayOrSave?: EvalMatcher[];
+    };
+}
+interface AgentEvalScenarioFilePromptRule {
+    type?: Prompt["type"];
+    when?: string | string[];
+    answer: unknown;
+}
+interface AgentEvalScenarioFileStep {
+    id?: string;
+    request: string;
+    behaviorBuckets?: string[];
+    human?: AgentEvalScenarioFilePromptRule[];
+    inspect?: AgentEvalScenarioFileInspection | AgentEvalScenarioFileInspection[];
+    expect?: {
+        replyIncludes?: string[];
+        replyExcludes?: string[];
+        actionIncludes?: string[];
+        actionExcludes?: string[];
+        codeIncludes?: string[];
+        codeExcludes?: string[];
+        behaviorBuckets?: string[];
+        actions?: {
+            required?: string[];
+            forbidden?: string[];
+        };
+        prompts?: AgentEvalExpectations["prompts"];
+        presentation?: {
+            mustMention?: string[];
+            mustNotMention?: string[];
+            mustDisplayOrSave?: string[];
+        };
+    };
+}
+interface AgentEvalScenarioFileInspection {
+    code: string;
+    includes?: string[];
+    excludes?: string[];
+}
+interface AgentEvalScenarioFileScenario {
+    id: string;
+    description?: string;
+    request?: string;
+    behaviorBuckets?: string[];
+    human?: AgentEvalScenarioFilePromptRule[];
+    steps?: AgentEvalScenarioFileStep[];
+    prepareRecords?: RecordObjectOptions[];
+    verify?: AgentEvalScenarioFileInspection;
+    inspect?: AgentEvalScenarioFileInspection | AgentEvalScenarioFileInspection[];
+    expect?: AgentEvalScenarioFileStep["expect"];
+}
+interface AgentEvalScenarioFileEnvironmentTarget {
+    mode: "environment";
+}
+interface AgentEvalScenarioFileSelfContainedTarget {
+    mode: "self-contained";
+    sandboxName?: string;
+    tag?: string;
+    userId?: string;
+    name?: string;
+    email?: string;
+    permissions?: string[];
+    effectPreset?: string;
+    extraRecords?: RecordObjectOptions[];
+}
+interface AgentEvalScenarioFile {
+    id: string;
+    title?: string;
+    description?: string;
+    domainDocumentation?: string;
+    manifest?: ManifestContent;
+    target?: AgentEvalScenarioFileSelfContainedTarget | AgentEvalScenarioFileEnvironmentTarget;
+    scenarios: AgentEvalScenarioFileScenario[];
 }
 interface AgentEvalCheckContext {
     conversation: AgentEvalConversation;
@@ -127,6 +215,7 @@ interface AgentEvalStep {
     id?: string;
     title?: string;
     request: string;
+    behaviorBuckets?: string[];
     setup?: AgentEvalSetup;
     human?: AgentEvalPromptResponder;
     expect?: AgentEvalExpectations;
@@ -139,6 +228,7 @@ interface AgentEvalScenario {
     id: string;
     description?: string;
     jobSource?: "llm" | "hardcoded" | "mixed";
+    behaviorBuckets?: string[];
     request?: string;
     steps?: AgentEvalStep[];
     setup?: AgentEvalSetup;
@@ -161,8 +251,15 @@ interface AgentEvalStepResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     inspectionResults: unknown[];
+    assertions?: AgentEvalAssertion[];
     turnDir: string;
 }
+interface AgentEvalAssertion {
+    id: string;
+    label: string;
+    status: "passed" | "failed";
+    message?: string;
+}
 interface AgentEvalResult {
     scenario: AgentEvalScenario;
     status: "passed" | "failed";
@@ -172,6 +269,7 @@ interface AgentEvalResult {
     actionSummary: string[];
     promptInteractions: AgentEvalPromptInteraction[];
     verification: unknown | null;
+    assertions?: AgentEvalAssertion[];
     tokenUsage?: AgentEvalTokenUsage;
     steps?: AgentEvalStepResult[];
     turnDir: string;
@@ -195,6 +293,26 @@ interface AgentEvalHarnessOptions {
     chatTimeoutMs?: number;
     jobTimeoutMs?: number;
     pollIntervalMs?: number;
+    harnessTemplateId?: string;
+    promptRenderer?: AgentEvalPromptRenderer;
+    continuationRenderer?: AgentEvalContinuationRenderer;
+    onProgress?: (event: AgentEvalProgressEvent) => void | Promise<void>;
+}
+type AgentEvalProgressStatus = "info" | "running" | "passed" | "failed";
+interface AgentEvalProgressEvent {
+    id: string;
+    timestamp: string;
+    phase: "scenario" | "setup" | "step" | "prompt" | "generation" | "job" | "interaction" | "continuation" | "assertion" | "inspection" | "check";
+    status: AgentEvalProgressStatus;
+    scenarioId?: string;
+    stepId?: string;
+    iteration?: number;
+    jobId?: string;
+    templateId?: string;
+    templateVersion?: string;
+    title: string;
+    message?: string;
+    data?: unknown;
 }
 interface AgentEvalConversation {
     label: string;
@@ -209,6 +327,10 @@ interface AgentEvalSessionLogIteration {
     iteration: number;
     request: string;
     systemPrompt: string;
+    templateId?: string;
+    templateVersion?: string;
+    templateHash?: string;
+    promptInstanceHash?: string;
     generationReply?: string;
     generatedCode?: string;
     rawGeneration?: unknown;
@@ -278,6 +400,8 @@ interface PendingPromptEvent {
     prompt: Prompt;
     receivedAt: number;
 }
+declare function scenariosFromAgentEvalFile(file: AgentEvalScenarioFile): AgentEvalScenario[];
+declare function loadAgentEvalScenarioFile(filePath: string): Promise<AgentEvalScenario[]>;
 declare function createTimestampedArtifactDirectory(options?: {
     baseDir?: string;
     suiteName?: string;
@@ -300,6 +424,9 @@ declare function runAgentEvalSuite(options: {
 declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
     artifactDir: string;
     granular: Granular;
+    emitProgress: (event: Omit<AgentEvalProgressEvent, "id" | "timestamp"> & {
+        id?: string;
+    }) => Promise<void>;
     openConversation: (label: string) => Promise<AgentEvalConversation>;
     closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
     runTurn: (input: {
@@ -349,6 +476,10 @@ interface AgentTesterOptions {
     chatTimeoutMs?: number;
     jobTimeoutMs?: number;
     pollIntervalMs?: number;
+    harnessTemplateId?: string;
+    promptRenderer?: AgentEvalPromptRenderer;
+    continuationRenderer?: AgentEvalContinuationRenderer;
+    onProgress?: (event: AgentEvalProgressEvent) => void | Promise<void>;
 }
 declare function createAgentTester(options: AgentTesterOptions): {
     granular: Granular;
@@ -370,6 +501,9 @@ declare function createAgentTester(options: AgentTesterOptions): {
     continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
     runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
     artifactDir: string;
+    emitProgress: (event: Omit<AgentEvalProgressEvent, "id" | "timestamp"> & {
+        id?: string;
+    }) => Promise<void>;
     openConversation: (label: string) => Promise<AgentEvalConversation>;
     closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
     runTurn: (input: {
@@ -393,4 +527,4 @@ declare const createHumanResponder: typeof createScriptedPromptResponder;
 declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
 declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
 
-export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, runAgentEvalSuite, runAgentTests };
+export { type AgentEvalAssertion, type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalContinuationRenderer, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationAttempt, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalOpenAIUsageEvent, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalProgressEvent, type AgentEvalProgressStatus, type AgentEvalPromptInteraction, type AgentEvalPromptRenderer, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalScenarioFile, type AgentEvalScenarioFileEnvironmentTarget, type AgentEvalScenarioFileInspection, type AgentEvalScenarioFilePromptRule, type AgentEvalScenarioFileScenario, type AgentEvalScenarioFileSelfContainedTarget, type AgentEvalScenarioFileStep, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTokenUsage, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, loadAgentEvalScenarioFile, runAgentEvalSuite, runAgentTests, scenariosFromAgentEvalFile };