@granular-software/sdk 0.4.18 → 0.4.20

package/README.md

@@ -456,6 +456,16 @@ Defines domain ontology: classes (with typed properties), and relationships (wit
 ### `environment.recordObject(options)`
 Creates or updates a class instance with fields and relationships. Returns `{ path, id, created }`.
 
+### `environment.recordObjects(records, options?)`
+Batch upsert for many instances. The SDK sends **chunks** (default **100** rows per HTTP `POST` to `/records/batch`) with **retries** on transient failures, so large arrays do not time out as a single oversized request.
+
+Optional **`options`**:
+- **`batchSize`** — max rows per request (default 100).
+- **`concurrency`** — how many chunk requests may run in parallel (default 1, max 16); can reduce wall time when the server can overlap work.
+- **`onChunkComplete`** — async-friendly hook after each chunk for progress UIs; the returned array is always ordered like `records`.
+
+**Sync batch vs queued import:** use **`recordObjects`** when you need **synchronous** commits and/or per-chunk feedback. Use **`enqueueRecordImport`** + **`getRecordImport` / `getRecordImportSummary`** for **background** ingestion with aggregate counters when admission latency matters more than immediate row-by-row completion.
+
 ### `environment.getRelationships(modelPath)`
 Returns relationship definitions for a given class.
 

package/dist/agent-evals.d.mts

@@ -0,0 +1,320 @@
+import { P as Prompt, c as Environment, b as SessionHeapSnapshot, a$ as ManifestContent, ax as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, y as CreateEnvironmentData, i as GranularOptions } from './client-DWYdWpS-.mjs';
+import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.mjs';
+import '@automerge/automerge';
+import '@automerge/automerge/slim';
+
+type EvalMatcher = string | RegExp;
+interface AgentEvalPromptInteraction {
+    promptId: string;
+    type: string;
+    title: string;
+    message: string;
+    answer: unknown;
+}
+interface AgentEvalGenerationInput {
+    systemPrompt: string;
+    history: Array<{
+        role: "user" | "assistant";
+        content: string;
+    }>;
+    request: string;
+    attempt: number;
+    repairIssues?: GeneratedJobCodeIssue[];
+}
+interface AgentEvalGenerationOutput {
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
+interface ScriptedPromptRule {
+    type?: Prompt["type"];
+    when?: EvalMatcher | EvalMatcher[];
+    answer: unknown | ((input: {
+        prompt: Prompt;
+        history: AgentEvalPromptInteraction[];
+    }) => unknown | Promise<unknown>);
+}
+type AgentEvalPromptResponder = (input: {
+    prompt: Prompt;
+    history: AgentEvalPromptInteraction[];
+}) => Promise<unknown>;
+interface AgentEvalExpectations {
+    replyIncludes?: EvalMatcher[];
+    replyExcludes?: EvalMatcher[];
+    actionIncludes?: EvalMatcher[];
+    actionExcludes?: EvalMatcher[];
+    codeIncludes?: EvalMatcher[];
+    codeExcludes?: EvalMatcher[];
+}
+interface AgentEvalCheckContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    result: unknown;
+    heap: SessionHeapSnapshot;
+    openPrompts: Prompt[];
+    liveDoc: Record<string, unknown>;
+    inspect: (code: string) => Promise<unknown>;
+    assertMatches: (label: string, text: string, includes?: EvalMatcher[], excludes?: EvalMatcher[]) => void;
+}
+type AgentEvalCheck = (context: AgentEvalCheckContext) => Promise<void> | void;
+interface AgentEvalInspection {
+    code: string;
+    includes?: EvalMatcher[];
+    excludes?: EvalMatcher[];
+    check?: AgentEvalCheck;
+}
+interface AgentEvalSetupContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalSetup {
+    manifest?: ManifestContent;
+    records?: RecordObjectOptions[];
+    effects?: ToolWithHandler[];
+    run?: (input: AgentEvalSetupContext) => Promise<void>;
+}
+interface AgentEvalPrepareContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalStep {
+    id?: string;
+    title?: string;
+    request: string;
+    setup?: AgentEvalSetup;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+    autoAnswerPrompts?: boolean;
+}
+interface AgentEvalScenario {
+    id: string;
+    request?: string;
+    steps?: AgentEvalStep[];
+    setup?: AgentEvalSetup;
+    prepareRecords?: RecordObjectOptions[];
+    prepareTools?: ToolWithHandler[];
+    prepare?: (input: AgentEvalPrepareContext) => Promise<void>;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    verify?: AgentEvalInspection;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+}
+interface AgentEvalStepResult {
+    id: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    inspectionResults: unknown[];
+    turnDir: string;
+}
+interface AgentEvalResult {
+    scenario: AgentEvalScenario;
+    status: "passed" | "failed";
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    steps?: AgentEvalStepResult[];
+    turnDir: string;
+    error?: string;
+}
+interface AgentEvalSuiteResult {
+    artifactDir: string;
+    results: AgentEvalResult[];
+}
+interface AgentEvalHarnessOptions {
+    granular: Granular;
+    environmentId?: string;
+    openEnvironment?: (input: {
+        label: string;
+        clientId: string;
+    }) => Promise<Environment>;
+    generator: AgentEvalTurnGenerator;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+interface AgentEvalConversation {
+    label: string;
+    environment: Environment;
+    history: AgentEvalHistoryEntry[];
+    promptEvents: PendingPromptEvent[];
+    artifactDir: string;
+    turnCount: number;
+}
+interface AgentEvalPendingTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    boundaryTimestamp: number;
+    finalCode: string;
+    finalReply: string;
+    stdout: string[];
+    stderr: string[];
+    prompts: Prompt[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    job: {
+        id: string;
+        result: Promise<unknown>;
+        on: (event: string, handler: (data: unknown) => void) => void;
+    };
+}
+interface AgentEvalCompletedTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    result: unknown;
+}
+interface AgentEvalHistoryEntry {
+    role: "user" | "assistant";
+    content: string;
+    code?: string;
+    jobStatus?: string;
+    jobResultPreview?: string;
+    error?: string;
+}
+interface PendingPromptEvent {
+    prompt: Prompt;
+    receivedAt: number;
+}
+declare function createTimestampedArtifactDirectory(options?: {
+    baseDir?: string;
+    suiteName?: string;
+}): string;
+declare function createScriptedPromptResponder(rules: ScriptedPromptRule[], fallback?: AgentEvalPromptResponder): AgentEvalPromptResponder;
+declare function createOpenAIChatTurnGenerator(options: {
+    apiKey: string;
+    model?: string;
+    baseUrl?: string;
+    temperature?: number;
+    headers?: Record<string, string>;
+}): AgentEvalTurnGenerator;
+declare function runAgentEvalSuite(options: {
+    harness: ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+type AgentTestTarget = {
+    environmentId: string;
+} | {
+    connect: ConnectOptions;
+} | {
+    createEnvironment: {
+        sandboxId: string;
+        data: CreateEnvironmentData;
+    };
+};
+interface AgentTesterOptions {
+    granular?: Granular;
+    client?: Granular;
+    apiKey?: string;
+    token?: string;
+    apiUrl?: string;
+    local?: boolean;
+    granularOptions?: Omit<GranularOptions, "apiKey" | "token" | "apiUrl" | "endpointMode">;
+    generator?: AgentEvalTurnGenerator;
+    openai?: {
+        apiKey: string;
+        model?: string;
+        baseUrl?: string;
+        temperature?: number;
+        headers?: Record<string, string>;
+    };
+    model?: string;
+    target: AgentTestTarget;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+declare function createAgentTester(options: AgentTesterOptions): {
+    granular: Granular;
+    generator: AgentEvalTurnGenerator;
+    environmentId: string | null;
+    openThread: (label: string) => Promise<AgentEvalConversation>;
+    closeThread: (conversation: AgentEvalConversation) => Promise<void>;
+    runStep: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+declare function runAgentTests(options: {
+    tester: ReturnType<typeof createAgentTester> | ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare const createHumanResponder: typeof createScriptedPromptResponder;
+declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
+declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
+
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };

package/dist/agent-evals.d.ts

@@ -0,0 +1,320 @@
+import { P as Prompt, c as Environment, b as SessionHeapSnapshot, a$ as ManifestContent, ax as RecordObjectOptions, T as ToolWithHandler, G as Granular, C as ConnectOptions, y as CreateEnvironmentData, i as GranularOptions } from './client-DWYdWpS-.js';
+import { GeneratedJobCodeIssue, HarnessControllerBudgets } from './agent-harness.js';
+import '@automerge/automerge';
+import '@automerge/automerge/slim';
+
+type EvalMatcher = string | RegExp;
+interface AgentEvalPromptInteraction {
+    promptId: string;
+    type: string;
+    title: string;
+    message: string;
+    answer: unknown;
+}
+interface AgentEvalGenerationInput {
+    systemPrompt: string;
+    history: Array<{
+        role: "user" | "assistant";
+        content: string;
+    }>;
+    request: string;
+    attempt: number;
+    repairIssues?: GeneratedJobCodeIssue[];
+}
+interface AgentEvalGenerationOutput {
+    reply?: string;
+    code?: string;
+    raw?: unknown;
+}
+type AgentEvalTurnGenerator = (input: AgentEvalGenerationInput) => Promise<AgentEvalGenerationOutput>;
+interface ScriptedPromptRule {
+    type?: Prompt["type"];
+    when?: EvalMatcher | EvalMatcher[];
+    answer: unknown | ((input: {
+        prompt: Prompt;
+        history: AgentEvalPromptInteraction[];
+    }) => unknown | Promise<unknown>);
+}
+type AgentEvalPromptResponder = (input: {
+    prompt: Prompt;
+    history: AgentEvalPromptInteraction[];
+}) => Promise<unknown>;
+interface AgentEvalExpectations {
+    replyIncludes?: EvalMatcher[];
+    replyExcludes?: EvalMatcher[];
+    actionIncludes?: EvalMatcher[];
+    actionExcludes?: EvalMatcher[];
+    codeIncludes?: EvalMatcher[];
+    codeExcludes?: EvalMatcher[];
+}
+interface AgentEvalCheckContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    result: unknown;
+    heap: SessionHeapSnapshot;
+    openPrompts: Prompt[];
+    liveDoc: Record<string, unknown>;
+    inspect: (code: string) => Promise<unknown>;
+    assertMatches: (label: string, text: string, includes?: EvalMatcher[], excludes?: EvalMatcher[]) => void;
+}
+type AgentEvalCheck = (context: AgentEvalCheckContext) => Promise<void> | void;
+interface AgentEvalInspection {
+    code: string;
+    includes?: EvalMatcher[];
+    excludes?: EvalMatcher[];
+    check?: AgentEvalCheck;
+}
+interface AgentEvalSetupContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalSetup {
+    manifest?: ManifestContent;
+    records?: RecordObjectOptions[];
+    effects?: ToolWithHandler[];
+    run?: (input: AgentEvalSetupContext) => Promise<void>;
+}
+interface AgentEvalPrepareContext {
+    conversation: AgentEvalConversation;
+    environment: Environment;
+    turnDir: string;
+}
+interface AgentEvalStep {
+    id?: string;
+    title?: string;
+    request: string;
+    setup?: AgentEvalSetup;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+    autoAnswerPrompts?: boolean;
+}
+interface AgentEvalScenario {
+    id: string;
+    request?: string;
+    steps?: AgentEvalStep[];
+    setup?: AgentEvalSetup;
+    prepareRecords?: RecordObjectOptions[];
+    prepareTools?: ToolWithHandler[];
+    prepare?: (input: AgentEvalPrepareContext) => Promise<void>;
+    human?: AgentEvalPromptResponder;
+    expect?: AgentEvalExpectations;
+    verify?: AgentEvalInspection;
+    inspect?: AgentEvalInspection | AgentEvalInspection[];
+    check?: AgentEvalCheck | AgentEvalCheck[];
+    maxIterations?: number;
+}
+interface AgentEvalStepResult {
+    id: string;
+    request: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    inspectionResults: unknown[];
+    turnDir: string;
+}
+interface AgentEvalResult {
+    scenario: AgentEvalScenario;
+    status: "passed" | "failed";
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    steps?: AgentEvalStepResult[];
+    turnDir: string;
+    error?: string;
+}
+interface AgentEvalSuiteResult {
+    artifactDir: string;
+    results: AgentEvalResult[];
+}
+interface AgentEvalHarnessOptions {
+    granular: Granular;
+    environmentId?: string;
+    openEnvironment?: (input: {
+        label: string;
+        clientId: string;
+    }) => Promise<Environment>;
+    generator: AgentEvalTurnGenerator;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+interface AgentEvalConversation {
+    label: string;
+    environment: Environment;
+    history: AgentEvalHistoryEntry[];
+    promptEvents: PendingPromptEvent[];
+    artifactDir: string;
+    turnCount: number;
+}
+interface AgentEvalPendingTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    boundaryTimestamp: number;
+    finalCode: string;
+    finalReply: string;
+    stdout: string[];
+    stderr: string[];
+    prompts: Prompt[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    job: {
+        id: string;
+        result: Promise<unknown>;
+        on: (event: string, handler: (data: unknown) => void) => void;
+    };
+}
+interface AgentEvalCompletedTurn {
+    conversation: AgentEvalConversation;
+    request: string;
+    turnDir: string;
+    responseText: string;
+    terminalKind: "reply" | "closure";
+    finalCode?: string;
+    actionSummary: string[];
+    promptInteractions: AgentEvalPromptInteraction[];
+    verification: unknown | null;
+    result: unknown;
+}
+interface AgentEvalHistoryEntry {
+    role: "user" | "assistant";
+    content: string;
+    code?: string;
+    jobStatus?: string;
+    jobResultPreview?: string;
+    error?: string;
+}
+interface PendingPromptEvent {
+    prompt: Prompt;
+    receivedAt: number;
+}
+declare function createTimestampedArtifactDirectory(options?: {
+    baseDir?: string;
+    suiteName?: string;
+}): string;
+declare function createScriptedPromptResponder(rules: ScriptedPromptRule[], fallback?: AgentEvalPromptResponder): AgentEvalPromptResponder;
+declare function createOpenAIChatTurnGenerator(options: {
+    apiKey: string;
+    model?: string;
+    baseUrl?: string;
+    temperature?: number;
+    headers?: Record<string, string>;
+}): AgentEvalTurnGenerator;
+declare function runAgentEvalSuite(options: {
+    harness: ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare function createAgentEvalHarness(options: AgentEvalHarnessOptions): {
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+type AgentTestTarget = {
+    environmentId: string;
+} | {
+    connect: ConnectOptions;
+} | {
+    createEnvironment: {
+        sandboxId: string;
+        data: CreateEnvironmentData;
+    };
+};
+interface AgentTesterOptions {
+    granular?: Granular;
+    client?: Granular;
+    apiKey?: string;
+    token?: string;
+    apiUrl?: string;
+    local?: boolean;
+    granularOptions?: Omit<GranularOptions, "apiKey" | "token" | "apiUrl" | "endpointMode">;
+    generator?: AgentEvalTurnGenerator;
+    openai?: {
+        apiKey: string;
+        model?: string;
+        baseUrl?: string;
+        temperature?: number;
+        headers?: Record<string, string>;
+    };
+    model?: string;
+    target: AgentTestTarget;
+    artifactBaseDir?: string;
+    suiteName?: string;
+    controllerBudgets?: HarnessControllerBudgets;
+    chatTimeoutMs?: number;
+    jobTimeoutMs?: number;
+    pollIntervalMs?: number;
+}
+declare function createAgentTester(options: AgentTesterOptions): {
+    granular: Granular;
+    generator: AgentEvalTurnGenerator;
+    environmentId: string | null;
+    openThread: (label: string) => Promise<AgentEvalConversation>;
+    closeThread: (conversation: AgentEvalConversation) => Promise<void>;
+    runStep: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    continueStep: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    runSuite: (scenarios: AgentEvalScenario[]) => Promise<AgentEvalSuiteResult>;
+    artifactDir: string;
+    openConversation: (label: string) => Promise<AgentEvalConversation>;
+    closeConversation: (conversation: AgentEvalConversation) => Promise<void>;
+    runTurn: (input: {
+        conversation: AgentEvalConversation;
+        request: string;
+        prepareRecords?: RecordObjectOptions[];
+        prepareTools?: ToolWithHandler[];
+        prepare?: (ctx: AgentEvalPrepareContext) => Promise<void>;
+        human?: AgentEvalPromptResponder;
+        verification?: AgentEvalInspection;
+        maxIterations?: number;
+        autoAnswerPrompts?: boolean;
+    }) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+    resumePendingTurn: (pending: AgentEvalPendingTurn, responder: AgentEvalPromptResponder) => Promise<AgentEvalCompletedTurn | AgentEvalPendingTurn>;
+};
+declare function runAgentTests(options: {
+    tester: ReturnType<typeof createAgentTester> | ReturnType<typeof createAgentEvalHarness>;
+    scenarios: AgentEvalScenario[];
+}): Promise<AgentEvalSuiteResult>;
+declare const createHumanResponder: typeof createScriptedPromptResponder;
+declare const createOpenAIGenerator: typeof createOpenAIChatTurnGenerator;
+declare const createTestArtifactsDirectory: typeof createTimestampedArtifactDirectory;
+
+export { type AgentEvalCheck, type AgentEvalCheckContext, type AgentEvalCompletedTurn, type AgentEvalConversation, type AgentEvalExpectations, type AgentEvalGenerationInput, type AgentEvalGenerationOutput, type AgentEvalHarnessOptions, type AgentEvalInspection, type AgentEvalPendingTurn, type AgentEvalPrepareContext, type AgentEvalPromptInteraction, type AgentEvalPromptResponder, type AgentEvalResult, type AgentEvalScenario, type AgentEvalSetup, type AgentEvalSetupContext, type AgentEvalStep, type AgentEvalStepResult, type AgentEvalSuiteResult, type AgentEvalTurnGenerator, type AgentTestTarget, type AgentTesterOptions, type EvalMatcher, type ScriptedPromptRule, createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, runAgentEvalSuite, runAgentTests };